Multi-channel streaming

Author: Raphael Dumusc

apps/SimpleStreamer/main.cpp

@@ -1,5 +1,5 @@
 /*********************************************************************/
-/* Copyright (c) 2014-2017, EPFL/Blue Brain Project                  */
+/* Copyright (c) 2014-2018, EPFL/Blue Brain Project                  */
 /*                          Raphael Dumusc <raphael.dumusc@epfl.ch>  */
 /*                                                                   */
 /* Copyright (c) 2011 - 2012, The University of Texas at Austin.     */
@@ -60,11 +60,15 @@ bool deflectCompressImage = true;
 bool deflectStereoStreamLeft = false;
 bool deflectStereoStreamRight = false;
 bool deflectStereoSideBySide = false;
+bool deflectDualChannel = false;
 unsigned int deflectCompressionQuality = 75;
 std::string deflectHost;
 std::string deflectStreamId = "SimpleStreamer";
 std::unique_ptr<deflect::Stream> deflectStream;
 
+bool deflectFirstEventReceived = false;
+bool waitToStart = false;
+
 void syntax(int exitStatus);
 void readCommandLineArguments(int argc, char** argv);
 void initGLWindow(int argc, char** argv);
@@ -118,6 +122,8 @@ void syntax(const int exitStatus)
               << std::endl;
     std::cout << " -2                 enable side-by-side stereo (default: OFF)"
               << std::endl;
+    std::cout << " -c                 enable dual-channel mode (default: OFF)"
+              << std::endl;
     exit(exitStatus);
 }
 
@@ -158,6 +164,9 @@ void readCommandLineArguments(int argc, char** argv)
             case '2':
                 deflectStereoSideBySide = true;
                 break;
+            case 'c':
+                deflectDualChannel = true;
+                break;
             case 'h':
                 syntax(EXIT_SUCCESS);
             default:
@@ -280,7 +289,7 @@ struct Image
     }
 };
 
-bool send(const Image& image, const deflect::View view)
+bool send(const Image& image, const deflect::View view, const uint8_t channel)
 {
     deflect::ImageWrapper deflectImage(image.data.data(), image.width,
                                        image.height, deflect::RGBA);
@@ -290,6 +299,7 @@ bool send(const Image& image, const deflect::View view)
     deflectImage.compressionQuality = deflectCompressionQuality;
     deflectImage.view = view;
     deflectImage.rowOrder = deflect::RowOrder::bottom_up;
+    deflectImage.channel = channel;
     return deflectStream->send(deflectImage).get();
 }
 
@@ -321,22 +331,18 @@ void drawMono()
     glutSolidTeapot(1.f);
 }
 
-void display()
+bool drawAndSend(const uint8_t channel)
 {
-    static Camera camera;
-    camera.apply();
-
     bool success = false;
-    bool waitToStart = false;
-    static bool deflectFirstEventReceived = false;
+
     if (deflectStereoSideBySide)
     {
         drawLeft();
         const auto leftImage = Image::readGlBuffer();
         drawRight();
         const auto rightImage = Image::readGlBuffer();
         const auto sideBySideImage = Image::sideBySide(leftImage, rightImage);
-        success = send(sideBySideImage, deflect::View::side_by_side);
+        success = send(sideBySideImage, deflect::View::side_by_side, channel);
     }
     else if (deflectStereoStreamLeft || deflectStereoStreamRight)
     {
@@ -352,21 +358,41 @@ void display()
         {
             drawLeft();
             const auto leftImage = Image::readGlBuffer();
-            success = send(leftImage, deflect::View::left_eye);
+            success = send(leftImage, deflect::View::left_eye, channel);
         }
         if (deflectStereoStreamRight && !waitToStart)
         {
             drawRight();
             const auto rightImage = Image::readGlBuffer();
             success = (!deflectStereoStreamLeft || success) &&
-                      send(rightImage, deflect::View::right_eye);
+                      send(rightImage, deflect::View::right_eye, channel);
         }
     }
     else
     {
         drawMono();
-        success = send(Image::readGlBuffer(), deflect::View::mono);
+        success = send(Image::readGlBuffer(), deflect::View::mono, channel);
+    }
+
+    return success;
+}
+
+void display()
+{
+    static Camera camera;
+    camera.apply();
+
+    bool success = false;
+    if (deflectDualChannel)
+    {
+        glPushMatrix();
+        glRotatef(90, -1.f, 0.f, 0.f);
+        success = drawAndSend(1);
+        glPopMatrix();
+        success = success && drawAndSend(0);
     }
+    else
+        success = drawAndSend(0);
 
     if (!waitToStart)
         success = success && deflectStream->finishFrame().get();

deflect/ImageSegmenter.cpp

@@ -243,6 +243,7 @@ ImageSegmenter::SegmentTasks ImageSegmenter::_generateSegmentTasks(
         segment.view =
             image.view == View::side_by_side ? View::left_eye : image.view;
         segment.rowOrder = image.rowOrder;
+        segment.channel = image.channel;
         segment.sourceImage = ℑ
         segments.push_back(segment);
     }

deflect/ImageWrapper.h

@@ -152,6 +152,12 @@ struct ImageWrapper
      */
     RowOrder rowOrder = RowOrder::top_down;
 
+    /**
+     * The index of the channel which the image is a part of.
+     * @version 1.0
+     */
+    uint8_t channel = 0;
+
     /**
      * Get the number of bytes per pixel based on the pixelFormat.
      * @version 1.0

deflect/MessageHeader.h

@@ -69,7 +69,8 @@ enum MessageType
     MESSAGE_TYPE_DATA = 14,
     MESSAGE_TYPE_IMAGE_VIEW = 15,
     MESSAGE_TYPE_OBSERVER_OPEN = 16,
-    MESSAGE_TYPE_IMAGE_ROW_ORDER = 17
+    MESSAGE_TYPE_IMAGE_ROW_ORDER = 17,
+    MESSAGE_TYPE_IMAGE_CHANNEL = 18
 };
 
 #define MESSAGE_HEADER_URI_LENGTH 64

deflect/Segment.h

@@ -58,6 +58,7 @@ struct Segment
 
     View view = View::mono;                 //!< Eye pass for the segment
     RowOrder rowOrder = RowOrder::top_down; //!< Row order of imageData
+    uint8_t channel = 0;                    //!< Channel index for the segment
 };
 }
 

deflect/StreamSendWorker.cpp

@@ -199,6 +199,7 @@ bool StreamSendWorker::_sendSegment(const Segment& segment)
         _currentView = segment.view;
     }
     _sendRowOrderIfChanged(segment.rowOrder);
+    _sendImageChannelIfChanged(segment.channel);
 
     auto message = QByteArray{(const char*)(&segment.parameters),
                               sizeof(SegmentParameters)};
@@ -229,6 +230,23 @@ bool StreamSendWorker::_sendImageRowOrder(const RowOrder rowOrder)
                  QByteArray{(const char*)(&rowOrder), sizeof(RowOrder)});
 }
 
+bool StreamSendWorker::_sendImageChannelIfChanged(const uint8_t channel)
+{
+    if (channel != _currentChannel)
+    {
+        if (!_sendImageChannel(channel))
+            return false;
+        _currentChannel = channel;
+    }
+    return true;
+}
+
+bool StreamSendWorker::_sendImageChannel(const uint8_t channel)
+{
+    return _send(MESSAGE_TYPE_IMAGE_CHANNEL,
+                 QByteArray{(const char*)(&channel), sizeof(uint8_t)});
+}
+
 bool StreamSendWorker::_sendFinish()
 {
     return _send(MESSAGE_TYPE_PIXELSTREAM_FINISH_FRAME, {});

deflect/StreamSendWorker.h

@@ -106,6 +106,7 @@ class StreamSendWorker : public QThread
     bool _running = false;
     View _currentView = View::mono;
     RowOrder _currentRowOrder = RowOrder::top_down;
+    uint8_t _currentChannel = 0;
 
     std::vector<Request> _dequeuedRequests;
     bool _pendingFinish = false;
@@ -127,6 +128,8 @@ class StreamSendWorker : public QThread
     bool _sendImageView(View view);
     bool _sendRowOrderIfChanged(RowOrder rowOrder);
     bool _sendImageRowOrder(RowOrder rowOrder);
+    bool _sendImageChannelIfChanged(uint8_t channel);
+    bool _sendImageChannel(uint8_t channel);
     bool _sendFinish();
     bool _sendData(const QByteArray data);
     bool _sendSizeHints(const SizeHints& hints);

deflect/server/Frame.cpp

@@ -43,12 +43,15 @@ namespace deflect
 {
 namespace server
 {
-QSize Frame::computeDimensions() const
+QSize Frame::computeDimensions(const uint8_t channel) const
 {
     QSize size(0, 0);
 
     for (const auto& tile : tiles)
     {
+        if (tile.channel != channel)
+            continue;
+
         size.setWidth(std::max(size.width(), (int)(tile.width + tile.x)));
         size.setHeight(std::max(size.height(), (int)(tile.height + tile.y)));
     }

deflect/server/Frame.h

@@ -61,8 +61,8 @@ struct Frame
     /** The PixelStream uri to which this frame is associated. */
     QString uri;
 
-    /** @return the total dimensions of this frame. */
-    DEFLECT_API QSize computeDimensions() const;
+    /** @return the total dimensions of the given channel of this frame. */
+    DEFLECT_API QSize computeDimensions(const uint8_t channel = 0) const;
 
     /**
      * @return the row order of all frame tiles.

deflect/server/ServerWorker.cpp

@@ -274,6 +274,13 @@ void ServerWorker::_handleMessage(const MessageHeader& messageHeader,
         break;
     }
 
+    case MESSAGE_TYPE_IMAGE_CHANNEL:
+    {
+        const auto channel = reinterpret_cast<const uint8_t*>(byteArray.data());
+        _activeChannel = *channel;
+        break;
+    }
+
     case MESSAGE_TYPE_BIND_EVENTS:
     case MESSAGE_TYPE_BIND_EVENTS_EX:
         if (_registeredToEvents)
@@ -323,6 +330,7 @@ void ServerWorker::_handlePixelStreamMessage(const QByteArray& message)
     tile.imageData = message.right(message.size() - sizeof(SegmentParameters));
     tile.view = _activeView;
     tile.rowOrder = _activeRowOrder;
+    tile.channel = _activeChannel;
 
     emit(receivedTile(_streamId, _sourceId, tile));
 }