Skip to content

Commit

Permalink
Add videostreaming as an alternative to JPEG streaming to RocketsPlugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@tribal-tec
tribal-tec committed Sep 11, 2019
1 parent 54bf8e1 commit 91a1018
Show file tree
Hide file tree
Showing 8 changed files with 570 additions and 4 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ endif()
# HTTP messaging
common_find_package(LibJpegTurbo)
common_find_package(Rockets)
common_find_package(FFMPEG 3.4 SYSTEM)
if(ROCKETS_FOUND AND ROCKETS_USE_LIBWEBSOCKETS)
option(BRAYNS_NETWORKING_ENABLED "Activate networking interfaces" ON)
else()
Expand Down
11 changes: 10 additions & 1 deletion brayns/parameters/ApplicationParameters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const std::string PARAM_PLUGIN = "plugin";
const std::string PARAM_STEREO = "stereo";
const std::string PARAM_WINDOW_SIZE = "window-size";
const std::string PARAM_ENV_MAP = "env-map";
#ifdef BRAYNS_USE_FFMPEG
const std::string PARAM_VIDEOSTREAMING = "videostreaming";
#endif

const size_t DEFAULT_WINDOW_WIDTH = 800;
const size_t DEFAULT_WINDOW_HEIGHT = 600;
Expand Down Expand Up @@ -80,7 +83,13 @@ ApplicationParameters::ApplicationParameters()
(PARAM_MAX_RENDER_FPS.c_str(), po::value<size_t>(&_maxRenderFPS),
"Max. render FPS") //
(PARAM_ENV_MAP.c_str(), po::value<std::string>(&_envMap),
"Path to environment map");
"Path to environment map")
#ifdef BRAYNS_USE_FFMPEG
(PARAM_VIDEOSTREAMING.c_str(),
po::bool_switch(&_useVideoStreaming)->default_value(false),
"Use videostreaming over websockets instead of JPEG")
#endif
;

_positionalArgs.add(PARAM_INPUT_PATHS.c_str(), -1);
}
Expand Down
5 changes: 3 additions & 2 deletions brayns/parameters/ApplicationParameters.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ class ApplicationParameters : public AbstractParameters
_updateValue(_imageStreamFPS, fps);
}

bool useVideoStreaming() const { return _useVideoStreaming; }
/** Max render FPS to limit */
size_t getMaxRenderFPS() const { return _maxRenderFPS; }
bool isStereo() const { return _stereo; }
Expand All @@ -88,8 +89,7 @@ class ApplicationParameters : public AbstractParameters
_updateValue(_httpServerURI, httpServerURI);
}

const std::string& getEnvMap() const {return _envMap;}

const std::string& getEnvMap() const { return _envMap; }
const strings& getInputPaths() const { return _inputPaths; }
po::positional_options_description& posArgs() { return _positionalArgs; }
protected:
Expand All @@ -106,6 +106,7 @@ class ApplicationParameters : public AbstractParameters
std::string _httpServerURI;
bool _parallelRendering{false};
bool _dynamicLoadBalancer{false};
bool _useVideoStreaming{false};
std::string _envMap;

strings _inputPaths;
Expand Down
9 changes: 9 additions & 0 deletions plugins/Rockets/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ set(BRAYNSROCKETS_LINK_LIBRARIES PRIVATE Rockets braynsParameters braynsTasks
if(LibJpegTurbo_FOUND)
list(APPEND BRAYNSROCKETS_LINK_LIBRARIES PRIVATE ${LibJpegTurbo_LIBRARIES})
endif()
if(FFMPEG_FOUND)
list(APPEND BRAYNSROCKETS_HEADERS encoder.h)
list(APPEND BRAYNSROCKETS_SOURCES encoder.cpp)
list(APPEND BRAYNSROCKETS_LINK_LIBRARIES PRIVATE ${FFMPEG_LIBRARIES})
endif()

if(libuv_FOUND)
list(APPEND BRAYNSROCKETS_LINK_LIBRARIES PRIVATE ${libuv_LIBRARIES})
Expand All @@ -44,6 +49,10 @@ set(BRAYNSROCKETS_INCLUDE_NAME rocketsplugin)
common_library(braynsRockets)
target_include_directories(braynsRockets SYSTEM PRIVATE ${FREEIMAGE_INCLUDE_DIRS})

if(FFMPEG_FOUND)
target_include_directories(braynsRockets SYSTEM PRIVATE ${FFMPEG_INCLUDE_DIR})
endif()

# needed for staticjson and rapidjson
target_include_directories(braynsRockets SYSTEM PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
Expand Down
142 changes: 141 additions & 1 deletion plugins/Rockets/RocketsPlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@
#include "ImageGenerator.h"
#include "Throttle.h"

#ifdef BRAYNS_USE_FFMPEG
#include "encoder.h"
#endif

namespace
{
constexpr int64_t INTERACTIVE_THROTTLE = 1;
Expand All @@ -56,6 +60,8 @@ const int INSTANCE_NOT_FOUND = -12346;
const int TASK_RESULT_TO_JSON_ERROR = -12347;
const int SCHEMA_RPC_ENDPOINT_NOT_FOUND = -12348;
const int PARAMETER_FROM_JSON_ERROR = -12349;
const int VIDEOSTREAMING_NOT_SUPPORTED = -12350;
const int VIDEOSTREAMING_NOT_ENABLED = -12351;

// REST PUT & GET, JSONRPC set-* notification, JSONRPC get-* request
const std::string ENDPOINT_ANIMATION_PARAMS = "animation-parameters";
Expand Down Expand Up @@ -85,6 +91,7 @@ const std::string METHOD_GET_LOADERS = "get-loaders";
const std::string METHOD_GET_MODEL_PROPERTIES = "get-model-properties";
const std::string METHOD_GET_MODEL_TRANSFER_FUNCTION =
"get-model-transfer-function";
const std::string METHOD_GET_VIDEOSTREAM = "get-videostream";
const std::string METHOD_IMAGE_JPEG = "image-jpeg";
const std::string METHOD_INSPECT = "inspect";
const std::string METHOD_MODEL_PROPERTIES_SCHEMA = "model-properties-schema";
Expand All @@ -95,6 +102,7 @@ const std::string METHOD_SET_ENVIRONMENT_MAP = "set-environment-map";
const std::string METHOD_SET_MODEL_PROPERTIES = "set-model-properties";
const std::string METHOD_SET_MODEL_TRANSFER_FUNCTION =
"set-model-transfer-function";
const std::string METHOD_SET_VIDEOSTREAM = "set-videostream";
const std::string METHOD_UPDATE_CLIP_PLANE = "update-clip-plane";
const std::string METHOD_UPDATE_INSTANCE = "update-instance";
const std::string METHOD_UPDATE_MODEL = "update-model";
Expand Down Expand Up @@ -148,6 +156,13 @@ std::string getRequestEndpointName(const std::string& endpoint)
{
return "get-" + endpoint;
}

const Response::Error VIDEOSTREAM_NOT_ENABLED_ERROR{
"Brayns was not started with videostream support enabled",
VIDEOSTREAMING_NOT_ENABLED};
const Response::Error VIDEOSTREAM_NOT_SUPPORTED_ERROR{
"Brayns was not build with videostream support",
VIDEOSTREAMING_NOT_SUPPORTED};
} // namespace

namespace brayns
Expand Down Expand Up @@ -266,7 +281,12 @@ class RocketsPlugin::Impl : public ActionInterface
if (!_rocketsServer || _rocketsServer->getConnectionCount() == 0)
return;

_broadcastImageJpeg();
if (!_parametersManager.getApplicationParameters().useVideoStreaming())
_broadcastImageJpeg();
#ifdef BRAYNS_USE_FFMPEG
else
_broadcastVideo();
#endif
}

void registerNotification(const RpcParameterDescription& desc,
Expand Down Expand Up @@ -960,6 +980,9 @@ class RocketsPlugin::Impl : public ActionInterface
_handleSetEnvironmentMap();
_handleGetEnvironmentMap();

_handleSetVideostream();
_handleGetVideostream();

_handleAddModel();
_handleRemoveModel();
_handleUpdateModel();
Expand Down Expand Up @@ -1041,6 +1064,58 @@ class RocketsPlugin::Impl : public ActionInterface
image.size);
}

#ifdef BRAYNS_USE_FFMPEG
void _broadcastVideo()
{
if (!_videoParams.enabled)
{
_encoder.reset();
if (_videoUpdatedResponse)
_videoUpdatedResponse();
_videoUpdatedResponse = nullptr;
return;
}

const auto& params = _parametersManager.getApplicationParameters();
const auto fps = params.getImageStreamFPS();
if (fps == 0)
return;

if (_encoder && _encoder->kbps != _videoParams.kbps)
_encoder.reset();

auto& frameBuffer = _engine.getFrameBuffer();
if (!_encoder)
{
int width = frameBuffer.getFrameSize().x;
if (width % 2 != 0)
width += 1;
int height = frameBuffer.getFrameSize().y;
if (height % 2 != 0)
height += 1;

_encoder =
std::make_unique<Encoder>(width, height, fps, _videoParams.kbps,
[& rs = _rocketsServer](auto a,
auto b) {
rs->broadcastBinary(a, b);
});
}

if (_videoUpdatedResponse)
_videoUpdatedResponse();
_videoUpdatedResponse = nullptr;

if (frameBuffer.getFrameBufferFormat() == FrameBufferFormat::none ||
!frameBuffer.isModified())
{
return;
}

_encoder->encode(frameBuffer);
}
#endif

void _handleVersion()
{
static brayns::Version version;
Expand Down Expand Up @@ -1798,6 +1873,65 @@ class RocketsPlugin::Impl : public ActionInterface
});
}

void _handleSetVideostream()
{
const RpcParameterDescription desc{METHOD_SET_VIDEOSTREAM,
"Set the video streaming parameters",
"params", "videostream parameters"};

auto action = [&](const VideoStreamParam& params BRAYNS_UNUSED,
auto clientID BRAYNS_UNUSED, auto respond, auto) {
if (!_parametersManager.getApplicationParameters()
.useVideoStreaming())
{
respond(
Response{Response::Error(VIDEOSTREAM_NOT_ENABLED_ERROR)});
return rockets::jsonrpc::CancelRequestCallback();
}

#ifdef BRAYNS_USE_FFMPEG
this->_rebroadcast(METHOD_SET_VIDEOSTREAM, to_json(params),
{clientID});

const bool changed = params != _videoParams;
if (!changed)
{
respond(Response{to_json(false)});
return rockets::jsonrpc::CancelRequestCallback();
}

_engine.triggerRender();
_videoParams = params;
_videoUpdatedResponse = [respond] {
respond(Response{to_json(true)});
};
return rockets::jsonrpc::CancelRequestCallback();
#else
respond(Response{Response::Error(VIDEOSTREAM_NOT_SUPPORTED_ERROR)});
return rockets::jsonrpc::CancelRequestCallback();
#endif
};
_handleAsyncRPC<VideoStreamParam, bool>(desc, action);
}

void _handleGetVideostream()
{
const RpcDescription desc{METHOD_GET_VIDEOSTREAM,
"Get the videostream parameters"};

_handleRPC<VideoStreamParam>(desc, [&]() -> VideoStreamParam {
#ifdef BRAYNS_USE_FFMPEG
if (!_parametersManager.getApplicationParameters()
.useVideoStreaming())
throw rockets::jsonrpc::response_error(
VIDEOSTREAM_NOT_ENABLED_ERROR);
return _videoParams;
#else
throw rockets::jsonrpc::response_error(VIDEOSTREAM_NOT_SUPPORTED_ERROR);
#endif
});
}

Engine& _engine;

std::unordered_map<std::string, std::pair<std::mutex, Throttle>> _throttle;
Expand Down Expand Up @@ -1834,6 +1968,12 @@ class RocketsPlugin::Impl : public ActionInterface
bool _endpointsRegistered{false};

std::vector<BaseObject*> _objects;

#ifdef BRAYNS_USE_FFMPEG
std::unique_ptr<Encoder> _encoder;
VideoStreamParam _videoParams;
std::function<void()> _videoUpdatedResponse;
#endif
};

RocketsPlugin::~RocketsPlugin()
Expand Down
Loading

0 comments on commit 91a1018

Please sign in to comment.