From 4320cc78d9117a1788b8d8fa10165ae00b120262 Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Tue, 1 Sep 2026 18:59:20 +0800 Subject: [PATCH] feat: camera via webrtc --- CMakeLists.txt | 43 ++ deps/CMakeLists.txt | 3 + deps/DataChannel/DataChannel.cmake | 20 + src/slic3r/CMakeLists.txt | 9 +- src/slic3r/GUI/IMediaController.hpp | 11 + src/slic3r/GUI/MediaPlayCtrl.cpp | 110 ++++- src/slic3r/GUI/MediaPlayCtrl.h | 6 + src/slic3r/GUI/StatusPanel.cpp | 14 +- src/slic3r/GUI/WebRtcFrameAssembler.cpp | 86 ++++ src/slic3r/GUI/WebRtcFrameAssembler.hpp | 39 ++ src/slic3r/GUI/WebRtcMediaController.cpp | 449 ++++++++++++++++++ src/slic3r/GUI/WebRtcMediaController.hpp | 96 ++++ src/slic3r/GUI/wxMediaCtrl3.cpp | 62 +++ src/slic3r/GUI/wxMediaCtrl3.h | 13 +- src/slic3r/Utils/ICameraSignalingChannel.hpp | 40 ++ src/slic3r/Utils/IPrinterAgent.hpp | 10 + src/slic3r/Utils/NetworkAgent.cpp | 8 + src/slic3r/Utils/NetworkAgent.hpp | 1 + .../Utils/OrcaCloudSignalingChannel.cpp | 320 +++++++++++++ .../Utils/OrcaCloudSignalingChannel.hpp | 63 +++ src/slic3r/Utils/OrcaPrinterAgent.cpp | 35 ++ src/slic3r/Utils/OrcaPrinterAgent.hpp | 7 + tests/slic3rutils/CMakeLists.txt | 1 + .../test_webrtc_frame_assembler.cpp | 67 +++ 24 files changed, 1505 insertions(+), 8 deletions(-) create mode 100644 deps/DataChannel/DataChannel.cmake create mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.cpp create mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.hpp create mode 100644 src/slic3r/GUI/WebRtcMediaController.cpp create mode 100644 src/slic3r/GUI/WebRtcMediaController.hpp create mode 100644 src/slic3r/Utils/ICameraSignalingChannel.hpp create mode 100644 src/slic3r/Utils/OrcaCloudSignalingChannel.cpp create mode 100644 src/slic3r/Utils/OrcaCloudSignalingChannel.hpp create mode 100644 tests/slic3rutils/test_webrtc_frame_assembler.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b954ef753..457a7b96c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -785,6 +785,49 @@ find_package(OpenSSL REQUIRED) find_package(CURL REQUIRED) find_package(Freetype REQUIRED) +if (SLIC3R_GUI) + # LibDataChannel's installed export references its bundled dependencies, + # but does not install their CMake targets. Recreate those targets from + # the same dependency prefix before loading the LibDataChannel config. + if (NOT TARGET Usrsctp::usrsctp) + find_library(_ORCA_USRSCTP_LIBRARY NAMES usrsctp + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_USRSCTP_LIBRARY) + add_library(Usrsctp::usrsctp UNKNOWN IMPORTED GLOBAL) + set_target_properties(Usrsctp::usrsctp PROPERTIES + IMPORTED_LOCATION "${_ORCA_USRSCTP_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "Threads::Threads") + endif() + endif() + + if (NOT TARGET libSRTP::srtp2) + find_library(_ORCA_SRTP_LIBRARY NAMES srtp2 + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_SRTP_LIBRARY) + add_library(libSRTP::srtp2 UNKNOWN IMPORTED GLOBAL) + set_target_properties(libSRTP::srtp2 PROPERTIES + IMPORTED_LOCATION "${_ORCA_SRTP_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "OpenSSL::Crypto") + endif() + endif() + + if (NOT TARGET LibJuice::LibJuice) + find_library(_ORCA_LIBJUICE_LIBRARY NAMES juice + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_LIBJUICE_LIBRARY) + add_library(LibJuice::LibJuice UNKNOWN IMPORTED GLOBAL) + set_target_properties(LibJuice::LibJuice PROPERTIES + IMPORTED_LOCATION "${_ORCA_LIBJUICE_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "Threads::Threads") + endif() + endif() + + find_package(LibDataChannel CONFIG REQUIRED) +endif() + add_library(libcurl INTERFACE) target_link_libraries(libcurl INTERFACE CURL::libcurl) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index ed3af70d03..a645e3a752 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -389,6 +389,8 @@ if(NOT OPENSSL_FOUND) set(OPENSSL_PKG dep_OpenSSL) endif() +include(DataChannel/DataChannel.cmake) + # we don't want to load a "wrong" openssl when loading curl # so, just don't even bother # ...i think this is how it works? change if wrong @@ -461,6 +463,7 @@ set(_dep_list dep_wxInspector dep_FFMPEG dep_Assimp + dep_DataChannel ) if (MSVC) diff --git a/deps/DataChannel/DataChannel.cmake b/deps/DataChannel/DataChannel.cmake new file mode 100644 index 0000000000..5e9b9a358b --- /dev/null +++ b/deps/DataChannel/DataChannel.cmake @@ -0,0 +1,20 @@ +# libdatachannel is the native ICE/DTLS/SCTP/SRTP implementation used by the +# GUI WebRTC camera controller. Keep the source revision fixed: the signaling +# protocol is evolving independently of this transport dependency. +orcaslicer_add_cmake_project(DataChannel + CMAKE_ARGS + -DNO_EXAMPLES=ON + -DNO_TESTS=ON + -DNO_WEBSOCKET=ON + -DNO_MEDIA=OFF + -DUSE_NICE=OFF + -DUSE_SYSTEM_SRTP=OFF + -DUSE_SYSTEM_JUICE=OFF + -DUSE_SYSTEM_USRSCTP=OFF + -DOPENSSL_ROOT_DIR:PATH=${DESTDIR} + -DOPENSSL_USE_STATIC_LIBS=ON + GIT_REPOSITORY https://github.com/paullouisageneau/libdatachannel.git + GIT_TAG v0.22.2 + GIT_SHALLOW ON + GIT_SUBMODULES_RECURSE ON +) diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 8308161a7b..322b3b3f0e 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -344,6 +344,10 @@ set(SLIC3R_GUI_SOURCES GUI/MediaFilePanel.h GUI/MediaPlayCtrl.cpp GUI/MediaPlayCtrl.h + GUI/WebRtcFrameAssembler.cpp + GUI/WebRtcFrameAssembler.hpp + GUI/WebRtcMediaController.cpp + GUI/WebRtcMediaController.hpp GUI/MeshUtils.cpp GUI/MeshUtils.hpp GUI/ModelMall.cpp @@ -723,10 +727,13 @@ set(SLIC3R_GUI_SOURCES Utils/NetworkAgentFactory.cpp Utils/ICloudServiceAgent.hpp Utils/IPrinterAgent.hpp + Utils/ICameraSignalingChannel.hpp Utils/OrcaCloudServiceAgent.cpp Utils/OrcaCloudServiceAgent.hpp Utils/OrcaPrinterAgent.cpp Utils/OrcaPrinterAgent.hpp + Utils/OrcaCloudSignalingChannel.cpp + Utils/OrcaCloudSignalingChannel.hpp Utils/QidiPrinterAgent.cpp Utils/QidiPrinterAgent.hpp Utils/SnapmakerPrinterAgent.cpp @@ -855,7 +862,7 @@ else() set(_opengl_link_lib OpenGL::GL) endif() -target_link_libraries(libslic3r_gui libslic3r cereal::cereal imgui imguizmo minilzo libvgcode md4c-html glad ${_opengl_link_lib} hidapi mdns ${wxWidgets_LIBRARIES} glfw libcurl OpenSSL::SSL OpenSSL::Crypto noise::noise pybind11::embed) +target_link_libraries(libslic3r_gui libslic3r cereal::cereal imgui imguizmo minilzo libvgcode md4c-html glad ${_opengl_link_lib} hidapi mdns ${wxWidgets_LIBRARIES} glfw libcurl OpenSSL::SSL OpenSSL::Crypto LibDataChannel::LibDataChannel noise::noise pybind11::embed) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") # Linux finds wxWidgets in module mode, whose include dirs and definitions diff --git a/src/slic3r/GUI/IMediaController.hpp b/src/slic3r/GUI/IMediaController.hpp index 982157e5bc..411932ec7d 100644 --- a/src/slic3r/GUI/IMediaController.hpp +++ b/src/slic3r/GUI/IMediaController.hpp @@ -3,6 +3,8 @@ #include #include +#include + #include namespace Slic3r { namespace GUI { @@ -10,6 +12,8 @@ namespace Slic3r { namespace GUI { class IMediaController { public: + virtual ~IMediaController() = default; + virtual void Load(wxURI url) = 0; // The default keeps existing media controllers unaware of camera-specific modes. @@ -29,6 +33,13 @@ public: virtual wxSize GetVideoSize() const { return {}; }; + virtual void StartSession(std::unique_ptr channel) + { + (void) channel; + } + + virtual void StopSession() {} + private: }; diff --git a/src/slic3r/GUI/MediaPlayCtrl.cpp b/src/slic3r/GUI/MediaPlayCtrl.cpp index 77a183f090..9b7198005a 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.cpp +++ b/src/slic3r/GUI/MediaPlayCtrl.cpp @@ -134,6 +134,11 @@ MediaPlayCtrl::MediaPlayCtrl(wxWindow *parent, wxMediaCtrl3 *media_ctrl, const w MediaPlayCtrl::~MediaPlayCtrl() { + m_webrtc_stopping = true; + if (m_webrtc_ctrl) + m_webrtc_ctrl->StopSession(); + m_media_ctrl->EndExternalStream(); + m_webrtc_stopping = false; { boost::unique_lock lock(m_mutex); m_tasks.push_back(""); @@ -159,7 +164,14 @@ CameraStreamMode MediaPlayCtrl::current_mode() const void MediaPlayCtrl::SetMachineObject(MachineObject* obj) { - switch (current_mode()) { + const CameraStreamMode mode = current_mode(); + if (mode != m_last_mode) { + if (m_last_state != MEDIASTATE_IDLE) + Stop(" "); + m_last_mode = mode; + } + + switch (mode) { case CameraStreamMode::http: case CameraStreamMode::http_snapshot: case CameraStreamMode::rtsp: { @@ -184,6 +196,28 @@ void MediaPlayCtrl::SetMachineObject(MachineObject* obj) Play(); return; } + case CameraStreamMode::webrtc: { + std::string machine = obj ? obj->get_dev_id() : ""; + m_camera_exists = obj != nullptr; + Enable(obj != nullptr); + const bool changed = machine != m_machine; + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::SetMachineObject webrtc: changed=" << changed + << " last_state=" << m_last_state << " web_user_stopped=" << m_web_user_stopped; + m_machine = machine; + m_url.clear(); + m_agent_camera_url.clear(); + if (!changed) { + if (m_last_state == MEDIASTATE_IDLE && IsEnabled() && !m_web_user_stopped) + Play(); + return; + } + m_web_user_stopped = false; + if (m_last_state != MEDIASTATE_IDLE) + Stop(" "); + if (IsEnabled()) + Play(); + return; + } default: break; } @@ -321,6 +355,48 @@ void MediaPlayCtrl::Play() m_button_play->SetIcon("media_stop"); load(); return; + case CameraStreamMode::webrtc: { + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Play webrtc: last_state=" << m_last_state + << " next_retry_valid=" << m_next_retry.IsValid() + << " next_retry_future=" << (m_next_retry.IsValid() && wxDateTime::Now() < m_next_retry) + << " failed_retry=" << m_failed_retry << " shown=" << IsShownOnScreen(); + if (m_webrtc_ctrl && m_webrtc_ctrl->is_active()) { + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Play webrtc: session already active, ignoring"; + return; + } + if (m_next_retry.IsValid() && wxDateTime::Now() < m_next_retry) + return; + if (!IsShownOnScreen() || m_last_state != MEDIASTATE_IDLE) + return; + m_failed_code = 0; + if (m_machine.empty() || !IsEnabled() || !m_camera_exists) { + Stop(_L("Please confirm if the printer is connected.")); + return; + } + auto agent = wxGetApp().getAgent(); + auto channel = agent ? agent->create_camera_signaling_channel(m_machine) : nullptr; + if (!channel) { + Stop(_L("Sign in to OrcaCloud to view the camera.")); + return; + } + if (!m_webrtc_ctrl) { + m_webrtc_ctrl = std::make_unique( + [this](const wxImage& image, wxSize size) { m_media_ctrl->SetExternalFrame(image, size); }, + [this, token = std::weak_ptr(m_token)](WebRtcMediaController::Status status) { + if (token.expired()) + return; + CallAfter([this, status] { on_webrtc_status(status); }); + }); + } + m_button_play->SetIcon("media_stop"); + m_media_ctrl->BeginExternalStream(); + m_last_state = MEDIASTATE_INITIALIZING; + SetStatus(_L("Initializing..."), false); + m_webrtc_stopping = false; + m_webrtc_ctrl->StartSession(std::move(channel)); + m_webrtc_epoch = m_webrtc_ctrl->epoch(); + return; + } default: break; } @@ -465,6 +541,17 @@ void MediaPlayCtrl::StopWebStream() void MediaPlayCtrl::Stop(wxString const &msg, wxString const &msg2) { + const bool webrtc_active = m_webrtc_ctrl && (m_last_mode == CameraStreamMode::webrtc || + current_mode() == CameraStreamMode::webrtc); + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Stop: last_state=" << m_last_state + << " webrtc_active=" << webrtc_active << " failed_code=" << m_failed_code + << " msg='" << msg.ToUTF8().data() << "'"; + if (webrtc_active) { + m_webrtc_stopping = true; + m_webrtc_ctrl->StopSession(); + m_media_ctrl->EndExternalStream(); + m_webrtc_stopping = false; + } switch (current_mode()) { case CameraStreamMode::http: case CameraStreamMode::http_snapshot: @@ -555,6 +642,27 @@ void MediaPlayCtrl::Stop(wxString const &msg, wxString const &msg2) m_next_retry = wxDateTime::Now() + wxTimeSpan::Seconds(5 * m_failed_retry); } +void MediaPlayCtrl::on_webrtc_status(WebRtcMediaController::Status status) +{ + // Drop CallAfter-queued events from a superseded StartSession attempt. + if (status.epoch != m_webrtc_epoch) + return; + if (status.kind == WebRtcMediaController::Status::Connecting) { + m_last_state = MEDIASTATE_INITIALIZING; + SetStatus(_L("Initializing..."), false); + } else if (status.kind == WebRtcMediaController::Status::Playing) { + m_last_state = wxMEDIASTATE_PLAYING; + m_failed_code = 0; + m_failed_retry = 0; + SetStatus(_L("Playing..."), false); + } else if (status.kind == WebRtcMediaController::Status::Failed) { + m_failed_code = static_cast(status.code) + 1; + Stop(); + } + // Status::Stopped needs no action: a genuine failure arrives as Failed, and + // a stop we initiated is already handled by Stop() itself. +} + void MediaPlayCtrl::TogglePlay() { BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::TogglePlay"; diff --git a/src/slic3r/GUI/MediaPlayCtrl.h b/src/slic3r/GUI/MediaPlayCtrl.h index e8ad0ee82c..64fb75498e 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.h +++ b/src/slic3r/GUI/MediaPlayCtrl.h @@ -10,6 +10,7 @@ #include "wxMediaCtrl3.h" #include "IMediaController.hpp" +#include "WebRtcMediaController.hpp" #include "slic3r/Utils/IPrinterAgent.hpp" #include @@ -60,6 +61,7 @@ protected: void TogglePlay(); void SetStatus(wxString const &msg, bool hyperlink = true); + void on_webrtc_status(WebRtcMediaController::Status status); private: void load(); @@ -85,6 +87,10 @@ private: wxMediaCtrl3 * m_media_ctrl; IMediaController * m_web_ctrl = nullptr; + std::unique_ptr m_webrtc_ctrl; + CameraStreamMode m_last_mode = CameraStreamMode::none; + bool m_webrtc_stopping = false; + std::uint64_t m_webrtc_epoch = 0; std::string m_agent_camera_url; bool m_web_user_stopped = false; wxMediaState m_last_state = MEDIASTATE_IDLE; diff --git a/src/slic3r/GUI/StatusPanel.cpp b/src/slic3r/GUI/StatusPanel.cpp index b00dee3dca..2d267df082 100644 --- a/src/slic3r/GUI/StatusPanel.cpp +++ b/src/slic3r/GUI/StatusPanel.cpp @@ -2317,10 +2317,16 @@ void StatusPanel::update_camera_state(MachineObject* obj) m_custom_camera_view->Show(); m_media_ctrl->Hide(); } - } else if (m_custom_camera_view->IsShown()) { - m_custom_camera_view->Hide(); - m_media_ctrl->Show(); - m_media_play_ctrl->StopWebStream(); + } else if (camera_mode == CameraStreamMode::rtsp || camera_mode == CameraStreamMode::webrtc || + m_custom_camera_view->IsShown()) { + // Only act on the actual transition away from the webview. Running this + // every tick would call StopWebStream() (which forces m_last_state to + // IDLE) on a live rtsp/webrtc session and desync the state machine. + if (m_custom_camera_view->IsShown()) { + m_custom_camera_view->Hide(); + m_media_ctrl->Show(); + m_media_play_ctrl->StopWebStream(); + } } //sdcard diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.cpp b/src/slic3r/GUI/WebRtcFrameAssembler.cpp new file mode 100644 index 0000000000..2771cc0887 --- /dev/null +++ b/src/slic3r/GUI/WebRtcFrameAssembler.cpp @@ -0,0 +1,86 @@ +#include "WebRtcFrameAssembler.hpp" + +#include +#include + +namespace Slic3r { namespace GUI { + +void WebRtcFrameAssembler::discard() +{ + m_active = false; + m_frame_id = 0; + m_chunk_count = 0; + m_received_chunks = 0; + m_total_size = 0; + m_chunks.clear(); + m_received.clear(); +} + +void WebRtcFrameAssembler::reset() +{ + discard(); +} + +void WebRtcFrameAssembler::feed(const std::byte* data, std::size_t len) +{ + if (data == nullptr || len < HeaderSize) + return; + + if (std::to_integer(data[0]) != 1) + return; + + const std::uint16_t chunk_index = static_cast((std::to_integer(data[2]) << 8) | + std::to_integer(data[3])); + const std::uint16_t chunk_count = static_cast((std::to_integer(data[4]) << 8) | + std::to_integer(data[5])); + const std::uint32_t frame_id = (static_cast(std::to_integer(data[6])) << 24) | + (static_cast(std::to_integer(data[7])) << 16) | + (static_cast(std::to_integer(data[8])) << 8) | + static_cast(std::to_integer(data[9])); + const std::size_t payload_size = len - HeaderSize; + + if (chunk_count == 0 || chunk_count > MaxChunkCount || chunk_index >= chunk_count || + payload_size > MaxChunkPayload) + return; + + if (!m_active || frame_id > m_frame_id) { + discard(); + m_active = true; + m_frame_id = frame_id; + m_chunk_count = chunk_count; + m_chunks.resize(chunk_count); + m_received.assign(chunk_count, false); + } else if (frame_id < m_frame_id) { + return; + } else if (chunk_count != m_chunk_count) { + discard(); + return; + } + + Frame& chunk = m_chunks[chunk_index]; + if (m_received[chunk_index]) + return; + + if (m_total_size > MaxFrameSize || payload_size > MaxFrameSize - m_total_size) { + discard(); + return; + } + + chunk.assign(data + HeaderSize, data + len); + m_received[chunk_index] = true; + m_total_size += payload_size; + ++m_received_chunks; + + if (m_received_chunks != m_chunk_count) + return; + + Frame frame; + frame.reserve(m_total_size); + for (const Frame& slice : m_chunks) + frame.insert(frame.end(), slice.begin(), slice.end()); + if (on_frame) + on_frame(std::move(frame)); + discard(); +} + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.hpp b/src/slic3r/GUI/WebRtcFrameAssembler.hpp new file mode 100644 index 0000000000..c526cad588 --- /dev/null +++ b/src/slic3r/GUI/WebRtcFrameAssembler.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +namespace Slic3r { namespace GUI { + +class WebRtcFrameAssembler { +public: + using Frame = std::vector; + + // The wire protocol uses a 10-byte header followed by one JPEG slice: + // version, flags, chunk index, chunk count, and frame id, all in network + // byte order where applicable. + static constexpr std::size_t HeaderSize = 10; + static constexpr std::size_t MaxChunkPayload = 16000; + static constexpr std::size_t MaxChunkCount = 4096; + static constexpr std::size_t MaxFrameSize = 8 * 1024 * 1024; + + std::function on_frame; + + void feed(const std::byte* data, std::size_t len); + void reset(); + +private: + void discard(); + + bool m_active = false; + std::uint32_t m_frame_id = 0; + std::uint16_t m_chunk_count = 0; + std::size_t m_received_chunks = 0; + std::size_t m_total_size = 0; + std::vector m_chunks; + std::vector m_received; +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.cpp b/src/slic3r/GUI/WebRtcMediaController.cpp new file mode 100644 index 0000000000..7eafbd4a3a --- /dev/null +++ b/src/slic3r/GUI/WebRtcMediaController.cpp @@ -0,0 +1,449 @@ +#include "WebRtcMediaController.hpp" + +#include "AVVideoDecoder.hpp" + +#include +#include + +#include +#include +#include + +#include + +#include + +namespace { +void init_rtc_logger_once() +{ + static std::once_flag flag; + std::call_once(flag, [] { + rtc::InitLogger(rtc::LogLevel::Verbose, [](rtc::LogLevel level, std::string message) { + BOOST_LOG_TRIVIAL(info) << "[rtc:" << static_cast(level) << "] " << message; + }); + }); +} +} // namespace + +extern "C" { +#include +} + +namespace Slic3r { namespace GUI { + +WebRtcMediaController::WebRtcMediaController(std::function frame_sink, + std::function on_status) + : m_frame_sink(std::move(frame_sink)) + , m_on_status(std::move(on_status)) +{ +} + +WebRtcMediaController::~WebRtcMediaController() +{ + StopSession(); +} + +void WebRtcMediaController::report(Status status) +{ + status.epoch = m_epoch.load(); + BOOST_LOG_TRIVIAL(info) << "WebRTC: report kind=" << static_cast(status.kind) + << " code=" << static_cast(status.code) << " epoch=" << status.epoch; + { + std::lock_guard lock(m_mutex); + if (status.kind == Status::Connecting) + m_state = static_cast(4); + else if (status.kind == Status::Playing) + m_state = wxMEDIASTATE_PLAYING; + else + m_state = static_cast(3); + } + if (m_on_status) + m_on_status(status); +} + +void WebRtcMediaController::StartSession(std::unique_ptr channel) +{ + // Tear down any previous attempt WITHOUT notifying: the Stopped that would + // otherwise be delivered (async, via CallAfter) races the new attempt's + // Connecting and makes the consumer cancel a session that is mid-connect. + teardown(false); + if (!channel) + return; + + m_epoch.fetch_add(1); + m_alive.store(true); + { + std::lock_guard lock(m_mutex); + m_signaling = std::move(channel); + m_chunk_queue.clear(); + m_nal_queue.clear(); + m_pending_candidates.clear(); + m_remote_description_set = false; + m_video_size = wxDefaultSize; + m_has_frame = false; + m_last_frame_time = {}; + } + + ICameraSignalingChannel* signaling = nullptr; + { + std::lock_guard lock(m_mutex); + signaling = m_signaling.get(); + } + signaling->on_ready = [this](std::vector servers) { + if (m_alive.load()) + on_ready(std::move(servers)); + }; + signaling->on_answer = [this](std::string sdp) { + if (m_alive.load()) + on_answer(std::move(sdp)); + }; + signaling->on_ice = [this](std::string candidate, std::string mid) { + if (m_alive.load()) + on_ice(std::move(candidate), std::move(mid)); + }; + signaling->on_unavailable = [this](CameraUnavailableReason reason, std::string detail) { + if (m_alive.load()) + on_unavailable(reason, std::move(detail)); + }; + + m_decode_thread = std::thread([this] { decode_loop(); }); + report({Status::Connecting}); + signaling->open(); +} + +void WebRtcMediaController::StopSession() +{ + teardown(true); +} + +void WebRtcMediaController::teardown(bool notify) +{ + const bool was_alive = m_alive.exchange(false); + if (!was_alive && !m_decode_thread.joinable()) + return; + + m_cond.notify_all(); + std::unique_ptr signaling; + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + signaling = std::move(m_signaling); + peer_connection = std::move(m_peer_connection); + m_data_channel.reset(); + m_video_track.reset(); + } + if (signaling) + signaling->close(); + if (peer_connection) + peer_connection->close(); + if (m_decode_thread.joinable()) + m_decode_thread.join(); + { + std::lock_guard lock(m_mutex); + m_chunk_queue.clear(); + m_nal_queue.clear(); + } + if (was_alive && notify) + report({Status::Stopped}); +} + +wxMediaState WebRtcMediaController::GetState() +{ + std::lock_guard lock(m_mutex); + return m_state; +} + +wxSize WebRtcMediaController::GetVideoSize() const +{ + std::lock_guard lock(m_mutex); + return m_video_size; +} + +void WebRtcMediaController::bind_data_channel(const std::shared_ptr& dc) +{ + const std::string label = dc->label(); + dc->onOpen([this, label] { BOOST_LOG_TRIVIAL(info) << "WebRTC: data channel '" << label << "' open"; }); + dc->onClosed([this, label] { BOOST_LOG_TRIVIAL(info) << "WebRTC: data channel '" << label << "' closed"; }); + dc->onError([label](std::string e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: data channel '" << label << "' error: " << e; + }); + dc->onMessage( + [this](rtc::binary data) { + if (m_alive.load()) + enqueue_chunk(std::vector(data.begin(), data.end())); + }, + [](rtc::string) {}); +} + +void WebRtcMediaController::on_ready(std::vector servers) +{ + init_rtc_logger_once(); + rtc::Configuration configuration; + for (const CameraIceServer& server : servers) { + try { + rtc::IceServer ice_server(server.urls); + ice_server.username = server.username; + ice_server.password = server.credential; + configuration.iceServers.emplace_back(std::move(ice_server)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: invalid ICE server: " << e.what(); + } + } + configuration.iceServers.emplace_back("stun:stun.cloudflare.com:3478"); + configuration.iceServers.emplace_back("stun:stun.l.google.com:19302"); + + auto peer_connection = std::make_shared(std::move(configuration)); + BOOST_LOG_TRIVIAL(info) << "WebRTC: creating peer connection with " << configuration.iceServers.size() + << " ice servers"; + peer_connection->onLocalDescription([this](rtc::Description description) { + if (!m_alive.load()) + return; + const std::string sdp(description); + BOOST_LOG_TRIVIAL(info) << "WebRTC: local description ready (" << description.typeString() + << "), OFFER SDP:\n" << sdp; + std::lock_guard lock(m_mutex); + if (m_signaling) + m_signaling->send_offer(sdp); + }); + peer_connection->onLocalCandidate([this](rtc::Candidate candidate) { + if (!m_alive.load()) + return; + std::lock_guard lock(m_mutex); + if (m_signaling) + m_signaling->send_ice(std::string(candidate), candidate.mid()); + }); + peer_connection->onStateChange([this](rtc::PeerConnection::State state) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: peer state -> " << static_cast(state); + if (!m_alive.load()) + return; + if (state == rtc::PeerConnection::State::Failed || state == rtc::PeerConnection::State::Disconnected) + report({Status::Failed, Status::ICE_FAILED}); + }); + peer_connection->onGatheringStateChange([](rtc::PeerConnection::GatheringState state) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: gathering state -> " << static_cast(state); + }); + + // Accept a DataChannel opened by the remote peer (OrcaSonar may create the + // "camera" channel from its side rather than answering the one we offer). + peer_connection->onDataChannel([this](std::shared_ptr dc) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: remote opened data channel '" << dc->label() << "'"; + bind_data_channel(dc); + std::lock_guard lock(m_mutex); + m_data_channel = std::move(dc); + }); + + rtc::DataChannelInit init; + init.reliability.unordered = true; + init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + auto data_channel = peer_connection->createDataChannel("camera", init); + if (data_channel) + bind_data_channel(data_channel); + + // NOTE: the H.264 RTP RecvOnly track is intentionally NOT added to the offer + // yet. OrcaSonar answers application-only (no m=video), which makes + // libdatachannel renegotiate and send a second offer that OrcaSonar rejects + // with "webrtc.unavailable: error". Re-add the video m-line (enqueue_nal / + // the decode_loop NAL branch are already in place) once OrcaSonar answers it. + + std::shared_ptr peer_for_description; + { + std::lock_guard lock(m_mutex); + if (!m_alive.load()) + return; + m_peer_connection = std::move(peer_connection); + peer_for_description = m_peer_connection; + m_data_channel = std::move(data_channel); + } + if (peer_for_description) + peer_for_description->setLocalDescription(); +} + +void WebRtcMediaController::on_answer(std::string sdp) +{ + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + peer_connection = m_peer_connection; + } + BOOST_LOG_TRIVIAL(info) << "WebRTC: applying remote answer (" << sdp.size() << " bytes), ANSWER SDP:\n" << sdp; + if (!peer_connection) + return; + try { + peer_connection->setRemoteDescription(rtc::Description(sdp, "answer")); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: setRemoteDescription failed: " << e.what(); + report({Status::Failed, Status::ICE_FAILED}); + return; + } + + // Flush any remote candidates that arrived before the answer. + std::vector> pending; + { + std::lock_guard lock(m_mutex); + m_remote_description_set = true; + pending.swap(m_pending_candidates); + } + BOOST_LOG_TRIVIAL(info) << "WebRTC: remote description set, flushing " << pending.size() + << " buffered candidate(s)"; + for (const auto& c : pending) { + try { + peer_connection->addRemoteCandidate(rtc::Candidate(c.first, c.second)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: addRemoteCandidate (buffered) failed: " << e.what(); + } + } +} + +void WebRtcMediaController::on_ice(std::string candidate, std::string mid) +{ + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + if (!m_remote_description_set) { + m_pending_candidates.emplace_back(std::move(candidate), std::move(mid)); + return; + } + peer_connection = m_peer_connection; + } + if (!peer_connection) + return; + try { + peer_connection->addRemoteCandidate(rtc::Candidate(candidate, mid)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: addRemoteCandidate failed: " << e.what(); + } +} + +void WebRtcMediaController::on_unavailable(CameraUnavailableReason reason, std::string detail) +{ + BOOST_LOG_TRIVIAL(warning) << "WebRTC camera unavailable: " << detail; + Status::Code code = Status::UNAVAILABLE_ERROR; + if (reason == CameraUnavailableReason::Busy) + code = Status::UNAVAILABLE_BUSY; + else if (reason == CameraUnavailableReason::Disabled) + code = Status::UNAVAILABLE_DISABLED; + else if (reason == CameraUnavailableReason::Closed) + code = Status::SIGNALING_CLOSED; + report({Status::Failed, code}); +} + +void WebRtcMediaController::enqueue_chunk(std::vector chunk) +{ + std::lock_guard lock(m_mutex); + if (m_chunk_queue.size() >= 8) + m_chunk_queue.pop_front(); + m_chunk_queue.emplace_back(std::move(chunk)); + m_cond.notify_one(); +} + +void WebRtcMediaController::enqueue_nal(std::vector nal) +{ + std::lock_guard lock(m_mutex); + if (m_nal_queue.size() >= 4) + m_nal_queue.pop_front(); + m_nal_queue.emplace_back(std::move(nal)); + m_cond.notify_one(); +} + +void WebRtcMediaController::deliver_jpeg(std::vector jpeg) +{ + const auto now = std::chrono::steady_clock::now(); + { + std::lock_guard lock(m_mutex); + if (m_last_frame_time != std::chrono::steady_clock::time_point{} && + now - m_last_frame_time < std::chrono::milliseconds(33)) + return; + m_last_frame_time = now; + } + wxMemoryInputStream stream(jpeg.data(), jpeg.size()); + wxImage image; + if (!image.LoadFile(stream, wxBITMAP_TYPE_JPEG)) { + report({Status::Failed, Status::DECODE_ERROR}); + return; + } + bool first_frame = false; + { + std::lock_guard lock(m_mutex); + m_video_size = image.GetSize(); + first_frame = !m_has_frame; + m_has_frame = true; + } + if (m_frame_sink) + m_frame_sink(image, image.GetSize()); + if (first_frame) + report({Status::Playing}); +} + +void WebRtcMediaController::decode_loop() +{ + WebRtcFrameAssembler assembler; + assembler.on_frame = [this](std::vector jpeg) { deliver_jpeg(std::move(jpeg)); }; + AVCodecParameters parameters{}; + parameters.codec_type = AVMEDIA_TYPE_VIDEO; + parameters.codec_id = AV_CODEC_ID_H264; + AVVideoDecoder decoder; + bool decoder_open = false; + + int stall_polls = 0; + std::unique_lock lock(m_mutex); + while (m_alive.load()) { + const bool woke = m_cond.wait_for(lock, std::chrono::seconds(2), [this] { + return !m_alive.load() || !m_chunk_queue.empty() || !m_nal_queue.empty(); + }); + if (!m_alive.load()) + break; + if (!woke && !m_has_frame) { + const int pc_state = m_peer_connection ? static_cast(m_peer_connection->state()) : -1; + std::string dc = "none"; + if (m_data_channel) + dc = "label='" + m_data_channel->label() + "' open=" + + (m_data_channel->isOpen() ? "1" : "0"); + lock.unlock(); + BOOST_LOG_TRIVIAL(info) << "WebRTC: waiting for frames; peer_state=" << pc_state + << " data_channel=" << dc; + if (++stall_polls >= 8) { // ~16s connected with no frame -> give up so the UI can retry + report({Status::Failed, Status::TIMEOUT}); + lock.lock(); + break; + } + lock.lock(); + continue; + } + stall_polls = 0; + if (!m_chunk_queue.empty()) { + auto chunk = std::move(m_chunk_queue.front()); + m_chunk_queue.pop_front(); + lock.unlock(); + assembler.feed(chunk.data(), chunk.size()); + lock.lock(); + } else if (!m_nal_queue.empty()) { + auto nal = std::move(m_nal_queue.front()); + m_nal_queue.pop_front(); + lock.unlock(); + if (!decoder_open) + decoder_open = decoder.open(parameters) == 0; + if (decoder_open) { + AVPacket* packet = av_packet_alloc(); + if (packet && av_new_packet(packet, static_cast(nal.size())) == 0) { + std::memcpy(packet->data, nal.data(), nal.size()); + if (decoder.decode(*packet) == 0) { + wxImage image; + if (decoder.toWxImage(image, wxDefaultSize)) { + { + std::lock_guard frame_lock(m_mutex); + m_video_size = image.GetSize(); + } + if (m_frame_sink) + m_frame_sink(image, image.GetSize()); + report({Status::Playing}); + } + } + } + av_packet_free(&packet); + } + lock.lock(); + } + } +} + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.hpp b/src/slic3r/GUI/WebRtcMediaController.hpp new file mode 100644 index 0000000000..1cbd05fcd7 --- /dev/null +++ b/src/slic3r/GUI/WebRtcMediaController.hpp @@ -0,0 +1,96 @@ +#pragma once + +#include "IMediaController.hpp" +#include "WebRtcFrameAssembler.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rtc { +class DataChannel; +class PeerConnection; +class Track; +} + +namespace Slic3r { namespace GUI { + +class WebRtcMediaController : public IMediaController { +public: + struct Status { + enum Kind { Connecting, Playing, Stopped, Failed } kind = Stopped; + enum Code { + ICE_FAILED, + SIGNALING_CLOSED, + UNAVAILABLE_BUSY, + UNAVAILABLE_ERROR, + UNAVAILABLE_DISABLED, + DECODE_ERROR, + TIMEOUT, + } code = ICE_FAILED; + // Identifies the StartSession attempt this status belongs to, so the + // consumer can drop CallAfter-queued events from a superseded attempt. + std::uint64_t epoch = 0; + }; + + WebRtcMediaController(std::function frame_sink, + std::function on_status); + ~WebRtcMediaController() override; + + void StartSession(std::unique_ptr channel) override; + void StopSession() override; + std::uint64_t epoch() const { return m_epoch.load(); } + bool is_active() const { return m_alive.load(); } + + void Load(wxURI) override {} + void Play() override {} + void Stop() override { StopSession(); } + wxMediaState GetState() override; + wxSize GetVideoSize() const override; + +private: + void teardown(bool notify); + void report(Status status); + void bind_data_channel(const std::shared_ptr& dc); + void on_ready(std::vector servers); + void on_answer(std::string sdp); + void on_ice(std::string candidate, std::string mid); + void on_unavailable(CameraUnavailableReason reason, std::string detail); + void decode_loop(); + void enqueue_chunk(std::vector chunk); + void enqueue_nal(std::vector nal); + void deliver_jpeg(std::vector jpeg); + + mutable std::mutex m_mutex; + std::condition_variable m_cond; + std::deque> m_chunk_queue; + std::deque> m_nal_queue; + // Remote candidates can arrive before the answer; libdatachannel rejects + // addRemoteCandidate until a remote description is set, so buffer them. + std::vector> m_pending_candidates; + bool m_remote_description_set = false; + std::unique_ptr m_signaling; + std::shared_ptr m_peer_connection; + std::shared_ptr m_data_channel; + std::shared_ptr m_video_track; + std::thread m_decode_thread; + std::atomic m_alive{false}; + std::atomic m_epoch{0}; + wxMediaState m_state = static_cast(3); + wxSize m_video_size = wxDefaultSize; + std::function m_frame_sink; + std::function m_on_status; + bool m_has_frame = false; + std::chrono::steady_clock::time_point m_last_frame_time{}; +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/wxMediaCtrl3.cpp b/src/slic3r/GUI/wxMediaCtrl3.cpp index d0d53072d6..2d6412afe4 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.cpp +++ b/src/slic3r/GUI/wxMediaCtrl3.cpp @@ -46,9 +46,13 @@ wxMediaCtrl3::~wxMediaCtrl3() m_thread.join(); } +static void adjust_frame_size(wxSize& frame, wxSize const& video, wxSize const& window); + void wxMediaCtrl3::Load(wxURI url) { std::unique_lock lk(m_mutex); + if (m_external) + return; m_video_size = wxDefaultSize; m_error = 0; m_url.reset(new wxURI(url)); @@ -58,6 +62,8 @@ void wxMediaCtrl3::Load(wxURI url) void wxMediaCtrl3::Play() { std::unique_lock lk(m_mutex); + if (m_external) + return; if (m_state != wxMEDIASTATE_PLAYING) { m_state = wxMEDIASTATE_PLAYING; wxMediaEvent event(wxEVT_MEDIA_STATECHANGED); @@ -77,6 +83,62 @@ void wxMediaCtrl3::Stop() Refresh(); } +void wxMediaCtrl3::SetExternalFrame(const wxImage& frame, wxSize videoSize) +{ + if (!frame.IsOk()) + return; + { + std::unique_lock lk(m_mutex); + if (!m_external) + return; + m_frame = frame; + m_video_size = videoSize.IsFullySpecified() ? videoSize : frame.GetSize(); + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + } + CallAfter([this] { Refresh(); }); +} + +#ifdef _WIN32 +void wxMediaCtrl3::SetExternalFrame(const wxBitmap& frame, wxSize videoSize) +{ + if (!frame.IsOk()) + return; + { + std::unique_lock lk(m_mutex); + if (!m_external) + return; + m_frame = frame; + m_video_size = videoSize.IsFullySpecified() ? videoSize : frame.GetSize(); + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + } + CallAfter([this] { Refresh(); }); +} +#endif + +void wxMediaCtrl3::BeginExternalStream() +{ + std::unique_lock lk(m_mutex); + m_external = true; + m_url.reset(); + m_active_url.reset(); + m_video_size = wxDefaultSize; + m_frame = wxImage(m_idle_image); + m_cond.notify_all(); + Refresh(); +} + +void wxMediaCtrl3::EndExternalStream() +{ + std::unique_lock lk(m_mutex); + m_external = false; + m_url.reset(); + m_active_url.reset(); + m_video_size = wxDefaultSize; + m_frame = wxImage(m_idle_image); + m_cond.notify_all(); + Refresh(); +} + void wxMediaCtrl3::SetIdleImage(wxString const &image) { if (m_idle_image == image) diff --git a/src/slic3r/GUI/wxMediaCtrl3.h b/src/slic3r/GUI/wxMediaCtrl3.h index bcc94a17ef..dee49ddbe8 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.h +++ b/src/slic3r/GUI/wxMediaCtrl3.h @@ -18,9 +18,7 @@ void wxMediaCtrl_OnSize(wxWindow * ctrl, wxSize const & videoSize, int width, in #define BAMBU_DYNAMIC #include #include -#ifndef _WIN32 #include -#endif #include "Printer/BambuTunnel.h" class AVVideoDecoder; @@ -38,6 +36,16 @@ public: void Stop(); + // Render frames supplied by a controller which owns its own transport. + // The frame is copied while m_mutex is held; callers may release it after + // this method returns. + void SetExternalFrame(const wxImage& frame, wxSize videoSize); +#ifdef _WIN32 + void SetExternalFrame(const wxBitmap& frame, wxSize videoSize); +#endif + void BeginExternalStream(); + void EndExternalStream(); + void SetIdleImage(wxString const & image); wxMediaState GetState(); @@ -77,6 +85,7 @@ private: std::shared_ptr m_url; std::shared_ptr m_active_url; + bool m_external = false; std::uint64_t m_last_PTS{0}; std::chrono::system_clock::time_point m_last_PTS_expected; std::chrono::system_clock::time_point m_last_PTS_practical; diff --git a/src/slic3r/Utils/ICameraSignalingChannel.hpp b/src/slic3r/Utils/ICameraSignalingChannel.hpp new file mode 100644 index 0000000000..70df298843 --- /dev/null +++ b/src/slic3r/Utils/ICameraSignalingChannel.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include + +namespace Slic3r { + +struct CameraIceServer { + std::string urls; + std::string username; + std::string credential; +}; + +enum class CameraUnavailableReason { + Busy, + Error, + Disabled, + Closed, +}; + +class ICameraSignalingChannel { +public: + virtual ~ICameraSignalingChannel() = default; + + virtual void open() = 0; + virtual void close() = 0; + virtual void send_offer(std::string sdp) = 0; + virtual void send_ice(std::string candidate, std::string mid) = 0; + + // These callbacks are invoked by the channel's worker thread. Consumers + // must marshal UI work to the GUI thread themselves. + std::function)> on_ready; + std::function on_answer; + std::function on_ice; + std::function on_unavailable; +}; + +} // namespace Slic3r diff --git a/src/slic3r/Utils/IPrinterAgent.hpp b/src/slic3r/Utils/IPrinterAgent.hpp index c076b57be5..240b00395b 100644 --- a/src/slic3r/Utils/IPrinterAgent.hpp +++ b/src/slic3r/Utils/IPrinterAgent.hpp @@ -14,6 +14,7 @@ #include #include #include +#include "ICameraSignalingChannel.hpp" #if 1 @@ -382,6 +383,15 @@ public: * Only meaningful when get_camera_stream_mode() returns an HTTP or RTSP mode. */ virtual std::string get_camera_url() const { return {}; } + + // Optional native camera signaling. Plugin agents retain the default + // nullptr until a plugin-facing WebRTC contract is defined. + virtual std::unique_ptr + create_camera_signaling_channel(const std::string& dev_id) + { + (void) dev_id; + return nullptr; + } }; } // namespace Slic3r diff --git a/src/slic3r/Utils/NetworkAgent.cpp b/src/slic3r/Utils/NetworkAgent.cpp index 01ed03f84f..4857151c58 100644 --- a/src/slic3r/Utils/NetworkAgent.cpp +++ b/src/slic3r/Utils/NetworkAgent.cpp @@ -1040,6 +1040,14 @@ std::string NetworkAgent::get_local_camera_stream_url() const return {}; } +std::unique_ptr +NetworkAgent::create_camera_signaling_channel(const std::string& dev_id) +{ + if (m_printer_agent) + return m_printer_agent->create_camera_signaling_channel(dev_id); + return nullptr; +} + int NetworkAgent::request_bind_ticket(std::string* ticket) { if (m_printer_agent) diff --git a/src/slic3r/Utils/NetworkAgent.hpp b/src/slic3r/Utils/NetworkAgent.hpp index 7eb220b341..a051bdd231 100644 --- a/src/slic3r/Utils/NetworkAgent.hpp +++ b/src/slic3r/Utils/NetworkAgent.hpp @@ -183,6 +183,7 @@ public: bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull); CameraStreamMode get_camera_stream_mode() const; std::string get_local_camera_stream_url() const; + std::unique_ptr create_camera_signaling_channel(const std::string& dev_id); int request_bind_ticket(std::string* ticket); int get_hms_snapshot(std::string dev_id, std::string file_name, std::function callback); diff --git a/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp b/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp new file mode 100644 index 0000000000..9c3a029db3 --- /dev/null +++ b/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp @@ -0,0 +1,320 @@ +#include "OrcaCloudSignalingChannel.hpp" + +#include "Http.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace Slic3r { + +OrcaCloudSignalingChannel::OrcaCloudSignalingChannel(std::shared_ptr cloud, std::string dev_id) + : m_cloud(std::move(cloud)) + , m_dev_id(std::move(dev_id)) +{ +} + +OrcaCloudSignalingChannel::~OrcaCloudSignalingChannel() +{ + close(); +} + +void OrcaCloudSignalingChannel::open() +{ + bool expected = false; + if (!m_open.compare_exchange_strong(expected, true)) + return; + m_stop.store(false); + m_thread = std::thread([this] { run(); }); +} + +void OrcaCloudSignalingChannel::close() +{ + m_stop.store(true); + std::shared_ptr conn; + { + std::lock_guard lock(m_mutex); + conn = m_conn; + } + if (conn) { + // Established session: close the socket on the io_context's own thread so + // the pending async_read completes and io_context.run() unwinds. + boost::asio::post(conn->io_context, [conn] { + boost::system::error_code ec; + boost::beast::get_lowest_layer(conn->websocket).cancel(ec); + boost::beast::get_lowest_layer(conn->websocket).close(ec); + }); + // Pre-run() phase (still in the synchronous connect/handshake): best-effort + // direct interruption. + boost::system::error_code ec; + boost::beast::get_lowest_layer(conn->websocket).cancel(ec); + boost::beast::get_lowest_layer(conn->websocket).close(ec); + } + if (m_thread.joinable()) + m_thread.join(); + m_open.store(false); +} + +void OrcaCloudSignalingChannel::send_offer(std::string sdp) +{ + send_json(nlohmann::json{{"type", "webrtc.offer"}, {"sdp", std::move(sdp)}}.dump()); +} + +void OrcaCloudSignalingChannel::send_ice(std::string candidate, std::string mid) +{ + send_json(nlohmann::json{{"type", "webrtc.ice"}, + {"candidate", std::move(candidate)}, + {"sdpMid", std::move(mid)}} + .dump()); +} + +std::string OrcaCloudSignalingChannel::encode_path_component(const std::string& value) +{ + std::ostringstream encoded; + encoded << std::uppercase << std::hex; + for (unsigned char c : value) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') + encoded << c; + else + encoded << '%' << std::setw(2) << std::setfill('0') << static_cast(c); + } + return encoded.str(); +} + +std::string OrcaCloudSignalingChannel::host_without_scheme(std::string value) +{ + const auto scheme = value.find("://"); + if (scheme != std::string::npos) + value.erase(0, scheme + 3); + const auto slash = value.find('/'); + if (slash != std::string::npos) + value.erase(slash); + return value; +} + +void OrcaCloudSignalingChannel::unavailable(CameraUnavailableReason reason, std::string detail) +{ + if (on_unavailable) + on_unavailable(reason, std::move(detail)); +} + +void OrcaCloudSignalingChannel::run() +{ + try { + if (!m_cloud || !m_cloud->ensure_token_fresh("camera")) { + unavailable(CameraUnavailableReason::Error, "Unable to refresh OrcaCloud credentials"); + m_open.store(false); + return; + } + const std::string token = m_cloud->get_access_token(); + const std::string host = host_without_scheme(m_cloud->get_cloud_service_host()); + if (token.empty() || host.empty()) { + unavailable(CameraUnavailableReason::Error, "OrcaCloud session is unavailable"); + m_open.store(false); + return; + } + + const std::string live_token_url = + "https://" + host + "/api/v1/printers/" + encode_path_component(m_dev_id) + "/live-token"; + BOOST_LOG_TRIVIAL(info) << "signaling: POST " << live_token_url << " (dev_id=" << m_dev_id << ")"; + + nlohmann::json token_response; + std::string token_body; + std::string token_error; + unsigned int http_code = 0; + auto request = Http::post(live_token_url); + request.set_post_body(std::string("{}")) + .header("Authorization", "Bearer " + token) + .header("Content-Type", "application/json") + .tls_verify(true) + .timeout_max(30) + .on_complete([&token_body, &http_code](std::string body, unsigned status) { + http_code = status; + token_body = std::move(body); + }) + .on_error([&token_body, &token_error, &http_code](std::string body, std::string error, unsigned status) { + http_code = status; + token_body = std::move(body); + token_error = std::move(error); + }) + .perform_sync(); + BOOST_LOG_TRIVIAL(info) << "signaling: live-token HTTP " << http_code + << (token_error.empty() ? "" : " error=" + token_error) + << " body=" << token_body.substr(0, 512); + try { + token_response = nlohmann::json::parse(token_body); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: live-token body is not JSON: " << e.what(); + } + if (http_code < 200 || http_code >= 300 || !token_response.contains("token")) { + unavailable(CameraUnavailableReason::Error, "Unable to mint camera live token"); + m_open.store(false); + return; + } + + std::vector ice_servers; + if (token_response.contains("ice_servers") && token_response["ice_servers"].is_array()) { + for (const auto& entry : token_response["ice_servers"]) { + if (entry.is_string()) { + ice_servers.push_back({entry.get(), {}, {}}); + } else if (entry.is_object()) { + // RTCIceServer.urls is "string | string[]" (Cloudflare + // Realtime returns an array). Emit one CameraIceServer per + // URL, sharing the credentials. + const std::string username = entry.value("username", std::string{}); + const std::string credential = entry.value("credential", std::string{}); + const auto add_url = [&](const nlohmann::json& url) { + if (url.is_string() && !url.get().empty()) + ice_servers.push_back({url.get(), username, credential}); + }; + const auto urls = entry.find("urls"); + if (urls != entry.end()) { + if (urls->is_array()) { + for (const auto& url : *urls) + add_url(url); + } else { + add_url(*urls); + } + } + } + } + } + + auto conn = std::make_shared(); + conn->ssl_context.set_default_verify_paths(); + { + std::lock_guard lock(m_mutex); + m_conn = conn; + } + auto& websocket = conn->websocket; + boost::asio::ip::tcp::resolver resolver(conn->io_context); + const auto endpoints = resolver.resolve(host, "443"); + boost::asio::connect(boost::beast::get_lowest_layer(websocket), endpoints); + if (!SSL_set_tlsext_host_name(websocket.next_layer().native_handle(), host.c_str())) + throw std::runtime_error("Unable to configure TLS server name"); + websocket.next_layer().set_verify_mode(boost::asio::ssl::verify_peer); + websocket.next_layer().handshake(boost::asio::ssl::stream_base::client); + const std::string ws_target = "/api/v1/printers/" + encode_path_component(m_dev_id) + + "/camera/live?token=" + + encode_path_component(token_response["token"].get()); + websocket.handshake(host, ws_target); + BOOST_LOG_TRIVIAL(info) << "signaling: websocket handshake ok (" << ice_servers.size() + << " ice servers)"; + + if (on_ready) + on_ready(std::move(ice_servers)); + send_json(nlohmann::json{{"type", "camera.mode"}, {"mode", "webrtc"}}.dump()); + // Async read loop, driven by the connection's own io_context. run() + // returns once close() has shut the socket down, giving a bounded, + // deadlock-free teardown from any thread. + do_read(conn); + conn->io_context.run(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: run() exception: " << e.what(); + if (!m_stop.load()) + unavailable(CameraUnavailableReason::Closed, e.what()); + } + { + std::lock_guard lock(m_mutex); + m_conn.reset(); + } + m_open.store(false); +} + +void OrcaCloudSignalingChannel::do_read(std::shared_ptr conn) +{ + auto buffer = std::make_shared(); + conn->websocket.async_read( + *buffer, [this, conn, buffer](boost::system::error_code ec, std::size_t) { + if (ec) { + if (!m_stop.load()) + unavailable(CameraUnavailableReason::Closed, ec.message()); + return; // do not re-arm; io_context.run() unwinds + } + const std::string raw = boost::beast::buffers_to_string(buffer->data()); + // A malformed or unexpectedly-shaped message must not tear down the + // session: parse/dispatch is guarded. + try { + dispatch_message(nlohmann::json::parse(raw), raw); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: ignoring malformed message: " << e.what() + << " raw=" << raw.substr(0, 256); + } + if (!m_stop.load()) + do_read(conn); + }); +} + +// Returns the string at key, or "" if absent or not a string (JSON null included). +static std::string json_string(const nlohmann::json& object, const char* key) +{ + const auto it = object.find(key); + return (it != object.end() && it->is_string()) ? it->get() : std::string{}; +} + +void OrcaCloudSignalingChannel::dispatch_message(const nlohmann::json& message, const std::string& raw) +{ + const std::string type = json_string(message, "type"); + BOOST_LOG_TRIVIAL(info) << "signaling: recv type=" << type << " raw=" << raw.substr(0, 256); + if (type == "webrtc.answer") { + const std::string sdp = json_string(message, "sdp"); + if (on_answer && !sdp.empty()) + on_answer(sdp); + } else if (type == "webrtc.ice" && message.contains("candidate")) { + // The peer may send "candidate" as a flat string or as a nested + // RTCIceCandidateInit object { candidate, sdpMid, sdpMLineIndex }. + const nlohmann::json& candidate = message["candidate"]; + std::string sdp_candidate; + std::string mid = json_string(message, "sdpMid"); + if (candidate.is_string()) { + sdp_candidate = candidate.get(); + } else if (candidate.is_object()) { + sdp_candidate = json_string(candidate, "candidate"); + std::string nested_mid = json_string(candidate, "sdpMid"); + if (!nested_mid.empty()) + mid = std::move(nested_mid); + } + if (on_ice && !sdp_candidate.empty()) + on_ice(sdp_candidate, mid); + } else if (type == "webrtc.unavailable") { + const std::string reason = json_string(message, "reason"); + unavailable(reason == "busy" ? CameraUnavailableReason::Busy + : reason == "disabled" ? CameraUnavailableReason::Disabled + : CameraUnavailableReason::Error, + reason.empty() ? "error" : reason); + } +} + +void OrcaCloudSignalingChannel::send_json(const std::string& message) +{ + std::shared_ptr conn; + { + std::lock_guard lock(m_mutex); + conn = m_conn; + } + if (!conn || m_stop.load()) + return; + // Serialize the write onto the io_context thread (same thread that runs + // async_read), so reads and writes never touch the stream concurrently. + auto payload = std::make_shared(message); + boost::asio::post(conn->io_context, [this, conn, payload] { + if (m_stop.load()) + return; + boost::system::error_code ec; + conn->websocket.write(boost::asio::buffer(*payload), ec); + if (ec && !m_stop.load()) + unavailable(CameraUnavailableReason::Closed, ec.message()); + }); +} + +} // namespace Slic3r diff --git a/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp b/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp new file mode 100644 index 0000000000..b87b7400e2 --- /dev/null +++ b/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include "ICameraSignalingChannel.hpp" +#include "ICloudServiceAgent.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace Slic3r { + +class OrcaCloudSignalingChannel : public ICameraSignalingChannel { +public: + OrcaCloudSignalingChannel(std::shared_ptr cloud, std::string dev_id); + ~OrcaCloudSignalingChannel() override; + + void open() override; + void close() override; + void send_offer(std::string sdp) override; + void send_ice(std::string candidate, std::string mid) override; + +private: + using WebSocket = boost::beast::websocket::stream< + boost::beast::ssl_stream>; + + // The io_context and ssl_context must outlive the websocket stream that + // references them. Bundling them here with the stream declared last makes + // the destruction order correct (stream first, then contexts), and lets a + // single shared_ptr own the whole set. + struct Connection { + boost::asio::io_context io_context; + boost::asio::ssl::context ssl_context{boost::asio::ssl::context::tls_client}; + WebSocket websocket{io_context, ssl_context}; + }; + + void run(); + void do_read(std::shared_ptr conn); + void dispatch_message(const nlohmann::json& message, const std::string& raw); + void send_json(const std::string& message); + void unavailable(CameraUnavailableReason reason, std::string detail); + static std::string encode_path_component(const std::string& value); + static std::string host_without_scheme(std::string value); + + std::shared_ptr m_cloud; + std::string m_dev_id; + std::atomic m_stop{false}; + std::atomic m_open{false}; + std::thread m_thread; + mutable std::mutex m_mutex; + std::shared_ptr m_conn; +}; + +} // namespace Slic3r diff --git a/src/slic3r/Utils/OrcaPrinterAgent.cpp b/src/slic3r/Utils/OrcaPrinterAgent.cpp index 93cadd9abd..520686db3a 100644 --- a/src/slic3r/Utils/OrcaPrinterAgent.cpp +++ b/src/slic3r/Utils/OrcaPrinterAgent.cpp @@ -1,4 +1,5 @@ #include "OrcaPrinterAgent.hpp" +#include "OrcaCloudSignalingChannel.hpp" #include "NetworkAgentFactory.hpp" #include "OrcaCloudServiceAgent.hpp" #include @@ -47,6 +48,31 @@ void OrcaPrinterAgent::set_cloud_agent(std::shared_ptr cloud BOOST_LOG_TRIVIAL(info) << "OrcaPrinterAgent::set_cloud_agent: status callback result=" << callback_result; } +CameraStreamMode OrcaPrinterAgent::get_camera_stream_mode() const +{ + std::lock_guard lock(state_mutex); + if (m_lan_connected && !m_lan_rtsp_url.empty()) + return CameraStreamMode::rtsp; + if (m_cloud_agent && m_cloud_agent->is_user_login()) + return CameraStreamMode::webrtc; + return CameraStreamMode::none; +} + +std::string OrcaPrinterAgent::get_camera_url() const +{ + std::lock_guard lock(state_mutex); + return m_lan_connected ? m_lan_rtsp_url : std::string{}; +} + +std::unique_ptr +OrcaPrinterAgent::create_camera_signaling_channel(const std::string& dev_id) +{ + std::lock_guard lock(state_mutex); + if (!m_cloud_agent) + return nullptr; + return std::make_unique(m_cloud_agent, dev_id); +} + // ============================================================================ // Communication - All Stubs // ============================================================================ @@ -86,11 +112,20 @@ int OrcaPrinterAgent::send_message(std::string dev_id, std::string json_str, int int OrcaPrinterAgent::connect_printer(std::string dev_id, std::string dev_ip, std::string username, std::string password, bool use_ssl) { + std::lock_guard lock(state_mutex); + m_lan_connected = !dev_ip.empty(); + // OrcaPrinterAgent currently has no LAN status-push decoder. Preserve the + // standard OrcaSonar endpoint convention until the reported RTSP field is + // available, while keeping the URL behind the connection-aware mode API. + m_lan_rtsp_url = m_lan_connected ? "rtsp://" + dev_ip + ":8554/stream" : std::string{}; return BAMBU_NETWORK_SUCCESS; } int OrcaPrinterAgent::disconnect_printer() { + std::lock_guard lock(state_mutex); + m_lan_connected = false; + m_lan_rtsp_url.clear(); return BAMBU_NETWORK_SUCCESS; } diff --git a/src/slic3r/Utils/OrcaPrinterAgent.hpp b/src/slic3r/Utils/OrcaPrinterAgent.hpp index 9cf2b29638..00d7e372f6 100644 --- a/src/slic3r/Utils/OrcaPrinterAgent.hpp +++ b/src/slic3r/Utils/OrcaPrinterAgent.hpp @@ -27,6 +27,10 @@ public: // ======================================================================== void set_cloud_agent(std::shared_ptr cloud) override; + CameraStreamMode get_camera_stream_mode() const override; + std::string get_camera_url() const override; + std::unique_ptr + create_camera_signaling_channel(const std::string& dev_id) override; // Communication int send_message(std::string dev_id, std::string json_str, int qos, int flag) override; @@ -85,6 +89,9 @@ private: std::shared_ptr m_cloud_agent; OrcaCloudServiceAgent* m_orca_cloud = nullptr; // == m_cloud_agent.get() when the Orca provider is active + bool m_lan_connected = false; + std::string m_lan_rtsp_url; + // MOCK: OrcaCloud does not yet relay the printer's info.get_version reply, so // synthesize it and feed it through on_message_fn (same sink as real report // messages). Delete once the backend answers info.get_version. diff --git a/tests/slic3rutils/CMakeLists.txt b/tests/slic3rutils/CMakeLists.txt index edff97ff42..fb5b0bda1e 100644 --- a/tests/slic3rutils/CMakeLists.txt +++ b/tests/slic3rutils/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(${_TEST_NAME}_tests test_plugin_capabilities_in_use.cpp test_plugin_status.cpp test_printer_agent.cpp + test_webrtc_frame_assembler.cpp test_qidi_printer_agent.cpp test_plugin_install.cpp test_plugin_lifecycle.cpp diff --git a/tests/slic3rutils/test_webrtc_frame_assembler.cpp b/tests/slic3rutils/test_webrtc_frame_assembler.cpp new file mode 100644 index 0000000000..377d2931b7 --- /dev/null +++ b/tests/slic3rutils/test_webrtc_frame_assembler.cpp @@ -0,0 +1,67 @@ +#include + +#include + +#include +#include +#include +#include + +using Slic3r::GUI::WebRtcFrameAssembler; + +static std::vector make_chunk(std::uint32_t frame_id, + std::uint16_t index, + std::uint16_t count, + std::initializer_list payload) +{ + std::vector result(WebRtcFrameAssembler::HeaderSize + payload.size()); + result[0] = std::byte{1}; + result[1] = std::byte{0}; + result[2] = std::byte{static_cast(index >> 8)}; + result[3] = std::byte{static_cast(index)}; + result[4] = std::byte{static_cast(count >> 8)}; + result[5] = std::byte{static_cast(count)}; + result[6] = std::byte{static_cast(frame_id >> 24)}; + result[7] = std::byte{static_cast(frame_id >> 16)}; + result[8] = std::byte{static_cast(frame_id >> 8)}; + result[9] = std::byte{static_cast(frame_id)}; + for (std::size_t i = 0; i < payload.size(); ++i) + result[WebRtcFrameAssembler::HeaderSize + i] = std::byte{static_cast(payload.begin()[i])}; + return result; +} + +TEST_CASE("WebRTC frame assembler joins chunks in order", "[webrtc][unit]") +{ + WebRtcFrameAssembler assembler; + std::vector frame; + assembler.on_frame = [&frame](std::vector value) { frame = std::move(value); }; + + const auto second = make_chunk(7, 1, 2, {'C', 'D'}); + const auto first = make_chunk(7, 0, 2, {'A', 'B'}); + assembler.feed(second.data(), second.size()); + assembler.feed(first.data(), first.size()); + + REQUIRE(frame.size() == 4); + CHECK(std::to_integer(frame[0]) == 'A'); + CHECK(std::to_integer(frame[3]) == 'D'); +} + +TEST_CASE("WebRTC frame assembler discards stale and malformed chunks", "[webrtc][unit]") +{ + WebRtcFrameAssembler assembler; + int frames = 0; + assembler.on_frame = [&frames](std::vector) { ++frames; }; + + const auto stale = make_chunk(1, 0, 2, {'A'}); + const auto current = make_chunk(2, 0, 1, {'B'}); + const auto stale_tail = make_chunk(1, 1, 2, {'C'}); + assembler.feed(stale.data(), stale.size()); + assembler.feed(current.data(), current.size()); + assembler.feed(stale_tail.data(), stale_tail.size()); + + CHECK(frames == 1); + + auto malformed = make_chunk(3, 0, 0, {'X'}); + assembler.feed(malformed.data(), malformed.size()); + CHECK(frames == 1); +}