From 84e929ea2405d49a5ec3d86aa18f1dc178fcc611 Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Tue, 1 Sep 2026 18:02:53 +0800 Subject: [PATCH 1/5] feat: generic camera stream support for http snapshot and rtsp --- CMakeLists.txt | 12 +- deps/FFMPEG/FFMPEG.cmake | 4 +- src/slic3r/CMakeLists.txt | 13 +- src/slic3r/GUI/AVVideoDecoder.cpp | 27 ++ src/slic3r/GUI/AVVideoDecoder.hpp | 2 + src/slic3r/GUI/CameraPopup.cpp | 60 ---- src/slic3r/GUI/CameraPopup.hpp | 9 - src/slic3r/GUI/DeviceManager.cpp | 1 - src/slic3r/GUI/DeviceManager.hpp | 2 +- src/slic3r/GUI/IMediaController.hpp | 35 +++ src/slic3r/GUI/MediaPlayCtrl.cpp | 110 ++++++- src/slic3r/GUI/MediaPlayCtrl.h | 11 + src/slic3r/GUI/StatusPanel.cpp | 130 +++------ src/slic3r/GUI/StatusPanel.hpp | 12 +- src/slic3r/GUI/WebMediaController.cpp | 83 ++++++ src/slic3r/GUI/WebMediaController.hpp | 36 +++ src/slic3r/GUI/wxMediaCtrl3.cpp | 274 ++++++++++++------ src/slic3r/GUI/wxMediaCtrl3.h | 3 + src/slic3r/Utils/IPrinterAgent.hpp | 22 ++ src/slic3r/Utils/MoonrakerPrinterAgent.cpp | 91 +++++- src/slic3r/Utils/MoonrakerPrinterAgent.hpp | 19 +- src/slic3r/Utils/NetworkAgent.cpp | 14 + src/slic3r/Utils/NetworkAgent.hpp | 2 + src/slic3r/Utils/SnapmakerPrinterAgent.cpp | 235 +++++++-------- src/slic3r/Utils/SnapmakerPrinterAgent.hpp | 3 + .../PrinterAgentPluginCapability.cpp | 10 + .../PrinterAgentPluginCapability.hpp | 2 + ...PrinterAgentPluginCapabilityTrampoline.hpp | 16 + 28 files changed, 839 insertions(+), 399 deletions(-) create mode 100644 src/slic3r/GUI/IMediaController.hpp create mode 100644 src/slic3r/GUI/WebMediaController.cpp create mode 100644 src/slic3r/GUI/WebMediaController.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f5ccbeb44..8b954ef753 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1067,6 +1067,7 @@ function(orcaslicer_copy_dlls target config postfix output_dlls) ${CMAKE_PREFIX_PATH}/bin/occt/TKXDESTEP.dll ${CMAKE_PREFIX_PATH}/bin/occt/TKXSBase.dll ${CMAKE_PREFIX_PATH}/bin/freetype.dll + ${CMAKE_PREFIX_PATH}/bin/avformat-61.dll ${CMAKE_PREFIX_PATH}/bin/avcodec-61.dll ${CMAKE_PREFIX_PATH}/bin/swresample-5.dll ${CMAKE_PREFIX_PATH}/bin/swscale-8.dll @@ -1106,6 +1107,7 @@ function(orcaslicer_copy_dlls target config postfix output_dlls) ${_out_dir}/TKXSBase.dll ${_out_dir}/freetype.dll + ${_out_dir}/avformat-61.dll ${_out_dir}/avcodec-61.dll ${_out_dir}/swresample-5.dll ${_out_dir}/swscale-8.dll @@ -1128,7 +1130,10 @@ function(orcaslicer_copy_sos target config postfix output_sos) set(_out_dir "${CMAKE_CURRENT_BINARY_DIR}") endif () - file(COPY ${CMAKE_PREFIX_PATH}/lib/libavcodec.so + file(COPY ${CMAKE_PREFIX_PATH}/lib/libavformat.so + ${CMAKE_PREFIX_PATH}/lib/libavformat.so.61 + ${CMAKE_PREFIX_PATH}/lib/libavformat.so.61.1.100 + ${CMAKE_PREFIX_PATH}/lib/libavcodec.so ${CMAKE_PREFIX_PATH}/lib/libavcodec.so.61 ${CMAKE_PREFIX_PATH}/lib/libavcodec.so.61.3.100 ${CMAKE_PREFIX_PATH}/lib/libavutil.so @@ -1143,6 +1148,9 @@ function(orcaslicer_copy_sos target config postfix output_sos) DESTINATION ${_out_dir}) set(${output_sos} + ${_out_dir}/libavformat.so + ${_out_dir}/libavformat.so.61 + ${_out_dir}/libavformat.so.61.1.100 ${_out_dir}/libavcodec.so ${_out_dir}/libavcodec.so.61 ${_out_dir}/libavcodec.so.61.3.100 @@ -1271,6 +1279,8 @@ endif () if (CMAKE_SYSTEM_NAME STREQUAL "Linux") set(LIBRARY_FILES + ${LIBDIR_BIN}/libavformat.so.61 + ${LIBDIR_BIN}/libavformat.so.61.1.100 ${LIBDIR_BIN}/libavcodec.so.61 ${LIBDIR_BIN}/libavcodec.so.61.3.100 ${LIBDIR_BIN}/libavutil.so.59 diff --git a/deps/FFMPEG/FFMPEG.cmake b/deps/FFMPEG/FFMPEG.cmake index 38e952bbdd..3c582f0682 100644 --- a/deps/FFMPEG/FFMPEG.cmake +++ b/deps/FFMPEG/FFMPEG.cmake @@ -69,14 +69,14 @@ else () --disable-filters --enable-filter=*null*,afade,*fifo,*format,*resample,aeval,allrgb,allyuv,atempo,pan,*bars,color,*key,crop,draw*,eq*,framerate,*_qsv,*_vaapi,*v4l2*,hw*,scale,volume,test* --disable-protocols - --enable-protocol=file,fd,pipe,rtp,udp + --enable-protocol=file,fd,pipe,rtp,tcp,udp --disable-muxers --enable-muxer=rtp --disable-encoders --disable-decoders --enable-decoder=*aac*,h264*,mp3*,mjpeg,rv* --disable-demuxers - --enable-demuxer=h264,mp3,mov + --enable-demuxer=h264,mp3,mov,rtsp,sdp --disable-zlib --disable-avdevice BUILD_IN_SOURCE ON diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 140b1cec39..8308161a7b 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -222,6 +222,7 @@ set(SLIC3R_GUI_SOURCES GUI/GLToolbar.hpp GUI/ImageDPIFrame.cpp GUI/ImageDPIFrame.hpp + GUI/IMediaController.hpp GUI/GUI_App.cpp GUI/GUI_App.hpp GUI/GUI_AuxiliaryList.cpp @@ -530,6 +531,8 @@ set(SLIC3R_GUI_SOURCES GUI/WebUserLoginDialog.hpp GUI/WebViewDialog.cpp GUI/WebViewDialog.hpp + GUI/WebMediaController.hpp + GUI/WebMediaController.cpp GUI/Widgets/AMSControl.cpp GUI/Widgets/AMSControl.hpp GUI/Widgets/AMSItem.cpp @@ -906,14 +909,15 @@ endif () if (APPLE) # Static FFmpeg from the deps install: nothing to bundle into the .app, - # no rpath/install_name handling. Order matters: avcodec -> swscale -> avutil. + # no rpath/install_name handling. Order matters: avformat -> avcodec -> swscale -> avutil. + find_library(LIBAVFORMAT_LIBRARY NAMES libavformat.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH) find_library(LIBAVCODEC_LIBRARY NAMES libavcodec.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH) find_library(LIBSWSCALE_LIBRARY NAMES libswscale.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH) find_library(LIBAVUTIL_LIBRARY NAMES libavutil.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH) - if (NOT LIBAVCODEC_LIBRARY OR NOT LIBSWSCALE_LIBRARY OR NOT LIBAVUTIL_LIBRARY) - message(FATAL_ERROR "Static FFmpeg (libavcodec.a/libswscale.a/libavutil.a) not found under ${CMAKE_PREFIX_PATH}/lib. Rebuild the deps — FFMPEG builds static-only on macOS.") + if (NOT LIBAVFORMAT_LIBRARY OR NOT LIBAVCODEC_LIBRARY OR NOT LIBSWSCALE_LIBRARY OR NOT LIBAVUTIL_LIBRARY) + message(FATAL_ERROR "Static FFmpeg (libavformat.a/libavcodec.a/libswscale.a/libavutil.a) not found under ${CMAKE_PREFIX_PATH}/lib. Rebuild the deps — FFMPEG builds static-only on macOS.") endif () - target_link_libraries(libslic3r_gui ${LIBAVCODEC_LIBRARY} ${LIBSWSCALE_LIBRARY} ${LIBAVUTIL_LIBRARY}) + target_link_libraries(libslic3r_gui ${LIBAVFORMAT_LIBRARY} ${LIBAVCODEC_LIBRARY} ${LIBSWSCALE_LIBRARY} ${LIBAVUTIL_LIBRARY}) target_include_directories(libslic3r_gui SYSTEM PRIVATE ${CMAKE_PREFIX_PATH}/include) elseif (WIN32) # Prebuilt shared FFmpeg from the deps install. Windows has no pkg-config, @@ -929,6 +933,7 @@ elseif (WIN32) target_include_directories(libslic3r_gui SYSTEM PRIVATE ${CMAKE_PREFIX_PATH}/include) else () pkg_check_modules(LIBAV REQUIRED IMPORTED_TARGET + libavformat libavcodec libswscale libavutil diff --git a/src/slic3r/GUI/AVVideoDecoder.cpp b/src/slic3r/GUI/AVVideoDecoder.cpp index d7b8432bd3..76965fedef 100644 --- a/src/slic3r/GUI/AVVideoDecoder.cpp +++ b/src/slic3r/GUI/AVVideoDecoder.cpp @@ -45,6 +45,25 @@ int AVVideoDecoder::open(Bambu_StreamInfo const &info) return 0; } +int AVVideoDecoder::open(AVCodecParameters const ¶meters) +{ + if (avcodec_parameters_to_context(codec_ctx_, ¶meters) < 0) + return -1; + + auto codec = avcodec_find_decoder(codec_ctx_->codec_id); + if (codec == nullptr) { + fprintf(stderr, "AVVideoDecoder: unsupported codec!\n"); + return -1; + } + if (avcodec_open2(codec_ctx_, codec, nullptr) < 0) { + fprintf(stderr, "AVVideoDecoder: could not open codec\n"); + return -1; + } + + frame_ = av_frame_alloc(); + return frame_ == nullptr ? -1 : 0; +} + int AVVideoDecoder::decode(const Bambu_Sample &sample) { int ret = -1; @@ -71,6 +90,14 @@ int AVVideoDecoder::decode(const Bambu_Sample &sample) return ret; } +int AVVideoDecoder::decode(const AVPacket &packet) +{ + int ret = avcodec_send_packet(codec_ctx_, &packet); + if (ret == 0) + got_frame_ = avcodec_receive_frame(codec_ctx_, frame_) == 0; + return ret; +} + int AVVideoDecoder::flush() { int ret = avcodec_send_packet(codec_ctx_, nullptr); diff --git a/src/slic3r/GUI/AVVideoDecoder.hpp b/src/slic3r/GUI/AVVideoDecoder.hpp index 4111e860a2..4277734e08 100644 --- a/src/slic3r/GUI/AVVideoDecoder.hpp +++ b/src/slic3r/GUI/AVVideoDecoder.hpp @@ -23,8 +23,10 @@ public: public: int open(Bambu_StreamInfo const &info); + int open(AVCodecParameters const ¶meters); int decode(Bambu_Sample const &sample); + int decode(AVPacket const &packet); int flush(); diff --git a/src/slic3r/GUI/CameraPopup.cpp b/src/slic3r/GUI/CameraPopup.cpp index 0ee5694a34..f0b71cb7bf 100644 --- a/src/slic3r/GUI/CameraPopup.cpp +++ b/src/slic3r/GUI/CameraPopup.cpp @@ -28,7 +28,6 @@ wxEND_EVENT_TABLE() wxDEFINE_EVENT(EVT_VCAMERA_SWITCH, wxMouseEvent); wxDEFINE_EVENT(EVT_SDCARD_ABSENT_HINT, wxCommandEvent); -wxDEFINE_EVENT(EVT_CAM_SOURCE_CHANGE, wxCommandEvent); #define CAMERAPOPUP_CLICK_INTERVAL 20 @@ -102,34 +101,6 @@ CameraPopup::CameraPopup(wxWindow *parent) top_sizer->Add(0, 0, wxALL, 0); } - // Orca: custom IP camera source — lets the user point Live Video at any camera URL (Orca feature; not in the reference) - m_custom_camera_input_confirm = new Button(m_panel, _L("Enable")); - m_custom_camera_input_confirm->SetBackgroundColor(wxColour(38, 166, 154)); - m_custom_camera_input_confirm->SetBorderColor(wxColour(38, 166, 154)); - m_custom_camera_input_confirm->SetTextColor(wxColour(0xFFFFFE)); - m_custom_camera_input_confirm->SetFont(Label::Body_14); - m_custom_camera_input_confirm->SetMinSize(wxSize(FromDIP(90), FromDIP(30))); - m_custom_camera_input_confirm->SetPosition(wxDefaultPosition); - m_custom_camera_input_confirm->SetCornerRadius(FromDIP(12)); - m_custom_camera_input = new TextInput(m_panel, wxEmptyString, wxEmptyString, wxEmptyString, wxDefaultPosition, wxDefaultSize); - m_custom_camera_input->GetTextCtrl()->SetHint(_L("Hostname or IP")); - m_custom_camera_input->GetTextCtrl()->SetFont(Label::Body_14); - m_custom_camera_hint = new wxStaticText(m_panel, wxID_ANY, _L("Custom camera source")); - m_custom_camera_hint->Wrap(-1); - m_custom_camera_hint->SetFont(Label::Head_14); - m_custom_camera_hint->SetForegroundColour(TEXT_COL); - - m_custom_camera_input_confirm->Bind(wxEVT_BUTTON, &CameraPopup::on_camera_source_changed, this); - - if (!wxGetApp().app_config->get("camera", "custom_source").empty()) { - m_custom_camera_input->GetTextCtrl()->SetValue(wxGetApp().app_config->get("camera", "custom_source")); - set_custom_cam_button_state(wxGetApp().app_config->get("camera", "enable_custom_source") == "true"); - } - - top_sizer->Add(m_custom_camera_hint, 0, wxALIGN_CENTER_VERTICAL | wxALIGN_LEFT | wxALL, FromDIP(5)); - top_sizer->Add(0, 0, wxALL, 0); - top_sizer->Add(m_custom_camera_input, 2, wxALIGN_CENTER_VERTICAL | wxEXPAND | wxALL, FromDIP(5)); - top_sizer->Add(m_custom_camera_input_confirm, 1, wxALIGN_CENTER_VERTICAL | wxALIGN_RIGHT | wxALL, FromDIP(5)); main_sizer->Add(top_sizer, 0, wxALL, FromDIP(10)); auto url = wxString(L"https://www.orcaslicer.com/wiki/"); // Orca: neutral wiki link (vendor URL removed) @@ -184,37 +155,6 @@ void CameraPopup::sdcard_absent_hint() GetEventHandler()->ProcessEvent(evt); } -void CameraPopup::on_camera_source_changed(wxCommandEvent &event) -{ - if (m_obj && !m_custom_camera_input->GetTextCtrl()->IsEmpty()) { - handle_camera_source_change(); - } -} - -void CameraPopup::handle_camera_source_change() -{ - m_custom_camera_enabled = !m_custom_camera_enabled; - - set_custom_cam_button_state(m_custom_camera_enabled); - - wxGetApp().app_config->set("camera", "custom_source", m_custom_camera_input->GetTextCtrl()->GetValue().ToStdString()); - wxGetApp().app_config->set("camera", "enable_custom_source", m_custom_camera_enabled); - - wxCommandEvent evt(EVT_CAM_SOURCE_CHANGE); - evt.SetEventObject(this); - GetEventHandler()->ProcessEvent(evt); -} - -void CameraPopup::set_custom_cam_button_state(bool state) -{ - m_custom_camera_enabled = state; - auto stateColour = state ? wxColour(170, 0, 0) : wxColour(38, 166, 154); - auto stateText = state ? "Disable" : "Enable"; - m_custom_camera_input_confirm->SetBackgroundColor(stateColour); - m_custom_camera_input_confirm->SetBorderColor(stateColour); - m_custom_camera_input_confirm->SetLabel(_L(stateText)); -} - void CameraPopup::on_switch_recording(wxCommandEvent& event) { if (!m_obj) return; diff --git a/src/slic3r/GUI/CameraPopup.hpp b/src/slic3r/GUI/CameraPopup.hpp index dbdb81a9cb..5295407e8f 100644 --- a/src/slic3r/GUI/CameraPopup.hpp +++ b/src/slic3r/GUI/CameraPopup.hpp @@ -15,14 +15,12 @@ #include "Widgets/SwitchButton.hpp" #include "Widgets/RadioBox.hpp" #include "Widgets/PopupWindow.hpp" -#include "Widgets/TextInput.hpp" namespace Slic3r { namespace GUI { wxDECLARE_EVENT(EVT_VCAMERA_SWITCH, wxMouseEvent); wxDECLARE_EVENT(EVT_SDCARD_ABSENT_HINT, wxCommandEvent); -wxDECLARE_EVENT(EVT_CAM_SOURCE_CHANGE, wxCommandEvent); class CameraPopup : public PopupWindow { @@ -53,9 +51,6 @@ protected: void on_switch_recording(wxCommandEvent& event); void on_set_resolution(); void sdcard_absent_hint(); - void on_camera_source_changed(wxCommandEvent& event); - void handle_camera_source_change(); - void set_custom_cam_button_state(bool state); wxWindow * create_item_radiobox(wxString title, wxWindow *parent, wxString tooltip, int padding_left); void select_curr_radiobox(int btn_idx); @@ -74,10 +69,6 @@ private: SwitchButton* m_switch_vcamera; wxStaticText* m_text_liveview_retry; SwitchButton* m_switch_liveview_retry; - wxStaticText* m_custom_camera_hint; - TextInput* m_custom_camera_input; - Button* m_custom_camera_input_confirm; - bool m_custom_camera_enabled{ false }; wxStaticText* m_text_resolution; wxWindow* m_resolution_options[RESOLUTION_OPTIONS_NUM]; wxScrolledWindow *m_panel; diff --git a/src/slic3r/GUI/DeviceManager.cpp b/src/slic3r/GUI/DeviceManager.cpp index 06f574c8a2..741ed788fc 100644 --- a/src/slic3r/GUI/DeviceManager.cpp +++ b/src/slic3r/GUI/DeviceManager.cpp @@ -1338,7 +1338,6 @@ int MachineObject::command_get_access_code() { return this->publish_json(j); } - int MachineObject::command_request_push_all(bool request_now) { auto curr_time = std::chrono::system_clock::now(); diff --git a/src/slic3r/GUI/DeviceManager.hpp b/src/slic3r/GUI/DeviceManager.hpp index 6df11d10f5..8788c2288d 100644 --- a/src/slic3r/GUI/DeviceManager.hpp +++ b/src/slic3r/GUI/DeviceManager.hpp @@ -749,6 +749,7 @@ public: int command_set_printer_nozzle(std::string nozzle_type, float diameter); int command_set_printer_nozzle2(int id, std::string nozzle_type, float diameter); int command_get_access_code(); + int command_start_camera(); int command_ack_proceed(json& proceed); int command_purification_disable(); int command_dont_remind_next_time(json& mqtt_guard_json); @@ -797,7 +798,6 @@ public: int command_ams_select_tray(std::string tray_id); int command_ams_refresh_rfid(std::string tray_id); int command_ams_refresh_rfid2(int ams_id, int slot_id); - int command_start_camera(); int command_ams_control(std::string action); int command_ams_drying_stop(); int command_start_extrusion_cali(int tray_index, int nozzle_temp, int bed_temp, float max_volumetric_speed, std::string setting_id = ""); diff --git a/src/slic3r/GUI/IMediaController.hpp b/src/slic3r/GUI/IMediaController.hpp new file mode 100644 index 0000000000..982157e5bc --- /dev/null +++ b/src/slic3r/GUI/IMediaController.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include + +namespace Slic3r { namespace GUI { + +class IMediaController +{ +public: + virtual void Load(wxURI url) = 0; + + // The default keeps existing media controllers unaware of camera-specific modes. + virtual void Load(wxURI url, CameraStreamMode mode) + { + (void) mode; + Load(url); + } + + virtual void Play() = 0; + + virtual void Stop() = 0; + + virtual wxMediaState GetState() { return wxMediaState{}; } + + virtual int GetLastError() const { return {}; }; + + virtual wxSize GetVideoSize() const { return {}; }; + +private: +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/MediaPlayCtrl.cpp b/src/slic3r/GUI/MediaPlayCtrl.cpp index 707c7f853c..77a183f090 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.cpp +++ b/src/slic3r/GUI/MediaPlayCtrl.cpp @@ -146,8 +146,48 @@ MediaPlayCtrl::~MediaPlayCtrl() BOOST_LOG_TRIVIAL(info) << __FUNCTION__ << ": " << this; } +void MediaPlayCtrl::SetWebMediaController(IMediaController *ctrl) +{ + m_web_ctrl = ctrl; +} + +CameraStreamMode MediaPlayCtrl::current_mode() const +{ + auto agent = wxGetApp().getAgent(); + return agent ? agent->get_camera_stream_mode() : CameraStreamMode::none; +} + void MediaPlayCtrl::SetMachineObject(MachineObject* obj) { + switch (current_mode()) { + case CameraStreamMode::http: + case CameraStreamMode::http_snapshot: + case CameraStreamMode::rtsp: { + std::string machine = obj ? obj->get_dev_id() : ""; + auto agent = wxGetApp().getAgent(); + std::string url = agent ? agent->get_local_camera_stream_url() : ""; + m_camera_exists = !url.empty(); + Enable(obj && m_camera_exists); + bool changed = machine != m_machine || url != m_agent_camera_url; + m_machine = machine; + m_agent_camera_url = url; + m_url = from_u8(url); + if (!changed) { + if (m_last_state == MEDIASTATE_IDLE && IsEnabled() && !m_web_user_stopped) + Play(); + return; + } + m_web_user_stopped = false; + if (m_last_state != MEDIASTATE_IDLE) + Stop(" "); + if (IsEnabled()) + Play(); + return; + } + default: + break; + } + std::string machine = obj ? obj->get_dev_id() : ""; if (obj) { m_camera_exists = obj->has_ipcam; @@ -249,6 +289,42 @@ void refresh_agora_url(char const* device, char const* dev_ver, char const* chan void MediaPlayCtrl::Play() { + switch (current_mode()) { + case CameraStreamMode::http: + case CameraStreamMode::http_snapshot: + if (!m_next_retry.IsValid() || wxDateTime::Now() < m_next_retry) + return; + if (!IsShownOnScreen()) return; + if (m_last_state != MEDIASTATE_IDLE) return; + if (m_machine.empty() || !IsEnabled() || !m_camera_exists || m_url.IsEmpty() || !m_web_ctrl) { + Stop(_L("Please confirm if the printer is connected.")); + return; + } + if (auto agent = wxGetApp().getAgent()) + agent->command_start_camera(m_machine); + m_button_play->SetIcon("media_stop"); + m_web_ctrl->Load(wxURI(m_url), current_mode()); + m_web_ctrl->Play(); + m_last_state = wxMEDIASTATE_PLAYING; + SetStatus(_L("Playing..."), false); + return; + case CameraStreamMode::rtsp: + if (m_next_retry.IsValid() && wxDateTime::Now() < m_next_retry) + return; + if (!IsShownOnScreen()) return; + if (m_last_state != MEDIASTATE_IDLE) return; + m_failed_code = 0; + if (m_machine.empty() || !IsEnabled() || !m_camera_exists || m_url.IsEmpty()) { + Stop(_L("Please confirm if the printer is connected.")); + return; + } + m_button_play->SetIcon("media_stop"); + load(); + return; + default: + break; + } + if (!m_next_retry.IsValid() || wxDateTime::Now() < m_next_retry) return; if (!IsShownOnScreen()) @@ -376,8 +452,38 @@ void MediaPlayCtrl::Play() void start_ping_test(); +void MediaPlayCtrl::StopWebStream() +{ + if (m_last_state == MEDIASTATE_IDLE) + return; + if (m_web_ctrl) + m_web_ctrl->Stop(); + m_button_play->SetIcon("media_play"); + m_last_state = MEDIASTATE_IDLE; + SetStatus(_L("Video Stopped."), false); +} + void MediaPlayCtrl::Stop(wxString const &msg, wxString const &msg2) { + switch (current_mode()) { + case CameraStreamMode::http: + case CameraStreamMode::http_snapshot: + if (m_last_state != MEDIASTATE_IDLE) { + if (m_web_ctrl) m_web_ctrl->Stop(); + m_button_play->SetIcon("media_play"); + m_last_state = MEDIASTATE_IDLE; + if (!msg.IsEmpty()) + SetStatus(msg); + else + SetStatus(_L("Video Stopped."), false); + } else if (!msg.IsEmpty()) { + SetStatus(msg, false); + } + return; + default: + break; + } + int last_state = m_last_state; if (m_last_state != MEDIASTATE_IDLE) { @@ -454,10 +560,12 @@ void MediaPlayCtrl::TogglePlay() BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::TogglePlay"; if (m_last_state != MEDIASTATE_IDLE) { m_next_retry = wxDateTime(); + m_web_user_stopped = true; Stop(); } else { m_failed_retry = 0; m_user_triggered = true; + m_web_user_stopped = false; if (m_last_user_play + wxTimeSpan::Minutes(5) < wxDateTime::Now()) { m_last_failed_codes.clear(); m_last_user_play = wxDateTime::Now(); @@ -661,7 +769,7 @@ void MediaPlayCtrl::load() { m_last_state = MEDIASTATE_LOADING; SetStatus(_L("Loading...")); - if (wxGetApp().app_config->get("internal_developer_mode") == "true") { + if (current_mode() != CameraStreamMode::rtsp) { std::string file_h264 = data_dir() + "/video.h264"; std::string file_info = data_dir() + "/video.info"; BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl dump video to " << file_h264; diff --git a/src/slic3r/GUI/MediaPlayCtrl.h b/src/slic3r/GUI/MediaPlayCtrl.h index 0a01daefc9..e8ad0ee82c 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.h +++ b/src/slic3r/GUI/MediaPlayCtrl.h @@ -9,6 +9,8 @@ #define MediaPlayCtrl_h #include "wxMediaCtrl3.h" +#include "IMediaController.hpp" +#include "slic3r/Utils/IPrinterAgent.hpp" #include @@ -36,6 +38,10 @@ public: void SetMachineObject(MachineObject * obj); + void SetWebMediaController(IMediaController *ctrl); + + void StopWebStream(); + bool IsStreaming() const; void ToggleStream(); @@ -66,6 +72,8 @@ private: static bool get_stream_url(std::string *url = nullptr); + CameraStreamMode current_mode() const; + private: static inline const wxMediaState MEDIASTATE_IDLE = static_cast(3); static inline const wxMediaState MEDIASTATE_INITIALIZING = static_cast(4); @@ -76,6 +84,9 @@ private: std::shared_ptr m_token = std::make_shared(0); wxMediaCtrl3 * m_media_ctrl; + IMediaController * m_web_ctrl = nullptr; + std::string m_agent_camera_url; + bool m_web_user_stopped = false; wxMediaState m_last_state = MEDIASTATE_IDLE; std::string m_machine; int m_lan_proto = 0; diff --git a/src/slic3r/GUI/StatusPanel.cpp b/src/slic3r/GUI/StatusPanel.cpp index 97cd1c6e2e..b00dee3dca 100644 --- a/src/slic3r/GUI/StatusPanel.cpp +++ b/src/slic3r/GUI/StatusPanel.cpp @@ -1494,27 +1494,26 @@ wxBoxSizer *StatusBasePanel::create_monitoring_page() m_setting_button->SetMinSize(wxSize(FromDIP(38), FromDIP(24))); m_setting_button->SetBackgroundColour(STATUS_TITLE_BG); - m_camera_switch_button = new wxStaticBitmap(m_panel_monitoring_title, wxID_ANY, wxNullBitmap, wxDefaultPosition, wxSize(FromDIP(38), FromDIP(24)), 0); - m_camera_switch_button->SetMinSize(wxSize(FromDIP(38), FromDIP(24))); - m_camera_switch_button->SetBackgroundColour(STATUS_TITLE_BG); - m_camera_switch_button->SetBitmap(m_bitmap_switch_camera.bmp()); - m_camera_switch_button->Bind(wxEVT_LEFT_DOWN, &StatusBasePanel::on_camera_switch_toggled, this); - m_camera_switch_button->Bind(wxEVT_RIGHT_DOWN, [this](auto& e) { - const std::string js_request_pip = R"( - document.querySelector('video').requestPictureInPicture(); - )"; - m_custom_camera_view->RunScript(js_request_pip); - }); - m_camera_switch_button->Hide(); + // m_camera_switch_button = new wxStaticBitmap(m_panel_monitoring_title, wxID_ANY, wxNullBitmap, wxDefaultPosition, wxSize(FromDIP(38), FromDIP(24)), 0); + // m_camera_switch_button->SetMinSize(wxSize(FromDIP(38), FromDIP(24))); + // m_camera_switch_button->SetBackgroundColour(STATUS_TITLE_BG); + // m_camera_switch_button->SetBitmap(m_bitmap_switch_camera.bmp()); + // m_camera_switch_button->Bind(wxEVT_RIGHT_DOWN, [this](auto& e) { + // const std::string js_request_pip = R"( + // document.querySelector('video').requestPictureInPicture(); + // )"; + // m_custom_camera_view->RunScript(js_request_pip); + // }); + // m_camera_switch_button->Hide(); m_bitmap_sdcard_img->SetToolTip(_L("Storage")); m_bitmap_timelapse_img->SetToolTip(_L("Timelapse")); m_bitmap_recording_img->SetToolTip(_L("Video")); m_bitmap_vcamera_img->SetToolTip(_L("Go Live")); m_setting_button->SetToolTip(_L("Camera Setting")); - m_camera_switch_button->SetToolTip(_L("Switch Camera View")); + // m_camera_switch_button->SetToolTip(_L("Switch Camera View")); - bSizer_monitoring_title->Add(m_camera_switch_button, 0, wxALIGN_CENTER_VERTICAL | wxALL, FromDIP(5)); + // bSizer_monitoring_title->Add(m_camera_switch_button, 0, wxALIGN_CENTER_VERTICAL | wxALL, FromDIP(5)); bSizer_monitoring_title->Add(m_bitmap_sdcard_img, 0, wxALIGN_CENTER_VERTICAL | wxALL, FromDIP(5)); bSizer_monitoring_title->Add(m_bitmap_timelapse_img, 0, wxALIGN_CENTER_VERTICAL | wxALL, FromDIP(5)); bSizer_monitoring_title->Add(m_bitmap_recording_img, 0, wxALIGN_CENTER_VERTICAL | wxALL, FromDIP(5)); @@ -1537,19 +1536,18 @@ wxBoxSizer *StatusBasePanel::create_monitoring_page() m_custom_camera_view = WebView::CreateWebView(this, wxEmptyString); m_custom_camera_view->EnableContextMenu(false); Bind(wxEVT_WEBVIEW_NAVIGATING, &StatusBasePanel::on_webview_navigating, this, m_custom_camera_view->GetId()); + m_web_media_controller = std::make_unique(m_custom_camera_view); m_media_play_ctrl = new MediaPlayCtrl(this, m_media_ctrl, wxDefaultPosition, wxSize(-1, FromDIP(40))); + m_media_play_ctrl->SetWebMediaController(m_web_media_controller.get()); m_custom_camera_view->Hide(); - m_custom_camera_view->Bind(wxEVT_WEBVIEW_SCRIPT_MESSAGE_RECEIVED, [this](wxWebViewEvent& evt) { - if (evt.GetString() == "leavepictureinpicture") { - // When leaving PiP, video gets paused in some cases and toggling play - // programmatically does not work. - m_custom_camera_view->Reload(); - } - else if (evt.GetString() == "enterpictureinpicture") { - toggle_builtin_camera(); - } - }); + // m_custom_camera_view->Bind(wxEVT_WEBVIEW_SCRIPT_MESSAGE_RECEIVED, [this](wxWebViewEvent& evt) { + // if (evt.GetString() == "leavepictureinpicture") { + // // When leaving PiP, video gets paused in some cases and toggling play + // // programmatically does not work. + // m_custom_camera_view->Reload(); + // } + // }); sizer->Add(m_media_ctrl, 1, wxEXPAND | wxALL, 0); sizer->Add(m_custom_camera_view, 1, wxEXPAND | wxALL, 0); @@ -1559,10 +1557,6 @@ wxBoxSizer *StatusBasePanel::create_monitoring_page() // // sizer->Add(media_ctrl_panel, 1, wxEXPAND | wxALL, 1); - if (wxGetApp().app_config->get("camera", "enable_custom_source") == "true") { - handle_camera_source_change(); - } - return sizer; } @@ -2312,25 +2306,21 @@ void StatusPanel::update_camera_state(MachineObject* obj) { if (!obj) return; - const bool has_printer_webcam = !obj->webcam_stream_url.empty(); + auto agent = wxGetApp().getAgent(); + const auto camera_mode = agent ? agent->get_camera_stream_mode() : CameraStreamMode::none; + const bool has_printer_webcam = camera_mode == CameraStreamMode::http || camera_mode == CameraStreamMode::http_snapshot; if (has_printer_webcam) { - if (m_printer_webcam_url != obj->webcam_stream_url) { - m_custom_camera_view->LoadURL(obj->webcam_stream_url); - m_custom_camera_view->Show(); - m_media_ctrl->Hide(); - m_media_play_ctrl->Hide(); - m_printer_webcam_url = obj->webcam_stream_url; - } - m_camera_switch_button->Hide(); + //m_camera_switch_button->Hide(); if (!m_custom_camera_view->IsShown()) { - // why: do not compare or reload the WebView URL per tick, or redirects can cause a reload loop. + // why: do not reload the WebView URL per tick, or redirects can cause a reload loop. + // MediaPlayCtrl (via its WebMediaController) owns loading/playing the stream itself. m_custom_camera_view->Show(); m_media_ctrl->Hide(); - m_media_play_ctrl->Hide(); } - } else if (!m_printer_webcam_url.empty()) { - handle_camera_source_change(); - m_printer_webcam_url.clear(); + } else if (m_custom_camera_view->IsShown()) { + m_custom_camera_view->Hide(); + m_media_ctrl->Show(); + m_media_play_ctrl->StopWebStream(); } //sdcard @@ -5056,7 +5046,6 @@ void StatusPanel::on_camera_enter(wxMouseEvent& event) } sdcard_hint_dlg->on_show(); }); - m_camera_popup->Bind(EVT_CAM_SOURCE_CHANGE, &StatusPanel::on_camera_source_change, this); wxWindow* ctrl = (wxWindow*)event.GetEventObject(); wxPoint pos = ctrl->ClientToScreen(wxPoint(0, 0)); wxSize sz = ctrl->GetSize(); @@ -5068,54 +5057,6 @@ void StatusPanel::on_camera_enter(wxMouseEvent& event) } } -void StatusBasePanel::on_camera_source_change(wxCommandEvent& event) -{ - handle_camera_source_change(); -} - -void StatusBasePanel::handle_camera_source_change() -{ - const auto new_cam_url = wxGetApp().app_config->get("camera", "custom_source"); - const auto enabled = wxGetApp().app_config->get("camera", "enable_custom_source") == "true"; - - if (enabled && !new_cam_url.empty()) { - m_custom_camera_view->LoadURL(new_cam_url); - toggle_custom_camera(); - m_camera_switch_button->Show(); - } else { - toggle_builtin_camera(); - m_camera_switch_button->Hide(); - } -} - -void StatusBasePanel::toggle_builtin_camera() -{ - m_custom_camera_view->Hide(); - m_media_ctrl->Show(); - m_media_play_ctrl->Show(); -} - -void StatusBasePanel::toggle_custom_camera() -{ - const auto enabled = wxGetApp().app_config->get("camera", "enable_custom_source") == "true"; - - if (enabled) { - m_custom_camera_view->Show(); - m_media_ctrl->Hide(); - m_media_play_ctrl->Hide(); - } -} - -void StatusBasePanel::on_camera_switch_toggled(wxMouseEvent& event) -{ - const auto enabled = wxGetApp().app_config->get("camera", "enable_custom_source") == "true"; - if (enabled && m_media_ctrl->IsShown()) { - toggle_custom_camera(); - } else { - toggle_builtin_camera(); - } -} - void StatusBasePanel::remove_controls() { const std::string js_cleanup_video_element = R"( @@ -5268,9 +5209,10 @@ bool StatusPanel::is_stage_list_info_changed(MachineObject *obj) void StatusPanel::set_default() { BOOST_LOG_TRIVIAL(trace) << "status_panel: set_default"; - if (!m_printer_webcam_url.empty()) { - handle_camera_source_change(); - m_printer_webcam_url.clear(); + if (m_custom_camera_view->IsShown()) { + m_custom_camera_view->Hide(); + m_media_ctrl->Show(); + m_media_play_ctrl->StopWebStream(); } obj = nullptr; last_subtask = nullptr; diff --git a/src/slic3r/GUI/StatusPanel.hpp b/src/slic3r/GUI/StatusPanel.hpp index ec863cb474..1478feef8b 100644 --- a/src/slic3r/GUI/StatusPanel.hpp +++ b/src/slic3r/GUI/StatusPanel.hpp @@ -15,7 +15,9 @@ #include #include #include +#include #include "MediaPlayCtrl.h" +#include "WebMediaController.hpp" #include "AMSSetting.hpp" #include "Calibration.hpp" #include "CalibrationWizardPage.hpp" @@ -440,7 +442,7 @@ protected: wxStaticBitmap *m_bitmap_sdcard_img; wxStaticBitmap *m_bitmap_static_use_time; wxStaticBitmap *m_bitmap_static_use_weight; - wxStaticBitmap* m_camera_switch_button; + // wxStaticBitmap* m_camera_switch_button; wxMediaCtrl3 * m_media_ctrl; @@ -462,6 +464,8 @@ protected: ScalableButton *m_button_abort; Button * m_button_clean; wxWebView * m_custom_camera_view{nullptr}; + std::unique_ptr m_web_media_controller; + wxSimplebook* m_extruder_book; std::vector m_extruderImage; @@ -577,13 +581,8 @@ protected: virtual void on_axis_ctrl_e_up_10(wxCommandEvent &event) { event.Skip(); } virtual void on_axis_ctrl_e_down_10(wxCommandEvent &event) { event.Skip(); } virtual void on_nozzle_selected(wxCommandEvent &event) { event.Skip(); } - void on_camera_source_change(wxCommandEvent& event); - void handle_camera_source_change(); void remove_controls(); void on_webview_navigating(wxWebViewEvent& evt); - void on_camera_switch_toggled(wxMouseEvent& event); - void toggle_custom_camera(); - void toggle_builtin_camera(); public: StatusBasePanel(wxWindow * parent, @@ -664,7 +663,6 @@ protected: int m_last_timelapse = -1; int m_last_extrusion = -1; int m_last_vcamera = -1; - std::string m_printer_webcam_url; int m_model_mall_request_count = 0; bool m_is_load_with_temp = false; json m_rating_result; diff --git a/src/slic3r/GUI/WebMediaController.cpp b/src/slic3r/GUI/WebMediaController.cpp new file mode 100644 index 0000000000..13d820c98f --- /dev/null +++ b/src/slic3r/GUI/WebMediaController.cpp @@ -0,0 +1,83 @@ +#include "WebMediaController.hpp" + +#include + +namespace Slic3r { namespace GUI { + +WebMediaController::WebMediaController(wxWebView *webview) : m_webview(webview) +{ + if (!m_webview) + return; + + m_webview->SetBackgroundColour(*wxBLACK); + m_webview->SetPage( + "", + ""); +} + +void WebMediaController::Load(wxURI url) +{ + Load(url, CameraStreamMode::http); +} + +void WebMediaController::Load(wxURI url, CameraStreamMode mode) +{ + m_url = url.BuildURI().ToStdString(); + m_stream_mode = mode; +} + +void WebMediaController::Play() +{ + if (!m_webview) + return; + + wxString url = wxString::FromUTF8(m_url); + wxString html = ""; + } else { + html += " src=\"" + url + "\">"; + } + m_webview->SetPage(html, url); +} + +void WebMediaController::Stop() +{ + m_url.clear(); + if (m_webview) + m_webview->Stop(); +} + +// wxMediaState WebMediaController::GetState() +// { +// return wxMediaState{}; +// } + +// int WebMediaController::GetLastError() const +// { +// return 0; +// } + +// wxSize WebMediaController::GetVideoSize() const +// { +// return wxSize{}; +// } + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebMediaController.hpp b/src/slic3r/GUI/WebMediaController.hpp new file mode 100644 index 0000000000..cdbee33cc3 --- /dev/null +++ b/src/slic3r/GUI/WebMediaController.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include + +class wxWebView; + +namespace Slic3r { namespace GUI { + +class WebMediaController : public IMediaController +{ +public: + explicit WebMediaController(wxWebView *webview); + + void Load(wxURI url) override; + + void Load(wxURI url, CameraStreamMode mode) override; + + void Play() override; + + void Stop() override; + + // wxMediaState GetState() override; + + // int GetLastError() const override; + + // wxSize GetVideoSize() const override; + +private: + wxWebView * m_webview; + std::string m_url; + CameraStreamMode m_stream_mode = CameraStreamMode::http; +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/wxMediaCtrl3.cpp b/src/slic3r/GUI/wxMediaCtrl3.cpp index 098db68808..d0d53072d6 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.cpp +++ b/src/slic3r/GUI/wxMediaCtrl3.cpp @@ -4,6 +4,9 @@ #include "libslic3r/Utils.hpp" #include #include +extern "C" { +#include +} #ifdef __WIN32__ #include #include @@ -185,6 +188,106 @@ void wxMediaCtrl3::bambu_log(void *ctx, int level, tchar const *msg2) BOOST_LOG_TRIVIAL(info) << msg.ToUTF8().data(); } +int wxMediaCtrl3::rtsp_interrupt_callback(void *opaque) +{ + auto *ctrl = static_cast(opaque); + std::lock_guard lock(ctrl->m_mutex); + return ctrl->m_url != ctrl->m_active_url; +} + +int wxMediaCtrl3::PlayRtsp(std::shared_ptr const &url, std::unique_lock &lock) +{ + if (avformat_network_init() < 0) + return 2; + + AVFormatContext *format_context = avformat_alloc_context(); + if (!format_context) { + avformat_network_deinit(); + return 2; + } + + format_context->interrupt_callback = {&wxMediaCtrl3::rtsp_interrupt_callback, this}; + m_active_url = url; + + auto finish = [&](int error) { + lock.unlock(); + avformat_close_input(&format_context); + avformat_network_deinit(); + lock.lock(); + m_active_url.reset(); + return error; + }; + + const std::string uri = url->BuildURI().ToUTF8().data(); + AVDictionary *options = nullptr; + av_dict_set(&options, "rtsp_transport", "tcp", 0); + lock.unlock(); + int error = avformat_open_input(&format_context, uri.c_str(), nullptr, &options); + av_dict_free(&options); + lock.lock(); + if (error < 0) + return finish(2); + + lock.unlock(); + error = avformat_find_stream_info(format_context, nullptr); + lock.lock(); + if (error < 0) + return finish(2); + + const int video_stream = av_find_best_stream(format_context, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + if (video_stream < 0) + return finish(2); + + AVVideoDecoder decoder; + if (decoder.open(*format_context->streams[video_stream]->codecpar) < 0) + return finish(2); + + m_video_size = {format_context->streams[video_stream]->codecpar->width, + format_context->streams[video_stream]->codecpar->height}; + if (!m_video_size.IsFullySpecified() || m_video_size.x <= 0 || m_video_size.y <= 0) + return finish(2); + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + NotifyStopped(); + + AVPacket *packet = av_packet_alloc(); + if (!packet) + return finish(2); + + while (m_url == url) { + lock.unlock(); + error = av_read_frame(format_context, packet); + lock.lock(); + if (m_url != url) + break; + if (error < 0) + break; + if (packet->stream_index == video_stream) { + const int decode_error = decoder.decode(*packet); + if (decode_error == 0) { + auto frame_size = m_frame_size; + lock.unlock(); +#ifdef _WIN32 + wxBitmap frame; + decoder.toWxBitmap(frame, frame_size); +#else + wxImage frame; + decoder.toWxImage(frame, frame_size); +#endif + lock.lock(); + if (m_url != url) + break; + if (frame.IsOk()) + m_frame = frame; + CallAfter([this] { Refresh(); }); + } + } + av_packet_unref(packet); + } + + av_packet_free(&packet); + return finish(m_url == url ? 2 : 1); +} + void wxMediaCtrl3::PlayThread() { using namespace std::chrono_literals; @@ -197,42 +300,21 @@ void wxMediaCtrl3::PlayThread() continue; if (!url->HasScheme()) break; - lk.unlock(); - Bambu_Tunnel tunnel = nullptr; - int error = Bambu_Create(&tunnel, m_url->BuildURI().ToUTF8()); - if (error == 0) { - Bambu_SetLogger(tunnel, &wxMediaCtrl3::bambu_log, this); - error = Bambu_Open(tunnel); - if (error == 0) - error = Bambu_would_block; - } - lk.lock(); - while (error == int(Bambu_would_block)) { - m_cond.wait_for(lk, 100ms); - if (m_url != url) { - error = 1; - break; + const wxString scheme = url->GetScheme(); + const bool generic_rtsp = scheme.CmpNoCase("rtsp") == 0 || scheme.CmpNoCase("rtsps") == 0; + int error = 0; + if (generic_rtsp) { + error = PlayRtsp(url, lk); + } else { + lk.unlock(); + Bambu_Tunnel tunnel = nullptr; + error = Bambu_Create(&tunnel, m_url->BuildURI().ToUTF8()); + if (error == 0) { + Bambu_SetLogger(tunnel, &wxMediaCtrl3::bambu_log, this); + error = Bambu_Open(tunnel); + if (error == 0) + error = Bambu_would_block; } - lk.unlock(); - error = Bambu_StartStream(tunnel, true); - lk.lock(); - } - Bambu_StreamInfo info; - if (error == 0) - error = Bambu_GetStreamInfo(tunnel, 0, &info); - AVVideoDecoder decoder; - int minFrameDuration = 0; - if (error == 0) { - decoder.open(info); - m_video_size = { info.format.video.width, info.format.video.height }; - adjust_frame_size(m_frame_size, m_video_size, GetSize()); - minFrameDuration = 800 / info.format.video.frame_rate; // 80% - NotifyStopped(); - } - Bambu_Sample sample; - while (error == 0) { - lk.unlock(); - error = Bambu_ReadSample(tunnel, &sample); lk.lock(); while (error == int(Bambu_would_block)) { m_cond.wait_for(lk, 100ms); @@ -240,59 +322,87 @@ void wxMediaCtrl3::PlayThread() error = 1; break; } + lk.unlock(); + error = Bambu_StartStream(tunnel, true); + lk.lock(); + } + Bambu_StreamInfo info; + if (error == 0) + error = Bambu_GetStreamInfo(tunnel, 0, &info); + AVVideoDecoder decoder; + int minFrameDuration = 0; + if (error == 0) { + decoder.open(info); + m_video_size = { info.format.video.width, info.format.video.height }; + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + minFrameDuration = 800 / info.format.video.frame_rate; // 80% + NotifyStopped(); + } + Bambu_Sample sample; + while (error == 0) { lk.unlock(); error = Bambu_ReadSample(tunnel, &sample); lk.lock(); - } - if (error == 0) { - auto frame_size = m_frame_size; - lk.unlock(); - decoder.decode(sample); -#ifdef _WIN32 - wxBitmap bm; - decoder.toWxBitmap(bm, frame_size); -#else - wxImage bm; - decoder.toWxImage(bm, frame_size); -#endif - lk.lock(); - if (m_url != url) { - error = 1; - break; - } - if (bm.IsOk()) { - auto now = std::chrono::system_clock::now(); - if (m_last_PTS && (sample.decode_time - m_last_PTS) < 30000000ULL) { // 3s - auto next_PTS_expected = m_last_PTS_expected + std::chrono::milliseconds((sample.decode_time - m_last_PTS) / 10000ULL); - // The frame is late, catch up a little - auto next_PTS_practical = m_last_PTS_practical + std::chrono::milliseconds(minFrameDuration); - auto next_PTS = std::max(next_PTS_expected, next_PTS_practical); - if(now < next_PTS) - std::this_thread::sleep_until(next_PTS); - else - next_PTS = now; - //auto text = wxString::Format(L"wxMediaCtrl3 pts diff %ld\n", std::chrono::duration_cast(next_PTS - next_PTS_expected).count()); - //OutputDebugString(text); - m_last_PTS = sample.decode_time; - m_last_PTS_expected = next_PTS_expected; - m_last_PTS_practical = next_PTS; - } else { - // Resync - m_last_PTS = sample.decode_time; - m_last_PTS_expected = now; - m_last_PTS_practical = now; + while (error == int(Bambu_would_block)) { + m_cond.wait_for(lk, 100ms); + if (m_url != url) { + error = 1; + break; } - m_frame = bm; + lk.unlock(); + error = Bambu_ReadSample(tunnel, &sample); + lk.lock(); + } + if (error == 0) { + auto frame_size = m_frame_size; + lk.unlock(); + decoder.decode(sample); +#ifdef _WIN32 + wxBitmap bm; + decoder.toWxBitmap(bm, frame_size); +#else + wxImage bm; + decoder.toWxImage(bm, frame_size); +#endif + lk.lock(); + if (m_url != url) { + error = 1; + break; + } + if (bm.IsOk()) { + auto now = std::chrono::system_clock::now(); + if (m_last_PTS && (sample.decode_time - m_last_PTS) < 30000000ULL) { // 3s + auto next_PTS_expected = m_last_PTS_expected + std::chrono::milliseconds((sample.decode_time - m_last_PTS) / 10000ULL); + // The frame is late, catch up a little + auto next_PTS_practical = m_last_PTS_practical + std::chrono::milliseconds(minFrameDuration); + auto next_PTS = std::max(next_PTS_expected, next_PTS_practical); + if(now < next_PTS) + std::this_thread::sleep_until(next_PTS); + else + next_PTS = now; + //auto text = wxString::Format(L"wxMediaCtrl3 pts diff %ld\n", std::chrono::duration_cast(next_PTS - next_PTS_expected).count()); + //OutputDebugString(text); + m_last_PTS = sample.decode_time; + m_last_PTS_expected = next_PTS_expected; + m_last_PTS_practical = next_PTS; + } else { + // Resync + m_last_PTS = sample.decode_time; + m_last_PTS_expected = now; + m_last_PTS_practical = now; + } + m_frame = bm; + } + CallAfter([this] { Refresh(); }); } - CallAfter([this] { Refresh(); }); } - } - if (tunnel) { - lk.unlock(); - Bambu_Close(tunnel); - Bambu_Destroy(tunnel); - tunnel = nullptr; - lk.lock(); + if (tunnel) { + lk.unlock(); + Bambu_Close(tunnel); + Bambu_Destroy(tunnel); + tunnel = nullptr; + lk.lock(); + } } if (m_url == url) m_error = error; diff --git a/src/slic3r/GUI/wxMediaCtrl3.h b/src/slic3r/GUI/wxMediaCtrl3.h index 1d64955ffd..bcc94a17ef 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.h +++ b/src/slic3r/GUI/wxMediaCtrl3.h @@ -56,8 +56,10 @@ protected: void DoSetSize(int x, int y, int width, int height, int sizeFlags) override; static void bambu_log(void *ctx, int level, tchar const *msg); + static int rtsp_interrupt_callback(void *opaque); void PlayThread(); + int PlayRtsp(std::shared_ptr const &url, std::unique_lock &lock); void NotifyStopped(); @@ -74,6 +76,7 @@ private: #endif std::shared_ptr m_url; + std::shared_ptr m_active_url; std::uint64_t m_last_PTS{0}; std::chrono::system_clock::time_point m_last_PTS_expected; std::chrono::system_clock::time_point m_last_PTS_practical; diff --git a/src/slic3r/Utils/IPrinterAgent.hpp b/src/slic3r/Utils/IPrinterAgent.hpp index b77a27276f..c076b57be5 100644 --- a/src/slic3r/Utils/IPrinterAgent.hpp +++ b/src/slic3r/Utils/IPrinterAgent.hpp @@ -81,6 +81,14 @@ enum class FilamentSyncMode { pull ///< On-demand fetch via REST API (blocking call) }; +enum class CameraStreamMode { + none = 0, + http, // LAN or Cloud + rtsp, // LAN only + webrtc, // Cloud only + http_snapshot // HTTP endpoint returning one image per request +}; + /** * IPrinterAgent - Interface for printer operations. * @@ -354,12 +362,26 @@ public: */ virtual FilamentSyncMode get_filament_sync_mode() const { return FilamentSyncMode::none; } + /** + * Get the camera stream mode for this agent. This value can be deterministic and derived at + * runtime if the printer supports multiple camera stream modes. E.g. LAN => HTTP/RTSP, Cloud => WebRTC. + * + * @return CameraStreamMode indicating how the camera stream is obtained or used: + */ + virtual CameraStreamMode get_camera_stream_mode() const { return CameraStreamMode::none; } + /** * Refresh filament info from the printer synchronously. * Should only be called when get_filament_sync_mode() returns FilamentSyncMode::pull. * Populates the MachineObject's DevFilaSystem with fetched filament data. */ virtual bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull) { return false; } + + /** + * Get the current camera stream URL for this agent's active machine. + * Only meaningful when get_camera_stream_mode() returns an HTTP or RTSP mode. + */ + virtual std::string get_camera_url() const { return {}; } }; } // namespace Slic3r diff --git a/src/slic3r/Utils/MoonrakerPrinterAgent.cpp b/src/slic3r/Utils/MoonrakerPrinterAgent.cpp index d898c51e3a..d2c8781fbe 100644 --- a/src/slic3r/Utils/MoonrakerPrinterAgent.cpp +++ b/src/slic3r/Utils/MoonrakerPrinterAgent.cpp @@ -750,6 +750,57 @@ bool MoonrakerPrinterAgent::fetch_filament_info(std::string dev_id, FilamentSync return false; } +CameraStreamMode MoonrakerPrinterAgent::get_camera_stream_mode() const +{ + refresh_webcam_info(); + std::lock_guard lock(payload_mutex); + return webcam_stream_mode; +} + +std::string MoonrakerPrinterAgent::get_camera_url() const +{ + refresh_webcam_info(); + std::lock_guard lock(payload_mutex); + return webcam_stream_url; +} + +void MoonrakerPrinterAgent::refresh_webcam_info() const +{ + std::string base_url; + std::string api_key; + uint64_t generation; + { + std::lock_guard lock(connect_mutex); + base_url = device_info.base_url; + api_key = device_info.api_key; + generation = connect_generation.load(); + } + + const uint64_t now_ms = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + + if (base_url.empty()) { + std::lock_guard lock(payload_mutex); + webcam_stream_url.clear(); + webcam_stream_mode = CameraStreamMode::none; + webcam_info_last_lookup_ms = 0; + webcam_info_generation = generation; + return; + } + + { + std::lock_guard lock(payload_mutex); + if (webcam_info_generation == generation && + now_ms - webcam_info_last_lookup_ms < WEBCAM_INFO_REFRESH_INTERVAL_MS) { + return; + } + webcam_info_generation = generation; + webcam_info_last_lookup_ms = now_ms; + } + + fetch_webcam_info(base_url, api_key, generation); +} + std::string MoonrakerPrinterAgent::trim_and_upper(const std::string& input) { std::string result = input; @@ -1605,10 +1656,11 @@ bool MoonrakerPrinterAgent::send_ws_rpc(const std::string& method, const nlohman return false; } -bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const std::string& api_key, uint64_t generation) +bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const std::string& api_key, uint64_t generation) const { - std::string stream_url; + std::string camera_url; std::string webcam_name; + CameraStreamMode stream_mode = CameraStreamMode::none; std::string error; try { std::string response_body; @@ -1649,16 +1701,24 @@ bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const error = "Unexpected JSON structure"; } else { for (const auto& webcam : result["webcams"]) { - if (webcam.is_object() && webcam.value("enabled", false) && webcam.contains("stream_url") && - webcam["stream_url"].is_string()) { - stream_url = webcam["stream_url"].get(); + if (webcam.is_object() && webcam.value("enabled", false)) { + if (webcam.contains("stream_url") && webcam["stream_url"].is_string() && + !webcam["stream_url"].get().empty()) { + camera_url = webcam["stream_url"].get(); + stream_mode = CameraStreamMode::http; + } else if (webcam.contains("snapshot_url") && webcam["snapshot_url"].is_string() && + !webcam["snapshot_url"].get().empty()) { + camera_url = webcam["snapshot_url"].get(); + stream_mode = CameraStreamMode::http_snapshot; + } if (webcam.contains("name") && webcam["name"].is_string()) { webcam_name = webcam["name"].get(); } - break; + if (!camera_url.empty()) + break; } } - if (stream_url.empty()) { + if (camera_url.empty()) { error = "No enabled webcam"; } } @@ -1666,8 +1726,8 @@ bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const } if (error.empty()) { - if (stream_url.rfind("http", 0) != 0 && !stream_url.empty() && stream_url.front() == '/') { - // why: Moonraker's API port serves a JSON 404 for /webcam; relative streams use the printer web root. + if (camera_url.rfind("http", 0) != 0 && !camera_url.empty() && camera_url.front() == '/') { + // why: Moonraker's API port serves a JSON 404 for /webcam; relative camera URLs use the printer web root. const size_t scheme_end = base_url.find("://"); const size_t authority_start = scheme_end == std::string::npos ? 0 : scheme_end + 3; const size_t authority_end = base_url.find('/', authority_start); @@ -1678,9 +1738,9 @@ bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const std::all_of(authority.begin() + port_start + 1, authority.end(), [](char c) { return c >= '0' && c <= '9'; })) { authority.erase(port_start); } - stream_url = scheme + authority + stream_url; - } else if (stream_url.rfind("http", 0) != 0) { - error = "Unsupported webcam stream URL"; + camera_url = scheme + authority + camera_url; + } else if (camera_url.rfind("http", 0) != 0) { + error = "Unsupported webcam URL"; } } } catch (const std::exception& e) { @@ -1692,14 +1752,15 @@ bool MoonrakerPrinterAgent::fetch_webcam_info(const std::string& base_url, const { std::lock_guard lock(payload_mutex); if (generation == connect_generation.load()) { - webcam_stream_url = error.empty() ? stream_url : ""; + webcam_stream_url = error.empty() ? camera_url : ""; + webcam_stream_mode = error.empty() ? stream_mode : CameraStreamMode::none; } } if (!error.empty()) { BOOST_LOG_TRIVIAL(warning) << "MoonrakerPrinterAgent: webcam discovery failed: " << error; return false; } - BOOST_LOG_TRIVIAL(info) << "MoonrakerPrinterAgent: selected webcam '" << webcam_name << "' with stream URL " << stream_url; + BOOST_LOG_TRIVIAL(info) << "MoonrakerPrinterAgent: selected webcam '" << webcam_name << "' with camera URL " << camera_url; return true; } @@ -2880,8 +2941,6 @@ void MoonrakerPrinterAgent::perform_connection_async(const std::string& dev_id, BOOST_LOG_TRIVIAL(warning) << "MoonrakerPrinterAgent: Initial status query failed: " << error_msg; } - fetch_webcam_info(base_url, api_key, generation); - // Start WebSocket status stream start_status_stream(dev_id, base_url, api_key); #endif diff --git a/src/slic3r/Utils/MoonrakerPrinterAgent.hpp b/src/slic3r/Utils/MoonrakerPrinterAgent.hpp index 19ef02ee22..8750085181 100644 --- a/src/slic3r/Utils/MoonrakerPrinterAgent.hpp +++ b/src/slic3r/Utils/MoonrakerPrinterAgent.hpp @@ -73,10 +73,9 @@ public: int set_on_local_connect_fn(OnLocalConnectedFn fn) override; int set_on_local_message_fn(OnMessageFn fn) override; int set_queue_on_main_fn(QueueOnMainFn fn) override; - - // Pull-mode agent (on-demand filament sync) - FilamentSyncMode get_filament_sync_mode() const override { return FilamentSyncMode::pull; } bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull) override; + CameraStreamMode get_camera_stream_mode() const override; + std::string get_camera_url() const override; protected: struct MoonrakerDeviceInfo @@ -153,7 +152,6 @@ private: bool fetch_object_list(const std::string& base_url, const std::string& api_key, std::set& objects, std::string& error) const; bool query_printer_status(const std::string& base_url, const std::string& api_key, nlohmann::json& status, std::string& error) const; - bool fetch_webcam_info(const std::string& base_url, const std::string& api_key, uint64_t generation); void announce_printhost_device(); void dispatch_local_connect(int state, const std::string& dev_id, const std::string& msg); @@ -188,6 +186,12 @@ private: const std::string& api_key, uint64_t generation); + // why: a printer with no /server/webcams/list entry can still name its stream directly; + // subclasses (e.g. printers with a fixed webcam path) can override this instead. + virtual std::string webcam_stream_override(const std::string& base_url) const { return {}; } + void refresh_webcam_info() const; + bool fetch_webcam_info(const std::string& base_url, const std::string& api_key, uint64_t generation) const; + // System-specific filament fetch methods bool fetch_hh_filament_info(std::vector& trays, int& max_lane_index); bool fetch_moonraker_filament_data(std::vector& trays, int& max_lane_index); @@ -219,9 +223,14 @@ private: // note: guarded by payload_mutex; filled by refresh_thumbnail_url(), empty url = looked up, none found std::string thumbnail_filename; std::string thumbnail_url; - std::string webcam_stream_url; + mutable std::string webcam_stream_url; + mutable CameraStreamMode webcam_stream_mode = CameraStreamMode::none; + mutable uint64_t webcam_info_last_lookup_ms = 0; + mutable uint64_t webcam_info_generation = 0; unsigned thumbnail_lookup_attempts = 0; + static constexpr uint64_t WEBCAM_INFO_REFRESH_INTERVAL_MS = 1000; + std::atomic next_jsonrpc_id{1}; std::set available_objects; // Track for feature detection bool assumed_light_on = false; diff --git a/src/slic3r/Utils/NetworkAgent.cpp b/src/slic3r/Utils/NetworkAgent.cpp index eca59d30ce..01ed03f84f 100644 --- a/src/slic3r/Utils/NetworkAgent.cpp +++ b/src/slic3r/Utils/NetworkAgent.cpp @@ -1026,6 +1026,20 @@ bool NetworkAgent::fetch_filament_info(std::string dev_id, FilamentSyncMode sync return false; } +CameraStreamMode NetworkAgent::get_camera_stream_mode() const +{ + if (m_printer_agent) + return m_printer_agent->get_camera_stream_mode(); + return CameraStreamMode::none; +} + +std::string NetworkAgent::get_local_camera_stream_url() const +{ + if (m_printer_agent) + return m_printer_agent->get_camera_url(); + return {}; +} + int NetworkAgent::request_bind_ticket(std::string* ticket) { if (m_printer_agent) diff --git a/src/slic3r/Utils/NetworkAgent.hpp b/src/slic3r/Utils/NetworkAgent.hpp index e9d05143a1..42ded7e885 100644 --- a/src/slic3r/Utils/NetworkAgent.hpp +++ b/src/slic3r/Utils/NetworkAgent.hpp @@ -181,6 +181,8 @@ public: int start_sdcard_print(PrintParams params, OnUpdateStatusFn update_fn, WasCancelledFn cancel_fn); FilamentSyncMode get_filament_sync_mode() const; bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull); + CameraStreamMode get_camera_stream_mode() const; + std::string get_local_camera_stream_url() const; int request_bind_ticket(std::string* ticket); int get_hms_snapshot(std::string dev_id, std::string file_name, std::function callback); diff --git a/src/slic3r/Utils/SnapmakerPrinterAgent.cpp b/src/slic3r/Utils/SnapmakerPrinterAgent.cpp index fe1a29af53..cd2241e0d6 100644 --- a/src/slic3r/Utils/SnapmakerPrinterAgent.cpp +++ b/src/slic3r/Utils/SnapmakerPrinterAgent.cpp @@ -96,15 +96,6 @@ void SnapmakerPrinterAgent::on_status_loop_tick(const std::string& dev_id) } } -int SnapmakerPrinterAgent::connect_printer(std::string dev_id, std::string dev_ip, std::string username, std::string password, bool use_ssl) -{ - const int rtn = MoonrakerPrinterAgent::connect_printer(dev_id, dev_ip, username, password, use_ssl); - if (rtn == BAMBU_NETWORK_SUCCESS) { - start_camera_monitor(); - } - return rtn; -} - int SnapmakerPrinterAgent::command_start_camera(std::string dev_id) { (void) dev_id; @@ -149,127 +140,139 @@ std::string SnapmakerPrinterAgent::combine_filament_type(const std::string& type bool SnapmakerPrinterAgent::fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode) { + (void) dev_id; if (sync_mode != get_filament_sync_mode()) return false; - std::string url = join_url(device_info.base_url, "/printer/objects/query?print_task_config&filament_detect"); + const std::string base_url = device_info.base_url; + const std::string api_key = device_info.api_key; - std::string response_body; - bool success = false; - std::string http_error; + filament_fetch_in_flight.fetch_add(1, std::memory_order_relaxed); - auto http = Http::get(url); - if (!device_info.api_key.empty()) { - http.header("X-Api-Key", device_info.api_key); - } - http.timeout_connect(5) - .timeout_max(10) - .on_complete([&](std::string body, unsigned status) { - if (status == 200) { - response_body = body; - success = true; - } else { - http_error = "HTTP error: " + std::to_string(status); - } - }) - .on_error([&](std::string body, std::string err, unsigned status) { - http_error = err; - if (status > 0) { - http_error += " (HTTP " + std::to_string(status) + ")"; - } - }) - .perform_sync(); + std::thread([this, base_url, api_key]() { + struct InFlightGuard + { + std::atomic& counter; + ~InFlightGuard() { counter.fetch_sub(1, std::memory_order_relaxed); } + } guard{filament_fetch_in_flight}; - if (!success) { - BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: HTTP request failed: " << http_error; - return false; - } + const std::string url = join_url(base_url, "/printer/objects/query?print_task_config&filament_detect"); - auto json = nlohmann::json::parse(response_body, nullptr, false, true); - if (json.is_discarded()) { - BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: Invalid JSON response"; - return false; - } + std::string response_body; + bool success = false; + std::string http_error; - // Navigate to result.status.print_task_config - if (!json.contains("result") || !json["result"].contains("status") || - !json["result"]["status"].contains("print_task_config")) { - BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: Missing print_task_config in response"; - return false; - } - - auto& ptc = json["result"]["status"]["print_task_config"]; - - // Read parallel arrays from print_task_config - auto filament_exist = ptc.value("filament_exist", std::vector{}); - auto filament_type = ptc.value("filament_type", std::vector{}); - auto filament_sub_type = ptc.value("filament_sub_type", std::vector{}); - auto filament_color = ptc.value("filament_color_rgba", std::vector{}); - auto filament_vendor = ptc.value("filament_vendor", std::vector{}); - - const int slot_count = static_cast(filament_exist.size()); - if (slot_count == 0) { - BOOST_LOG_TRIVIAL(info) << "SnapmakerPrinterAgent::fetch_filament_info: No filament slots reported"; - return false; - } - - // Read NFC filament_detect data for temperature info (optional) - nlohmann::json nfc_info; - if (json["result"]["status"].contains("filament_detect") && - json["result"]["status"]["filament_detect"].contains("info")) { - nfc_info = json["result"]["status"]["filament_detect"]["info"]; - } - - static const std::string empty_str; - static const std::string default_color = "FFFFFFFF"; - - std::vector trays; - trays.reserve(slot_count); - - for (int i = 0; i < slot_count; ++i) { - AmsTrayData tray; - tray.slot_index = i; - tray.has_filament = filament_exist[i]; - - if (tray.has_filament) { - tray.tray_type = combine_filament_type(safe_at(filament_type, i, empty_str), - safe_at(filament_sub_type, i, empty_str)); - tray.tray_color = safe_at(filament_color, i, default_color); - - auto* bundle = GUI::wxGetApp().preset_bundle; - // Try to find a matching preset for this filament based on vendor, type and color. - // If not found, default to traditional search by type only or generic type mapping. - if (bundle) { - std::string vendor = safe_at(filament_vendor, i, empty_str); - std::string filament_id = find_closest_color_preset_by_vendor_and_type(bundle->filaments, vendor, tray.tray_type, - tray.tray_color); - - if (!filament_id.empty()) { - tray.tray_info_idx = filament_id; - BOOST_LOG_TRIVIAL(warning) << "Filament sync: Found manufacturer-specific profile for slot " << i << ": " - << filament_id; + auto http = Http::get(url); + if (!api_key.empty()) { + http.header("X-Api-Key", api_key); + } + http.timeout_connect(5) + .timeout_max(10) + .on_complete([&](std::string body, unsigned status) { + if (status == 200) { + response_body = body; + success = true; } else { - tray.tray_info_idx = bundle->filaments.filament_id_by_type(tray.tray_type); + http_error = "HTTP error: " + std::to_string(status); } - } else { - tray.tray_info_idx = map_filament_type_to_generic_id(tray.tray_type); - } + }) + .on_error([&](std::string body, std::string err, unsigned status) { + http_error = err; + if (status > 0) { + http_error += " (HTTP " + std::to_string(status) + ")"; + } + }) + .perform_sync(); - // Extract NFC temperature data if available - if (nfc_info.is_array() && i < static_cast(nfc_info.size()) && nfc_info[i].is_object()) { - auto& nfc_slot = nfc_info[i]; - std::string vendor = nfc_slot.value("VENDOR", "NONE"); - if (vendor != "NONE" && !vendor.empty()) { - tray.bed_temp = nfc_slot.value("BED_TEMP", 0); - tray.nozzle_temp = nfc_slot.value("FIRST_LAYER_TEMP", 0); - } - } + if (!success) { + BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: HTTP request failed: " << http_error; + return; } - trays.emplace_back(std::move(tray)); - } + auto json = nlohmann::json::parse(response_body, nullptr, false, true); + if (json.is_discarded()) { + BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: Invalid JSON response"; + return; + } + + // Navigate to result.status.print_task_config + if (!json.contains("result") || !json["result"].contains("status") || !json["result"]["status"].contains("print_task_config")) { + BOOST_LOG_TRIVIAL(warning) << "SnapmakerPrinterAgent::fetch_filament_info: Missing print_task_config in response"; + return; + } + + auto& ptc = json["result"]["status"]["print_task_config"]; + + // Read parallel arrays from print_task_config + auto filament_exist = ptc.value("filament_exist", std::vector{}); + auto filament_type = ptc.value("filament_type", std::vector{}); + auto filament_sub_type = ptc.value("filament_sub_type", std::vector{}); + auto filament_color = ptc.value("filament_color_rgba", std::vector{}); + auto filament_vendor = ptc.value("filament_vendor", std::vector{}); + + const int slot_count = static_cast(filament_exist.size()); + if (slot_count == 0) { + BOOST_LOG_TRIVIAL(info) << "SnapmakerPrinterAgent::fetch_filament_info: No filament slots reported"; + return; + } + + // Read NFC filament_detect data for temperature info (optional) + nlohmann::json nfc_info; + if (json["result"]["status"].contains("filament_detect") && json["result"]["status"]["filament_detect"].contains("info")) { + nfc_info = json["result"]["status"]["filament_detect"]["info"]; + } + + static const std::string empty_str; + static const std::string default_color = "FFFFFFFF"; + + std::vector trays; + trays.reserve(slot_count); + + for (int i = 0; i < slot_count; ++i) { + AmsTrayData tray; + tray.slot_index = i; + tray.has_filament = filament_exist[i]; + + if (tray.has_filament) { + tray.tray_type = combine_filament_type(safe_at(filament_type, i, empty_str), safe_at(filament_sub_type, i, empty_str)); + tray.tray_color = safe_at(filament_color, i, default_color); + + auto* bundle = GUI::wxGetApp().preset_bundle; + // Try to find a matching preset for this filament based on vendor, type and color. + // If not found, default to traditional search by type only or generic type mapping. + if (bundle) { + std::string vendor = safe_at(filament_vendor, i, empty_str); + std::string filament_id = find_closest_color_preset_by_vendor_and_type(bundle->filaments, vendor, tray.tray_type, + tray.tray_color); + + if (!filament_id.empty()) { + tray.tray_info_idx = filament_id; + BOOST_LOG_TRIVIAL(warning) + << "Filament sync: Found manufacturer-specific profile for slot " << i << ": " << filament_id; + } else { + tray.tray_info_idx = bundle->filaments.filament_id_by_type(tray.tray_type); + } + } else { + tray.tray_info_idx = map_filament_type_to_generic_id(tray.tray_type); + } + + // Extract NFC temperature data if available + if (nfc_info.is_array() && i < static_cast(nfc_info.size()) && nfc_info[i].is_object()) { + auto& nfc_slot = nfc_info[i]; + std::string vendor = nfc_slot.value("VENDOR", "NONE"); + if (vendor != "NONE" && !vendor.empty()) { + tray.bed_temp = nfc_slot.value("BED_TEMP", 0); + tray.nozzle_temp = nfc_slot.value("FIRST_LAYER_TEMP", 0); + } + } + } + + trays.emplace_back(std::move(tray)); + } + + build_ams_payload(1, slot_count - 1, trays); + }).detach(); - build_ams_payload(1, slot_count - 1, trays); return true; } diff --git a/src/slic3r/Utils/SnapmakerPrinterAgent.hpp b/src/slic3r/Utils/SnapmakerPrinterAgent.hpp index 5f7b6b1c30..c1e35701c3 100644 --- a/src/slic3r/Utils/SnapmakerPrinterAgent.hpp +++ b/src/slic3r/Utils/SnapmakerPrinterAgent.hpp @@ -1,5 +1,6 @@ #pragma once +#include "IPrinterAgent.hpp" #include "MoonrakerPrinterAgent.hpp" #include @@ -21,6 +22,8 @@ public: FilamentSyncMode get_filament_sync_mode() const override; int connect_printer(std::string dev_id, std::string dev_ip, std::string username, std::string password, bool use_ssl) override; int command_start_camera(std::string dev_id) override; + CameraStreamMode get_camera_stream_mode() const override { return CameraStreamMode::http_snapshot; } + std::string get_camera_url() const override { return device_info.base_url + "/server/files/camera/monitor.jpg"; } private: // Combine filament_type + filament_sub_type into a unified type string diff --git a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.cpp b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.cpp index d428775c12..c9ea52e8bb 100644 --- a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.cpp +++ b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.cpp @@ -23,6 +23,14 @@ void PrinterAgentPluginCapability::RegisterBindings(pybind11::module_& module) .value("Pull", FilamentSyncMode::pull) .export_values(); + py::enum_(printer_agent_module, "CameraStreamMode") + .value("None_", CameraStreamMode::none) + .value("HTTP", CameraStreamMode::http) + .value("HTTP_SNAPSHOT", CameraStreamMode::http_snapshot) + .value("RTSP", CameraStreamMode::rtsp) + .value("WebRTC", CameraStreamMode::webrtc) + .export_values(); + py::class_(printer_agent_module, "AgentInfo") .def(py::init<>()) .def(py::init([](std::string id, std::string name, std::string version, std::string description) { @@ -105,7 +113,9 @@ void PrinterAgentPluginCapability::RegisterBindings(pybind11::module_& module) .def("start_send_gcode_to_sdcard", &PrinterAgentPluginCapability::start_send_gcode_to_sdcard) .def("start_local_print", &PrinterAgentPluginCapability::start_local_print) .def("get_filament_sync_mode", &PrinterAgentPluginCapability::get_filament_sync_mode) + .def("get_camera_stream_mode", &PrinterAgentPluginCapability::get_camera_stream_mode) .def("fetch_filament_info", &PrinterAgentPluginCapability::fetch_filament_info) + .def("get_camera_url", &PrinterAgentPluginCapability::get_camera_url) .def("check_cert", &PrinterAgentPluginCapability::check_cert) .def("install_device_cert", &PrinterAgentPluginCapability::install_device_cert) .def("ping_bind", &PrinterAgentPluginCapability::ping_bind) diff --git a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.hpp b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.hpp index af869b68f3..ae602ca38c 100644 --- a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.hpp +++ b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapability.hpp @@ -41,6 +41,8 @@ public: int start_local_print(PrintParams params, OnUpdateStatusFn update_fn, WasCancelledFn cancel_fn) override = 0; FilamentSyncMode get_filament_sync_mode() const override = 0; bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull) override = 0; + CameraStreamMode get_camera_stream_mode() const override = 0; + std::string get_camera_url() const override = 0; int check_cert() override = 0; void install_device_cert(std::string dev_id, bool lan_only) override = 0; diff --git a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapabilityTrampoline.hpp b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapabilityTrampoline.hpp index 7e017ca55e..88051ed649 100644 --- a/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapabilityTrampoline.hpp +++ b/src/slic3r/plugin/pluginTypes/printerAgent/PrinterAgentPluginCapabilityTrampoline.hpp @@ -5,7 +5,10 @@ #include "../../PyPluginTrampoline.hpp" #include "IPrinterAgent.hpp" +#include "pybind11/pybind11.h" +#include #include +#include namespace Slic3r { class PyPrinterAgentPluginCapabilityTrampoline : public PyPluginCommonTrampoline @@ -96,6 +99,19 @@ public: get_filament_sync_mode); } + CameraStreamMode get_camera_stream_mode() const override + { + ORCA_PY_OVERRIDE_AUDITED( + ::Slic3r::PluginAuditManager::AuditMode::Loading, [] {}, PYBIND11_OVERRIDE_PURE, CameraStreamMode, PrinterAgentPluginCapability, + get_camera_stream_mode); + } + + std::string get_camera_url() const override + { + ORCA_PY_OVERRIDE_AUDITED( + ::Slic3r::PluginAuditManager::AuditMode::Loading, [] {}, PYBIND11_OVERRIDE_PURE, std::string, PrinterAgentPluginCapability, get_camera_url); + } + bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull) override { ORCA_PY_OVERRIDE_AUDITED( From e8f089dfa401ce525cc69cc3de47d227cc4c121b Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Tue, 1 Sep 2026 18:04:37 +0800 Subject: [PATCH 2/5] fix: build & access code UI --- src/slic3r/GUI/ReleaseNote.cpp | 1 - src/slic3r/Utils/NetworkAgent.hpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/slic3r/GUI/ReleaseNote.cpp b/src/slic3r/GUI/ReleaseNote.cpp index 6c9adb015b..034f40283f 100644 --- a/src/slic3r/GUI/ReleaseNote.cpp +++ b/src/slic3r/GUI/ReleaseNote.cpp @@ -2063,7 +2063,6 @@ void InputIpAddressDialog::on_text(wxCommandEvent &evt) if (str_access_code.IsEmpty()) { str_access_code = "88888888"; - m_input_access_code->GetTextCtrl()->SetValue(str_access_code); } auto str_name = m_input_printer_name->GetTextCtrl()->GetValue().Strip(wxString::both); diff --git a/src/slic3r/Utils/NetworkAgent.hpp b/src/slic3r/Utils/NetworkAgent.hpp index 42ded7e885..7eb220b341 100644 --- a/src/slic3r/Utils/NetworkAgent.hpp +++ b/src/slic3r/Utils/NetworkAgent.hpp @@ -5,6 +5,7 @@ #include "libslic3r/ProjectTask.hpp" #include "ICloudServiceAgent.hpp" +#include "IPrinterAgent.hpp" #include #include @@ -14,7 +15,6 @@ namespace Slic3r { class IPrinterAgent; -enum class FilamentSyncMode; // Forward declaration class BBLNetworkPlugin; From 1e8805d43d3ae24eeaa5f83c292694842bad270f Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Fri, 28 Aug 2026 18:10:31 +0800 Subject: [PATCH 3/5] feat: connect to cloud printer and monitor From 4320cc78d9117a1788b8d8fa10165ae00b120262 Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Tue, 1 Sep 2026 18:59:20 +0800 Subject: [PATCH 4/5] feat: camera via webrtc --- CMakeLists.txt | 43 ++ deps/CMakeLists.txt | 3 + deps/DataChannel/DataChannel.cmake | 20 + src/slic3r/CMakeLists.txt | 9 +- src/slic3r/GUI/IMediaController.hpp | 11 + src/slic3r/GUI/MediaPlayCtrl.cpp | 110 ++++- src/slic3r/GUI/MediaPlayCtrl.h | 6 + src/slic3r/GUI/StatusPanel.cpp | 14 +- src/slic3r/GUI/WebRtcFrameAssembler.cpp | 86 ++++ src/slic3r/GUI/WebRtcFrameAssembler.hpp | 39 ++ src/slic3r/GUI/WebRtcMediaController.cpp | 449 ++++++++++++++++++ src/slic3r/GUI/WebRtcMediaController.hpp | 96 ++++ src/slic3r/GUI/wxMediaCtrl3.cpp | 62 +++ src/slic3r/GUI/wxMediaCtrl3.h | 13 +- src/slic3r/Utils/ICameraSignalingChannel.hpp | 40 ++ src/slic3r/Utils/IPrinterAgent.hpp | 10 + src/slic3r/Utils/NetworkAgent.cpp | 8 + src/slic3r/Utils/NetworkAgent.hpp | 1 + .../Utils/OrcaCloudSignalingChannel.cpp | 320 +++++++++++++ .../Utils/OrcaCloudSignalingChannel.hpp | 63 +++ src/slic3r/Utils/OrcaPrinterAgent.cpp | 35 ++ src/slic3r/Utils/OrcaPrinterAgent.hpp | 7 + tests/slic3rutils/CMakeLists.txt | 1 + .../test_webrtc_frame_assembler.cpp | 67 +++ 24 files changed, 1505 insertions(+), 8 deletions(-) create mode 100644 deps/DataChannel/DataChannel.cmake create mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.cpp create mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.hpp create mode 100644 src/slic3r/GUI/WebRtcMediaController.cpp create mode 100644 src/slic3r/GUI/WebRtcMediaController.hpp create mode 100644 src/slic3r/Utils/ICameraSignalingChannel.hpp create mode 100644 src/slic3r/Utils/OrcaCloudSignalingChannel.cpp create mode 100644 src/slic3r/Utils/OrcaCloudSignalingChannel.hpp create mode 100644 tests/slic3rutils/test_webrtc_frame_assembler.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b954ef753..457a7b96c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -785,6 +785,49 @@ find_package(OpenSSL REQUIRED) find_package(CURL REQUIRED) find_package(Freetype REQUIRED) +if (SLIC3R_GUI) + # LibDataChannel's installed export references its bundled dependencies, + # but does not install their CMake targets. Recreate those targets from + # the same dependency prefix before loading the LibDataChannel config. + if (NOT TARGET Usrsctp::usrsctp) + find_library(_ORCA_USRSCTP_LIBRARY NAMES usrsctp + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_USRSCTP_LIBRARY) + add_library(Usrsctp::usrsctp UNKNOWN IMPORTED GLOBAL) + set_target_properties(Usrsctp::usrsctp PROPERTIES + IMPORTED_LOCATION "${_ORCA_USRSCTP_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "Threads::Threads") + endif() + endif() + + if (NOT TARGET libSRTP::srtp2) + find_library(_ORCA_SRTP_LIBRARY NAMES srtp2 + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_SRTP_LIBRARY) + add_library(libSRTP::srtp2 UNKNOWN IMPORTED GLOBAL) + set_target_properties(libSRTP::srtp2 PROPERTIES + IMPORTED_LOCATION "${_ORCA_SRTP_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "OpenSSL::Crypto") + endif() + endif() + + if (NOT TARGET LibJuice::LibJuice) + find_library(_ORCA_LIBJUICE_LIBRARY NAMES juice + PATHS "${CMAKE_PREFIX_PATH}/lib" NO_DEFAULT_PATH) + if (_ORCA_LIBJUICE_LIBRARY) + add_library(LibJuice::LibJuice UNKNOWN IMPORTED GLOBAL) + set_target_properties(LibJuice::LibJuice PROPERTIES + IMPORTED_LOCATION "${_ORCA_LIBJUICE_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_LINK_LIBRARIES "Threads::Threads") + endif() + endif() + + find_package(LibDataChannel CONFIG REQUIRED) +endif() + add_library(libcurl INTERFACE) target_link_libraries(libcurl INTERFACE CURL::libcurl) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index ed3af70d03..a645e3a752 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -389,6 +389,8 @@ if(NOT OPENSSL_FOUND) set(OPENSSL_PKG dep_OpenSSL) endif() +include(DataChannel/DataChannel.cmake) + # we don't want to load a "wrong" openssl when loading curl # so, just don't even bother # ...i think this is how it works? change if wrong @@ -461,6 +463,7 @@ set(_dep_list dep_wxInspector dep_FFMPEG dep_Assimp + dep_DataChannel ) if (MSVC) diff --git a/deps/DataChannel/DataChannel.cmake b/deps/DataChannel/DataChannel.cmake new file mode 100644 index 0000000000..5e9b9a358b --- /dev/null +++ b/deps/DataChannel/DataChannel.cmake @@ -0,0 +1,20 @@ +# libdatachannel is the native ICE/DTLS/SCTP/SRTP implementation used by the +# GUI WebRTC camera controller. Keep the source revision fixed: the signaling +# protocol is evolving independently of this transport dependency. +orcaslicer_add_cmake_project(DataChannel + CMAKE_ARGS + -DNO_EXAMPLES=ON + -DNO_TESTS=ON + -DNO_WEBSOCKET=ON + -DNO_MEDIA=OFF + -DUSE_NICE=OFF + -DUSE_SYSTEM_SRTP=OFF + -DUSE_SYSTEM_JUICE=OFF + -DUSE_SYSTEM_USRSCTP=OFF + -DOPENSSL_ROOT_DIR:PATH=${DESTDIR} + -DOPENSSL_USE_STATIC_LIBS=ON + GIT_REPOSITORY https://github.com/paullouisageneau/libdatachannel.git + GIT_TAG v0.22.2 + GIT_SHALLOW ON + GIT_SUBMODULES_RECURSE ON +) diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 8308161a7b..322b3b3f0e 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -344,6 +344,10 @@ set(SLIC3R_GUI_SOURCES GUI/MediaFilePanel.h GUI/MediaPlayCtrl.cpp GUI/MediaPlayCtrl.h + GUI/WebRtcFrameAssembler.cpp + GUI/WebRtcFrameAssembler.hpp + GUI/WebRtcMediaController.cpp + GUI/WebRtcMediaController.hpp GUI/MeshUtils.cpp GUI/MeshUtils.hpp GUI/ModelMall.cpp @@ -723,10 +727,13 @@ set(SLIC3R_GUI_SOURCES Utils/NetworkAgentFactory.cpp Utils/ICloudServiceAgent.hpp Utils/IPrinterAgent.hpp + Utils/ICameraSignalingChannel.hpp Utils/OrcaCloudServiceAgent.cpp Utils/OrcaCloudServiceAgent.hpp Utils/OrcaPrinterAgent.cpp Utils/OrcaPrinterAgent.hpp + Utils/OrcaCloudSignalingChannel.cpp + Utils/OrcaCloudSignalingChannel.hpp Utils/QidiPrinterAgent.cpp Utils/QidiPrinterAgent.hpp Utils/SnapmakerPrinterAgent.cpp @@ -855,7 +862,7 @@ else() set(_opengl_link_lib OpenGL::GL) endif() -target_link_libraries(libslic3r_gui libslic3r cereal::cereal imgui imguizmo minilzo libvgcode md4c-html glad ${_opengl_link_lib} hidapi mdns ${wxWidgets_LIBRARIES} glfw libcurl OpenSSL::SSL OpenSSL::Crypto noise::noise pybind11::embed) +target_link_libraries(libslic3r_gui libslic3r cereal::cereal imgui imguizmo minilzo libvgcode md4c-html glad ${_opengl_link_lib} hidapi mdns ${wxWidgets_LIBRARIES} glfw libcurl OpenSSL::SSL OpenSSL::Crypto LibDataChannel::LibDataChannel noise::noise pybind11::embed) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") # Linux finds wxWidgets in module mode, whose include dirs and definitions diff --git a/src/slic3r/GUI/IMediaController.hpp b/src/slic3r/GUI/IMediaController.hpp index 982157e5bc..411932ec7d 100644 --- a/src/slic3r/GUI/IMediaController.hpp +++ b/src/slic3r/GUI/IMediaController.hpp @@ -3,6 +3,8 @@ #include #include +#include + #include namespace Slic3r { namespace GUI { @@ -10,6 +12,8 @@ namespace Slic3r { namespace GUI { class IMediaController { public: + virtual ~IMediaController() = default; + virtual void Load(wxURI url) = 0; // The default keeps existing media controllers unaware of camera-specific modes. @@ -29,6 +33,13 @@ public: virtual wxSize GetVideoSize() const { return {}; }; + virtual void StartSession(std::unique_ptr channel) + { + (void) channel; + } + + virtual void StopSession() {} + private: }; diff --git a/src/slic3r/GUI/MediaPlayCtrl.cpp b/src/slic3r/GUI/MediaPlayCtrl.cpp index 77a183f090..9b7198005a 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.cpp +++ b/src/slic3r/GUI/MediaPlayCtrl.cpp @@ -134,6 +134,11 @@ MediaPlayCtrl::MediaPlayCtrl(wxWindow *parent, wxMediaCtrl3 *media_ctrl, const w MediaPlayCtrl::~MediaPlayCtrl() { + m_webrtc_stopping = true; + if (m_webrtc_ctrl) + m_webrtc_ctrl->StopSession(); + m_media_ctrl->EndExternalStream(); + m_webrtc_stopping = false; { boost::unique_lock lock(m_mutex); m_tasks.push_back(""); @@ -159,7 +164,14 @@ CameraStreamMode MediaPlayCtrl::current_mode() const void MediaPlayCtrl::SetMachineObject(MachineObject* obj) { - switch (current_mode()) { + const CameraStreamMode mode = current_mode(); + if (mode != m_last_mode) { + if (m_last_state != MEDIASTATE_IDLE) + Stop(" "); + m_last_mode = mode; + } + + switch (mode) { case CameraStreamMode::http: case CameraStreamMode::http_snapshot: case CameraStreamMode::rtsp: { @@ -184,6 +196,28 @@ void MediaPlayCtrl::SetMachineObject(MachineObject* obj) Play(); return; } + case CameraStreamMode::webrtc: { + std::string machine = obj ? obj->get_dev_id() : ""; + m_camera_exists = obj != nullptr; + Enable(obj != nullptr); + const bool changed = machine != m_machine; + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::SetMachineObject webrtc: changed=" << changed + << " last_state=" << m_last_state << " web_user_stopped=" << m_web_user_stopped; + m_machine = machine; + m_url.clear(); + m_agent_camera_url.clear(); + if (!changed) { + if (m_last_state == MEDIASTATE_IDLE && IsEnabled() && !m_web_user_stopped) + Play(); + return; + } + m_web_user_stopped = false; + if (m_last_state != MEDIASTATE_IDLE) + Stop(" "); + if (IsEnabled()) + Play(); + return; + } default: break; } @@ -321,6 +355,48 @@ void MediaPlayCtrl::Play() m_button_play->SetIcon("media_stop"); load(); return; + case CameraStreamMode::webrtc: { + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Play webrtc: last_state=" << m_last_state + << " next_retry_valid=" << m_next_retry.IsValid() + << " next_retry_future=" << (m_next_retry.IsValid() && wxDateTime::Now() < m_next_retry) + << " failed_retry=" << m_failed_retry << " shown=" << IsShownOnScreen(); + if (m_webrtc_ctrl && m_webrtc_ctrl->is_active()) { + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Play webrtc: session already active, ignoring"; + return; + } + if (m_next_retry.IsValid() && wxDateTime::Now() < m_next_retry) + return; + if (!IsShownOnScreen() || m_last_state != MEDIASTATE_IDLE) + return; + m_failed_code = 0; + if (m_machine.empty() || !IsEnabled() || !m_camera_exists) { + Stop(_L("Please confirm if the printer is connected.")); + return; + } + auto agent = wxGetApp().getAgent(); + auto channel = agent ? agent->create_camera_signaling_channel(m_machine) : nullptr; + if (!channel) { + Stop(_L("Sign in to OrcaCloud to view the camera.")); + return; + } + if (!m_webrtc_ctrl) { + m_webrtc_ctrl = std::make_unique( + [this](const wxImage& image, wxSize size) { m_media_ctrl->SetExternalFrame(image, size); }, + [this, token = std::weak_ptr(m_token)](WebRtcMediaController::Status status) { + if (token.expired()) + return; + CallAfter([this, status] { on_webrtc_status(status); }); + }); + } + m_button_play->SetIcon("media_stop"); + m_media_ctrl->BeginExternalStream(); + m_last_state = MEDIASTATE_INITIALIZING; + SetStatus(_L("Initializing..."), false); + m_webrtc_stopping = false; + m_webrtc_ctrl->StartSession(std::move(channel)); + m_webrtc_epoch = m_webrtc_ctrl->epoch(); + return; + } default: break; } @@ -465,6 +541,17 @@ void MediaPlayCtrl::StopWebStream() void MediaPlayCtrl::Stop(wxString const &msg, wxString const &msg2) { + const bool webrtc_active = m_webrtc_ctrl && (m_last_mode == CameraStreamMode::webrtc || + current_mode() == CameraStreamMode::webrtc); + BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::Stop: last_state=" << m_last_state + << " webrtc_active=" << webrtc_active << " failed_code=" << m_failed_code + << " msg='" << msg.ToUTF8().data() << "'"; + if (webrtc_active) { + m_webrtc_stopping = true; + m_webrtc_ctrl->StopSession(); + m_media_ctrl->EndExternalStream(); + m_webrtc_stopping = false; + } switch (current_mode()) { case CameraStreamMode::http: case CameraStreamMode::http_snapshot: @@ -555,6 +642,27 @@ void MediaPlayCtrl::Stop(wxString const &msg, wxString const &msg2) m_next_retry = wxDateTime::Now() + wxTimeSpan::Seconds(5 * m_failed_retry); } +void MediaPlayCtrl::on_webrtc_status(WebRtcMediaController::Status status) +{ + // Drop CallAfter-queued events from a superseded StartSession attempt. + if (status.epoch != m_webrtc_epoch) + return; + if (status.kind == WebRtcMediaController::Status::Connecting) { + m_last_state = MEDIASTATE_INITIALIZING; + SetStatus(_L("Initializing..."), false); + } else if (status.kind == WebRtcMediaController::Status::Playing) { + m_last_state = wxMEDIASTATE_PLAYING; + m_failed_code = 0; + m_failed_retry = 0; + SetStatus(_L("Playing..."), false); + } else if (status.kind == WebRtcMediaController::Status::Failed) { + m_failed_code = static_cast(status.code) + 1; + Stop(); + } + // Status::Stopped needs no action: a genuine failure arrives as Failed, and + // a stop we initiated is already handled by Stop() itself. +} + void MediaPlayCtrl::TogglePlay() { BOOST_LOG_TRIVIAL(info) << "MediaPlayCtrl::TogglePlay"; diff --git a/src/slic3r/GUI/MediaPlayCtrl.h b/src/slic3r/GUI/MediaPlayCtrl.h index e8ad0ee82c..64fb75498e 100644 --- a/src/slic3r/GUI/MediaPlayCtrl.h +++ b/src/slic3r/GUI/MediaPlayCtrl.h @@ -10,6 +10,7 @@ #include "wxMediaCtrl3.h" #include "IMediaController.hpp" +#include "WebRtcMediaController.hpp" #include "slic3r/Utils/IPrinterAgent.hpp" #include @@ -60,6 +61,7 @@ protected: void TogglePlay(); void SetStatus(wxString const &msg, bool hyperlink = true); + void on_webrtc_status(WebRtcMediaController::Status status); private: void load(); @@ -85,6 +87,10 @@ private: wxMediaCtrl3 * m_media_ctrl; IMediaController * m_web_ctrl = nullptr; + std::unique_ptr m_webrtc_ctrl; + CameraStreamMode m_last_mode = CameraStreamMode::none; + bool m_webrtc_stopping = false; + std::uint64_t m_webrtc_epoch = 0; std::string m_agent_camera_url; bool m_web_user_stopped = false; wxMediaState m_last_state = MEDIASTATE_IDLE; diff --git a/src/slic3r/GUI/StatusPanel.cpp b/src/slic3r/GUI/StatusPanel.cpp index b00dee3dca..2d267df082 100644 --- a/src/slic3r/GUI/StatusPanel.cpp +++ b/src/slic3r/GUI/StatusPanel.cpp @@ -2317,10 +2317,16 @@ void StatusPanel::update_camera_state(MachineObject* obj) m_custom_camera_view->Show(); m_media_ctrl->Hide(); } - } else if (m_custom_camera_view->IsShown()) { - m_custom_camera_view->Hide(); - m_media_ctrl->Show(); - m_media_play_ctrl->StopWebStream(); + } else if (camera_mode == CameraStreamMode::rtsp || camera_mode == CameraStreamMode::webrtc || + m_custom_camera_view->IsShown()) { + // Only act on the actual transition away from the webview. Running this + // every tick would call StopWebStream() (which forces m_last_state to + // IDLE) on a live rtsp/webrtc session and desync the state machine. + if (m_custom_camera_view->IsShown()) { + m_custom_camera_view->Hide(); + m_media_ctrl->Show(); + m_media_play_ctrl->StopWebStream(); + } } //sdcard diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.cpp b/src/slic3r/GUI/WebRtcFrameAssembler.cpp new file mode 100644 index 0000000000..2771cc0887 --- /dev/null +++ b/src/slic3r/GUI/WebRtcFrameAssembler.cpp @@ -0,0 +1,86 @@ +#include "WebRtcFrameAssembler.hpp" + +#include +#include + +namespace Slic3r { namespace GUI { + +void WebRtcFrameAssembler::discard() +{ + m_active = false; + m_frame_id = 0; + m_chunk_count = 0; + m_received_chunks = 0; + m_total_size = 0; + m_chunks.clear(); + m_received.clear(); +} + +void WebRtcFrameAssembler::reset() +{ + discard(); +} + +void WebRtcFrameAssembler::feed(const std::byte* data, std::size_t len) +{ + if (data == nullptr || len < HeaderSize) + return; + + if (std::to_integer(data[0]) != 1) + return; + + const std::uint16_t chunk_index = static_cast((std::to_integer(data[2]) << 8) | + std::to_integer(data[3])); + const std::uint16_t chunk_count = static_cast((std::to_integer(data[4]) << 8) | + std::to_integer(data[5])); + const std::uint32_t frame_id = (static_cast(std::to_integer(data[6])) << 24) | + (static_cast(std::to_integer(data[7])) << 16) | + (static_cast(std::to_integer(data[8])) << 8) | + static_cast(std::to_integer(data[9])); + const std::size_t payload_size = len - HeaderSize; + + if (chunk_count == 0 || chunk_count > MaxChunkCount || chunk_index >= chunk_count || + payload_size > MaxChunkPayload) + return; + + if (!m_active || frame_id > m_frame_id) { + discard(); + m_active = true; + m_frame_id = frame_id; + m_chunk_count = chunk_count; + m_chunks.resize(chunk_count); + m_received.assign(chunk_count, false); + } else if (frame_id < m_frame_id) { + return; + } else if (chunk_count != m_chunk_count) { + discard(); + return; + } + + Frame& chunk = m_chunks[chunk_index]; + if (m_received[chunk_index]) + return; + + if (m_total_size > MaxFrameSize || payload_size > MaxFrameSize - m_total_size) { + discard(); + return; + } + + chunk.assign(data + HeaderSize, data + len); + m_received[chunk_index] = true; + m_total_size += payload_size; + ++m_received_chunks; + + if (m_received_chunks != m_chunk_count) + return; + + Frame frame; + frame.reserve(m_total_size); + for (const Frame& slice : m_chunks) + frame.insert(frame.end(), slice.begin(), slice.end()); + if (on_frame) + on_frame(std::move(frame)); + discard(); +} + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.hpp b/src/slic3r/GUI/WebRtcFrameAssembler.hpp new file mode 100644 index 0000000000..c526cad588 --- /dev/null +++ b/src/slic3r/GUI/WebRtcFrameAssembler.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +namespace Slic3r { namespace GUI { + +class WebRtcFrameAssembler { +public: + using Frame = std::vector; + + // The wire protocol uses a 10-byte header followed by one JPEG slice: + // version, flags, chunk index, chunk count, and frame id, all in network + // byte order where applicable. + static constexpr std::size_t HeaderSize = 10; + static constexpr std::size_t MaxChunkPayload = 16000; + static constexpr std::size_t MaxChunkCount = 4096; + static constexpr std::size_t MaxFrameSize = 8 * 1024 * 1024; + + std::function on_frame; + + void feed(const std::byte* data, std::size_t len); + void reset(); + +private: + void discard(); + + bool m_active = false; + std::uint32_t m_frame_id = 0; + std::uint16_t m_chunk_count = 0; + std::size_t m_received_chunks = 0; + std::size_t m_total_size = 0; + std::vector m_chunks; + std::vector m_received; +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.cpp b/src/slic3r/GUI/WebRtcMediaController.cpp new file mode 100644 index 0000000000..7eafbd4a3a --- /dev/null +++ b/src/slic3r/GUI/WebRtcMediaController.cpp @@ -0,0 +1,449 @@ +#include "WebRtcMediaController.hpp" + +#include "AVVideoDecoder.hpp" + +#include +#include + +#include +#include +#include + +#include + +#include + +namespace { +void init_rtc_logger_once() +{ + static std::once_flag flag; + std::call_once(flag, [] { + rtc::InitLogger(rtc::LogLevel::Verbose, [](rtc::LogLevel level, std::string message) { + BOOST_LOG_TRIVIAL(info) << "[rtc:" << static_cast(level) << "] " << message; + }); + }); +} +} // namespace + +extern "C" { +#include +} + +namespace Slic3r { namespace GUI { + +WebRtcMediaController::WebRtcMediaController(std::function frame_sink, + std::function on_status) + : m_frame_sink(std::move(frame_sink)) + , m_on_status(std::move(on_status)) +{ +} + +WebRtcMediaController::~WebRtcMediaController() +{ + StopSession(); +} + +void WebRtcMediaController::report(Status status) +{ + status.epoch = m_epoch.load(); + BOOST_LOG_TRIVIAL(info) << "WebRTC: report kind=" << static_cast(status.kind) + << " code=" << static_cast(status.code) << " epoch=" << status.epoch; + { + std::lock_guard lock(m_mutex); + if (status.kind == Status::Connecting) + m_state = static_cast(4); + else if (status.kind == Status::Playing) + m_state = wxMEDIASTATE_PLAYING; + else + m_state = static_cast(3); + } + if (m_on_status) + m_on_status(status); +} + +void WebRtcMediaController::StartSession(std::unique_ptr channel) +{ + // Tear down any previous attempt WITHOUT notifying: the Stopped that would + // otherwise be delivered (async, via CallAfter) races the new attempt's + // Connecting and makes the consumer cancel a session that is mid-connect. + teardown(false); + if (!channel) + return; + + m_epoch.fetch_add(1); + m_alive.store(true); + { + std::lock_guard lock(m_mutex); + m_signaling = std::move(channel); + m_chunk_queue.clear(); + m_nal_queue.clear(); + m_pending_candidates.clear(); + m_remote_description_set = false; + m_video_size = wxDefaultSize; + m_has_frame = false; + m_last_frame_time = {}; + } + + ICameraSignalingChannel* signaling = nullptr; + { + std::lock_guard lock(m_mutex); + signaling = m_signaling.get(); + } + signaling->on_ready = [this](std::vector servers) { + if (m_alive.load()) + on_ready(std::move(servers)); + }; + signaling->on_answer = [this](std::string sdp) { + if (m_alive.load()) + on_answer(std::move(sdp)); + }; + signaling->on_ice = [this](std::string candidate, std::string mid) { + if (m_alive.load()) + on_ice(std::move(candidate), std::move(mid)); + }; + signaling->on_unavailable = [this](CameraUnavailableReason reason, std::string detail) { + if (m_alive.load()) + on_unavailable(reason, std::move(detail)); + }; + + m_decode_thread = std::thread([this] { decode_loop(); }); + report({Status::Connecting}); + signaling->open(); +} + +void WebRtcMediaController::StopSession() +{ + teardown(true); +} + +void WebRtcMediaController::teardown(bool notify) +{ + const bool was_alive = m_alive.exchange(false); + if (!was_alive && !m_decode_thread.joinable()) + return; + + m_cond.notify_all(); + std::unique_ptr signaling; + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + signaling = std::move(m_signaling); + peer_connection = std::move(m_peer_connection); + m_data_channel.reset(); + m_video_track.reset(); + } + if (signaling) + signaling->close(); + if (peer_connection) + peer_connection->close(); + if (m_decode_thread.joinable()) + m_decode_thread.join(); + { + std::lock_guard lock(m_mutex); + m_chunk_queue.clear(); + m_nal_queue.clear(); + } + if (was_alive && notify) + report({Status::Stopped}); +} + +wxMediaState WebRtcMediaController::GetState() +{ + std::lock_guard lock(m_mutex); + return m_state; +} + +wxSize WebRtcMediaController::GetVideoSize() const +{ + std::lock_guard lock(m_mutex); + return m_video_size; +} + +void WebRtcMediaController::bind_data_channel(const std::shared_ptr& dc) +{ + const std::string label = dc->label(); + dc->onOpen([this, label] { BOOST_LOG_TRIVIAL(info) << "WebRTC: data channel '" << label << "' open"; }); + dc->onClosed([this, label] { BOOST_LOG_TRIVIAL(info) << "WebRTC: data channel '" << label << "' closed"; }); + dc->onError([label](std::string e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: data channel '" << label << "' error: " << e; + }); + dc->onMessage( + [this](rtc::binary data) { + if (m_alive.load()) + enqueue_chunk(std::vector(data.begin(), data.end())); + }, + [](rtc::string) {}); +} + +void WebRtcMediaController::on_ready(std::vector servers) +{ + init_rtc_logger_once(); + rtc::Configuration configuration; + for (const CameraIceServer& server : servers) { + try { + rtc::IceServer ice_server(server.urls); + ice_server.username = server.username; + ice_server.password = server.credential; + configuration.iceServers.emplace_back(std::move(ice_server)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: invalid ICE server: " << e.what(); + } + } + configuration.iceServers.emplace_back("stun:stun.cloudflare.com:3478"); + configuration.iceServers.emplace_back("stun:stun.l.google.com:19302"); + + auto peer_connection = std::make_shared(std::move(configuration)); + BOOST_LOG_TRIVIAL(info) << "WebRTC: creating peer connection with " << configuration.iceServers.size() + << " ice servers"; + peer_connection->onLocalDescription([this](rtc::Description description) { + if (!m_alive.load()) + return; + const std::string sdp(description); + BOOST_LOG_TRIVIAL(info) << "WebRTC: local description ready (" << description.typeString() + << "), OFFER SDP:\n" << sdp; + std::lock_guard lock(m_mutex); + if (m_signaling) + m_signaling->send_offer(sdp); + }); + peer_connection->onLocalCandidate([this](rtc::Candidate candidate) { + if (!m_alive.load()) + return; + std::lock_guard lock(m_mutex); + if (m_signaling) + m_signaling->send_ice(std::string(candidate), candidate.mid()); + }); + peer_connection->onStateChange([this](rtc::PeerConnection::State state) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: peer state -> " << static_cast(state); + if (!m_alive.load()) + return; + if (state == rtc::PeerConnection::State::Failed || state == rtc::PeerConnection::State::Disconnected) + report({Status::Failed, Status::ICE_FAILED}); + }); + peer_connection->onGatheringStateChange([](rtc::PeerConnection::GatheringState state) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: gathering state -> " << static_cast(state); + }); + + // Accept a DataChannel opened by the remote peer (OrcaSonar may create the + // "camera" channel from its side rather than answering the one we offer). + peer_connection->onDataChannel([this](std::shared_ptr dc) { + BOOST_LOG_TRIVIAL(info) << "WebRTC: remote opened data channel '" << dc->label() << "'"; + bind_data_channel(dc); + std::lock_guard lock(m_mutex); + m_data_channel = std::move(dc); + }); + + rtc::DataChannelInit init; + init.reliability.unordered = true; + init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + auto data_channel = peer_connection->createDataChannel("camera", init); + if (data_channel) + bind_data_channel(data_channel); + + // NOTE: the H.264 RTP RecvOnly track is intentionally NOT added to the offer + // yet. OrcaSonar answers application-only (no m=video), which makes + // libdatachannel renegotiate and send a second offer that OrcaSonar rejects + // with "webrtc.unavailable: error". Re-add the video m-line (enqueue_nal / + // the decode_loop NAL branch are already in place) once OrcaSonar answers it. + + std::shared_ptr peer_for_description; + { + std::lock_guard lock(m_mutex); + if (!m_alive.load()) + return; + m_peer_connection = std::move(peer_connection); + peer_for_description = m_peer_connection; + m_data_channel = std::move(data_channel); + } + if (peer_for_description) + peer_for_description->setLocalDescription(); +} + +void WebRtcMediaController::on_answer(std::string sdp) +{ + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + peer_connection = m_peer_connection; + } + BOOST_LOG_TRIVIAL(info) << "WebRTC: applying remote answer (" << sdp.size() << " bytes), ANSWER SDP:\n" << sdp; + if (!peer_connection) + return; + try { + peer_connection->setRemoteDescription(rtc::Description(sdp, "answer")); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: setRemoteDescription failed: " << e.what(); + report({Status::Failed, Status::ICE_FAILED}); + return; + } + + // Flush any remote candidates that arrived before the answer. + std::vector> pending; + { + std::lock_guard lock(m_mutex); + m_remote_description_set = true; + pending.swap(m_pending_candidates); + } + BOOST_LOG_TRIVIAL(info) << "WebRTC: remote description set, flushing " << pending.size() + << " buffered candidate(s)"; + for (const auto& c : pending) { + try { + peer_connection->addRemoteCandidate(rtc::Candidate(c.first, c.second)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: addRemoteCandidate (buffered) failed: " << e.what(); + } + } +} + +void WebRtcMediaController::on_ice(std::string candidate, std::string mid) +{ + std::shared_ptr peer_connection; + { + std::lock_guard lock(m_mutex); + if (!m_remote_description_set) { + m_pending_candidates.emplace_back(std::move(candidate), std::move(mid)); + return; + } + peer_connection = m_peer_connection; + } + if (!peer_connection) + return; + try { + peer_connection->addRemoteCandidate(rtc::Candidate(candidate, mid)); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "WebRTC: addRemoteCandidate failed: " << e.what(); + } +} + +void WebRtcMediaController::on_unavailable(CameraUnavailableReason reason, std::string detail) +{ + BOOST_LOG_TRIVIAL(warning) << "WebRTC camera unavailable: " << detail; + Status::Code code = Status::UNAVAILABLE_ERROR; + if (reason == CameraUnavailableReason::Busy) + code = Status::UNAVAILABLE_BUSY; + else if (reason == CameraUnavailableReason::Disabled) + code = Status::UNAVAILABLE_DISABLED; + else if (reason == CameraUnavailableReason::Closed) + code = Status::SIGNALING_CLOSED; + report({Status::Failed, code}); +} + +void WebRtcMediaController::enqueue_chunk(std::vector chunk) +{ + std::lock_guard lock(m_mutex); + if (m_chunk_queue.size() >= 8) + m_chunk_queue.pop_front(); + m_chunk_queue.emplace_back(std::move(chunk)); + m_cond.notify_one(); +} + +void WebRtcMediaController::enqueue_nal(std::vector nal) +{ + std::lock_guard lock(m_mutex); + if (m_nal_queue.size() >= 4) + m_nal_queue.pop_front(); + m_nal_queue.emplace_back(std::move(nal)); + m_cond.notify_one(); +} + +void WebRtcMediaController::deliver_jpeg(std::vector jpeg) +{ + const auto now = std::chrono::steady_clock::now(); + { + std::lock_guard lock(m_mutex); + if (m_last_frame_time != std::chrono::steady_clock::time_point{} && + now - m_last_frame_time < std::chrono::milliseconds(33)) + return; + m_last_frame_time = now; + } + wxMemoryInputStream stream(jpeg.data(), jpeg.size()); + wxImage image; + if (!image.LoadFile(stream, wxBITMAP_TYPE_JPEG)) { + report({Status::Failed, Status::DECODE_ERROR}); + return; + } + bool first_frame = false; + { + std::lock_guard lock(m_mutex); + m_video_size = image.GetSize(); + first_frame = !m_has_frame; + m_has_frame = true; + } + if (m_frame_sink) + m_frame_sink(image, image.GetSize()); + if (first_frame) + report({Status::Playing}); +} + +void WebRtcMediaController::decode_loop() +{ + WebRtcFrameAssembler assembler; + assembler.on_frame = [this](std::vector jpeg) { deliver_jpeg(std::move(jpeg)); }; + AVCodecParameters parameters{}; + parameters.codec_type = AVMEDIA_TYPE_VIDEO; + parameters.codec_id = AV_CODEC_ID_H264; + AVVideoDecoder decoder; + bool decoder_open = false; + + int stall_polls = 0; + std::unique_lock lock(m_mutex); + while (m_alive.load()) { + const bool woke = m_cond.wait_for(lock, std::chrono::seconds(2), [this] { + return !m_alive.load() || !m_chunk_queue.empty() || !m_nal_queue.empty(); + }); + if (!m_alive.load()) + break; + if (!woke && !m_has_frame) { + const int pc_state = m_peer_connection ? static_cast(m_peer_connection->state()) : -1; + std::string dc = "none"; + if (m_data_channel) + dc = "label='" + m_data_channel->label() + "' open=" + + (m_data_channel->isOpen() ? "1" : "0"); + lock.unlock(); + BOOST_LOG_TRIVIAL(info) << "WebRTC: waiting for frames; peer_state=" << pc_state + << " data_channel=" << dc; + if (++stall_polls >= 8) { // ~16s connected with no frame -> give up so the UI can retry + report({Status::Failed, Status::TIMEOUT}); + lock.lock(); + break; + } + lock.lock(); + continue; + } + stall_polls = 0; + if (!m_chunk_queue.empty()) { + auto chunk = std::move(m_chunk_queue.front()); + m_chunk_queue.pop_front(); + lock.unlock(); + assembler.feed(chunk.data(), chunk.size()); + lock.lock(); + } else if (!m_nal_queue.empty()) { + auto nal = std::move(m_nal_queue.front()); + m_nal_queue.pop_front(); + lock.unlock(); + if (!decoder_open) + decoder_open = decoder.open(parameters) == 0; + if (decoder_open) { + AVPacket* packet = av_packet_alloc(); + if (packet && av_new_packet(packet, static_cast(nal.size())) == 0) { + std::memcpy(packet->data, nal.data(), nal.size()); + if (decoder.decode(*packet) == 0) { + wxImage image; + if (decoder.toWxImage(image, wxDefaultSize)) { + { + std::lock_guard frame_lock(m_mutex); + m_video_size = image.GetSize(); + } + if (m_frame_sink) + m_frame_sink(image, image.GetSize()); + report({Status::Playing}); + } + } + } + av_packet_free(&packet); + } + lock.lock(); + } + } +} + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.hpp b/src/slic3r/GUI/WebRtcMediaController.hpp new file mode 100644 index 0000000000..1cbd05fcd7 --- /dev/null +++ b/src/slic3r/GUI/WebRtcMediaController.hpp @@ -0,0 +1,96 @@ +#pragma once + +#include "IMediaController.hpp" +#include "WebRtcFrameAssembler.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rtc { +class DataChannel; +class PeerConnection; +class Track; +} + +namespace Slic3r { namespace GUI { + +class WebRtcMediaController : public IMediaController { +public: + struct Status { + enum Kind { Connecting, Playing, Stopped, Failed } kind = Stopped; + enum Code { + ICE_FAILED, + SIGNALING_CLOSED, + UNAVAILABLE_BUSY, + UNAVAILABLE_ERROR, + UNAVAILABLE_DISABLED, + DECODE_ERROR, + TIMEOUT, + } code = ICE_FAILED; + // Identifies the StartSession attempt this status belongs to, so the + // consumer can drop CallAfter-queued events from a superseded attempt. + std::uint64_t epoch = 0; + }; + + WebRtcMediaController(std::function frame_sink, + std::function on_status); + ~WebRtcMediaController() override; + + void StartSession(std::unique_ptr channel) override; + void StopSession() override; + std::uint64_t epoch() const { return m_epoch.load(); } + bool is_active() const { return m_alive.load(); } + + void Load(wxURI) override {} + void Play() override {} + void Stop() override { StopSession(); } + wxMediaState GetState() override; + wxSize GetVideoSize() const override; + +private: + void teardown(bool notify); + void report(Status status); + void bind_data_channel(const std::shared_ptr& dc); + void on_ready(std::vector servers); + void on_answer(std::string sdp); + void on_ice(std::string candidate, std::string mid); + void on_unavailable(CameraUnavailableReason reason, std::string detail); + void decode_loop(); + void enqueue_chunk(std::vector chunk); + void enqueue_nal(std::vector nal); + void deliver_jpeg(std::vector jpeg); + + mutable std::mutex m_mutex; + std::condition_variable m_cond; + std::deque> m_chunk_queue; + std::deque> m_nal_queue; + // Remote candidates can arrive before the answer; libdatachannel rejects + // addRemoteCandidate until a remote description is set, so buffer them. + std::vector> m_pending_candidates; + bool m_remote_description_set = false; + std::unique_ptr m_signaling; + std::shared_ptr m_peer_connection; + std::shared_ptr m_data_channel; + std::shared_ptr m_video_track; + std::thread m_decode_thread; + std::atomic m_alive{false}; + std::atomic m_epoch{0}; + wxMediaState m_state = static_cast(3); + wxSize m_video_size = wxDefaultSize; + std::function m_frame_sink; + std::function m_on_status; + bool m_has_frame = false; + std::chrono::steady_clock::time_point m_last_frame_time{}; +}; + +}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/wxMediaCtrl3.cpp b/src/slic3r/GUI/wxMediaCtrl3.cpp index d0d53072d6..2d6412afe4 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.cpp +++ b/src/slic3r/GUI/wxMediaCtrl3.cpp @@ -46,9 +46,13 @@ wxMediaCtrl3::~wxMediaCtrl3() m_thread.join(); } +static void adjust_frame_size(wxSize& frame, wxSize const& video, wxSize const& window); + void wxMediaCtrl3::Load(wxURI url) { std::unique_lock lk(m_mutex); + if (m_external) + return; m_video_size = wxDefaultSize; m_error = 0; m_url.reset(new wxURI(url)); @@ -58,6 +62,8 @@ void wxMediaCtrl3::Load(wxURI url) void wxMediaCtrl3::Play() { std::unique_lock lk(m_mutex); + if (m_external) + return; if (m_state != wxMEDIASTATE_PLAYING) { m_state = wxMEDIASTATE_PLAYING; wxMediaEvent event(wxEVT_MEDIA_STATECHANGED); @@ -77,6 +83,62 @@ void wxMediaCtrl3::Stop() Refresh(); } +void wxMediaCtrl3::SetExternalFrame(const wxImage& frame, wxSize videoSize) +{ + if (!frame.IsOk()) + return; + { + std::unique_lock lk(m_mutex); + if (!m_external) + return; + m_frame = frame; + m_video_size = videoSize.IsFullySpecified() ? videoSize : frame.GetSize(); + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + } + CallAfter([this] { Refresh(); }); +} + +#ifdef _WIN32 +void wxMediaCtrl3::SetExternalFrame(const wxBitmap& frame, wxSize videoSize) +{ + if (!frame.IsOk()) + return; + { + std::unique_lock lk(m_mutex); + if (!m_external) + return; + m_frame = frame; + m_video_size = videoSize.IsFullySpecified() ? videoSize : frame.GetSize(); + adjust_frame_size(m_frame_size, m_video_size, GetSize()); + } + CallAfter([this] { Refresh(); }); +} +#endif + +void wxMediaCtrl3::BeginExternalStream() +{ + std::unique_lock lk(m_mutex); + m_external = true; + m_url.reset(); + m_active_url.reset(); + m_video_size = wxDefaultSize; + m_frame = wxImage(m_idle_image); + m_cond.notify_all(); + Refresh(); +} + +void wxMediaCtrl3::EndExternalStream() +{ + std::unique_lock lk(m_mutex); + m_external = false; + m_url.reset(); + m_active_url.reset(); + m_video_size = wxDefaultSize; + m_frame = wxImage(m_idle_image); + m_cond.notify_all(); + Refresh(); +} + void wxMediaCtrl3::SetIdleImage(wxString const &image) { if (m_idle_image == image) diff --git a/src/slic3r/GUI/wxMediaCtrl3.h b/src/slic3r/GUI/wxMediaCtrl3.h index bcc94a17ef..dee49ddbe8 100644 --- a/src/slic3r/GUI/wxMediaCtrl3.h +++ b/src/slic3r/GUI/wxMediaCtrl3.h @@ -18,9 +18,7 @@ void wxMediaCtrl_OnSize(wxWindow * ctrl, wxSize const & videoSize, int width, in #define BAMBU_DYNAMIC #include #include -#ifndef _WIN32 #include -#endif #include "Printer/BambuTunnel.h" class AVVideoDecoder; @@ -38,6 +36,16 @@ public: void Stop(); + // Render frames supplied by a controller which owns its own transport. + // The frame is copied while m_mutex is held; callers may release it after + // this method returns. + void SetExternalFrame(const wxImage& frame, wxSize videoSize); +#ifdef _WIN32 + void SetExternalFrame(const wxBitmap& frame, wxSize videoSize); +#endif + void BeginExternalStream(); + void EndExternalStream(); + void SetIdleImage(wxString const & image); wxMediaState GetState(); @@ -77,6 +85,7 @@ private: std::shared_ptr m_url; std::shared_ptr m_active_url; + bool m_external = false; std::uint64_t m_last_PTS{0}; std::chrono::system_clock::time_point m_last_PTS_expected; std::chrono::system_clock::time_point m_last_PTS_practical; diff --git a/src/slic3r/Utils/ICameraSignalingChannel.hpp b/src/slic3r/Utils/ICameraSignalingChannel.hpp new file mode 100644 index 0000000000..70df298843 --- /dev/null +++ b/src/slic3r/Utils/ICameraSignalingChannel.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include + +namespace Slic3r { + +struct CameraIceServer { + std::string urls; + std::string username; + std::string credential; +}; + +enum class CameraUnavailableReason { + Busy, + Error, + Disabled, + Closed, +}; + +class ICameraSignalingChannel { +public: + virtual ~ICameraSignalingChannel() = default; + + virtual void open() = 0; + virtual void close() = 0; + virtual void send_offer(std::string sdp) = 0; + virtual void send_ice(std::string candidate, std::string mid) = 0; + + // These callbacks are invoked by the channel's worker thread. Consumers + // must marshal UI work to the GUI thread themselves. + std::function)> on_ready; + std::function on_answer; + std::function on_ice; + std::function on_unavailable; +}; + +} // namespace Slic3r diff --git a/src/slic3r/Utils/IPrinterAgent.hpp b/src/slic3r/Utils/IPrinterAgent.hpp index c076b57be5..240b00395b 100644 --- a/src/slic3r/Utils/IPrinterAgent.hpp +++ b/src/slic3r/Utils/IPrinterAgent.hpp @@ -14,6 +14,7 @@ #include #include #include +#include "ICameraSignalingChannel.hpp" #if 1 @@ -382,6 +383,15 @@ public: * Only meaningful when get_camera_stream_mode() returns an HTTP or RTSP mode. */ virtual std::string get_camera_url() const { return {}; } + + // Optional native camera signaling. Plugin agents retain the default + // nullptr until a plugin-facing WebRTC contract is defined. + virtual std::unique_ptr + create_camera_signaling_channel(const std::string& dev_id) + { + (void) dev_id; + return nullptr; + } }; } // namespace Slic3r diff --git a/src/slic3r/Utils/NetworkAgent.cpp b/src/slic3r/Utils/NetworkAgent.cpp index 01ed03f84f..4857151c58 100644 --- a/src/slic3r/Utils/NetworkAgent.cpp +++ b/src/slic3r/Utils/NetworkAgent.cpp @@ -1040,6 +1040,14 @@ std::string NetworkAgent::get_local_camera_stream_url() const return {}; } +std::unique_ptr +NetworkAgent::create_camera_signaling_channel(const std::string& dev_id) +{ + if (m_printer_agent) + return m_printer_agent->create_camera_signaling_channel(dev_id); + return nullptr; +} + int NetworkAgent::request_bind_ticket(std::string* ticket) { if (m_printer_agent) diff --git a/src/slic3r/Utils/NetworkAgent.hpp b/src/slic3r/Utils/NetworkAgent.hpp index 7eb220b341..a051bdd231 100644 --- a/src/slic3r/Utils/NetworkAgent.hpp +++ b/src/slic3r/Utils/NetworkAgent.hpp @@ -183,6 +183,7 @@ public: bool fetch_filament_info(std::string dev_id, FilamentSyncMode sync_mode = FilamentSyncMode::pull); CameraStreamMode get_camera_stream_mode() const; std::string get_local_camera_stream_url() const; + std::unique_ptr create_camera_signaling_channel(const std::string& dev_id); int request_bind_ticket(std::string* ticket); int get_hms_snapshot(std::string dev_id, std::string file_name, std::function callback); diff --git a/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp b/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp new file mode 100644 index 0000000000..9c3a029db3 --- /dev/null +++ b/src/slic3r/Utils/OrcaCloudSignalingChannel.cpp @@ -0,0 +1,320 @@ +#include "OrcaCloudSignalingChannel.hpp" + +#include "Http.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace Slic3r { + +OrcaCloudSignalingChannel::OrcaCloudSignalingChannel(std::shared_ptr cloud, std::string dev_id) + : m_cloud(std::move(cloud)) + , m_dev_id(std::move(dev_id)) +{ +} + +OrcaCloudSignalingChannel::~OrcaCloudSignalingChannel() +{ + close(); +} + +void OrcaCloudSignalingChannel::open() +{ + bool expected = false; + if (!m_open.compare_exchange_strong(expected, true)) + return; + m_stop.store(false); + m_thread = std::thread([this] { run(); }); +} + +void OrcaCloudSignalingChannel::close() +{ + m_stop.store(true); + std::shared_ptr conn; + { + std::lock_guard lock(m_mutex); + conn = m_conn; + } + if (conn) { + // Established session: close the socket on the io_context's own thread so + // the pending async_read completes and io_context.run() unwinds. + boost::asio::post(conn->io_context, [conn] { + boost::system::error_code ec; + boost::beast::get_lowest_layer(conn->websocket).cancel(ec); + boost::beast::get_lowest_layer(conn->websocket).close(ec); + }); + // Pre-run() phase (still in the synchronous connect/handshake): best-effort + // direct interruption. + boost::system::error_code ec; + boost::beast::get_lowest_layer(conn->websocket).cancel(ec); + boost::beast::get_lowest_layer(conn->websocket).close(ec); + } + if (m_thread.joinable()) + m_thread.join(); + m_open.store(false); +} + +void OrcaCloudSignalingChannel::send_offer(std::string sdp) +{ + send_json(nlohmann::json{{"type", "webrtc.offer"}, {"sdp", std::move(sdp)}}.dump()); +} + +void OrcaCloudSignalingChannel::send_ice(std::string candidate, std::string mid) +{ + send_json(nlohmann::json{{"type", "webrtc.ice"}, + {"candidate", std::move(candidate)}, + {"sdpMid", std::move(mid)}} + .dump()); +} + +std::string OrcaCloudSignalingChannel::encode_path_component(const std::string& value) +{ + std::ostringstream encoded; + encoded << std::uppercase << std::hex; + for (unsigned char c : value) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') + encoded << c; + else + encoded << '%' << std::setw(2) << std::setfill('0') << static_cast(c); + } + return encoded.str(); +} + +std::string OrcaCloudSignalingChannel::host_without_scheme(std::string value) +{ + const auto scheme = value.find("://"); + if (scheme != std::string::npos) + value.erase(0, scheme + 3); + const auto slash = value.find('/'); + if (slash != std::string::npos) + value.erase(slash); + return value; +} + +void OrcaCloudSignalingChannel::unavailable(CameraUnavailableReason reason, std::string detail) +{ + if (on_unavailable) + on_unavailable(reason, std::move(detail)); +} + +void OrcaCloudSignalingChannel::run() +{ + try { + if (!m_cloud || !m_cloud->ensure_token_fresh("camera")) { + unavailable(CameraUnavailableReason::Error, "Unable to refresh OrcaCloud credentials"); + m_open.store(false); + return; + } + const std::string token = m_cloud->get_access_token(); + const std::string host = host_without_scheme(m_cloud->get_cloud_service_host()); + if (token.empty() || host.empty()) { + unavailable(CameraUnavailableReason::Error, "OrcaCloud session is unavailable"); + m_open.store(false); + return; + } + + const std::string live_token_url = + "https://" + host + "/api/v1/printers/" + encode_path_component(m_dev_id) + "/live-token"; + BOOST_LOG_TRIVIAL(info) << "signaling: POST " << live_token_url << " (dev_id=" << m_dev_id << ")"; + + nlohmann::json token_response; + std::string token_body; + std::string token_error; + unsigned int http_code = 0; + auto request = Http::post(live_token_url); + request.set_post_body(std::string("{}")) + .header("Authorization", "Bearer " + token) + .header("Content-Type", "application/json") + .tls_verify(true) + .timeout_max(30) + .on_complete([&token_body, &http_code](std::string body, unsigned status) { + http_code = status; + token_body = std::move(body); + }) + .on_error([&token_body, &token_error, &http_code](std::string body, std::string error, unsigned status) { + http_code = status; + token_body = std::move(body); + token_error = std::move(error); + }) + .perform_sync(); + BOOST_LOG_TRIVIAL(info) << "signaling: live-token HTTP " << http_code + << (token_error.empty() ? "" : " error=" + token_error) + << " body=" << token_body.substr(0, 512); + try { + token_response = nlohmann::json::parse(token_body); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: live-token body is not JSON: " << e.what(); + } + if (http_code < 200 || http_code >= 300 || !token_response.contains("token")) { + unavailable(CameraUnavailableReason::Error, "Unable to mint camera live token"); + m_open.store(false); + return; + } + + std::vector ice_servers; + if (token_response.contains("ice_servers") && token_response["ice_servers"].is_array()) { + for (const auto& entry : token_response["ice_servers"]) { + if (entry.is_string()) { + ice_servers.push_back({entry.get(), {}, {}}); + } else if (entry.is_object()) { + // RTCIceServer.urls is "string | string[]" (Cloudflare + // Realtime returns an array). Emit one CameraIceServer per + // URL, sharing the credentials. + const std::string username = entry.value("username", std::string{}); + const std::string credential = entry.value("credential", std::string{}); + const auto add_url = [&](const nlohmann::json& url) { + if (url.is_string() && !url.get().empty()) + ice_servers.push_back({url.get(), username, credential}); + }; + const auto urls = entry.find("urls"); + if (urls != entry.end()) { + if (urls->is_array()) { + for (const auto& url : *urls) + add_url(url); + } else { + add_url(*urls); + } + } + } + } + } + + auto conn = std::make_shared(); + conn->ssl_context.set_default_verify_paths(); + { + std::lock_guard lock(m_mutex); + m_conn = conn; + } + auto& websocket = conn->websocket; + boost::asio::ip::tcp::resolver resolver(conn->io_context); + const auto endpoints = resolver.resolve(host, "443"); + boost::asio::connect(boost::beast::get_lowest_layer(websocket), endpoints); + if (!SSL_set_tlsext_host_name(websocket.next_layer().native_handle(), host.c_str())) + throw std::runtime_error("Unable to configure TLS server name"); + websocket.next_layer().set_verify_mode(boost::asio::ssl::verify_peer); + websocket.next_layer().handshake(boost::asio::ssl::stream_base::client); + const std::string ws_target = "/api/v1/printers/" + encode_path_component(m_dev_id) + + "/camera/live?token=" + + encode_path_component(token_response["token"].get()); + websocket.handshake(host, ws_target); + BOOST_LOG_TRIVIAL(info) << "signaling: websocket handshake ok (" << ice_servers.size() + << " ice servers)"; + + if (on_ready) + on_ready(std::move(ice_servers)); + send_json(nlohmann::json{{"type", "camera.mode"}, {"mode", "webrtc"}}.dump()); + // Async read loop, driven by the connection's own io_context. run() + // returns once close() has shut the socket down, giving a bounded, + // deadlock-free teardown from any thread. + do_read(conn); + conn->io_context.run(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: run() exception: " << e.what(); + if (!m_stop.load()) + unavailable(CameraUnavailableReason::Closed, e.what()); + } + { + std::lock_guard lock(m_mutex); + m_conn.reset(); + } + m_open.store(false); +} + +void OrcaCloudSignalingChannel::do_read(std::shared_ptr conn) +{ + auto buffer = std::make_shared(); + conn->websocket.async_read( + *buffer, [this, conn, buffer](boost::system::error_code ec, std::size_t) { + if (ec) { + if (!m_stop.load()) + unavailable(CameraUnavailableReason::Closed, ec.message()); + return; // do not re-arm; io_context.run() unwinds + } + const std::string raw = boost::beast::buffers_to_string(buffer->data()); + // A malformed or unexpectedly-shaped message must not tear down the + // session: parse/dispatch is guarded. + try { + dispatch_message(nlohmann::json::parse(raw), raw); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(warning) << "signaling: ignoring malformed message: " << e.what() + << " raw=" << raw.substr(0, 256); + } + if (!m_stop.load()) + do_read(conn); + }); +} + +// Returns the string at key, or "" if absent or not a string (JSON null included). +static std::string json_string(const nlohmann::json& object, const char* key) +{ + const auto it = object.find(key); + return (it != object.end() && it->is_string()) ? it->get() : std::string{}; +} + +void OrcaCloudSignalingChannel::dispatch_message(const nlohmann::json& message, const std::string& raw) +{ + const std::string type = json_string(message, "type"); + BOOST_LOG_TRIVIAL(info) << "signaling: recv type=" << type << " raw=" << raw.substr(0, 256); + if (type == "webrtc.answer") { + const std::string sdp = json_string(message, "sdp"); + if (on_answer && !sdp.empty()) + on_answer(sdp); + } else if (type == "webrtc.ice" && message.contains("candidate")) { + // The peer may send "candidate" as a flat string or as a nested + // RTCIceCandidateInit object { candidate, sdpMid, sdpMLineIndex }. + const nlohmann::json& candidate = message["candidate"]; + std::string sdp_candidate; + std::string mid = json_string(message, "sdpMid"); + if (candidate.is_string()) { + sdp_candidate = candidate.get(); + } else if (candidate.is_object()) { + sdp_candidate = json_string(candidate, "candidate"); + std::string nested_mid = json_string(candidate, "sdpMid"); + if (!nested_mid.empty()) + mid = std::move(nested_mid); + } + if (on_ice && !sdp_candidate.empty()) + on_ice(sdp_candidate, mid); + } else if (type == "webrtc.unavailable") { + const std::string reason = json_string(message, "reason"); + unavailable(reason == "busy" ? CameraUnavailableReason::Busy + : reason == "disabled" ? CameraUnavailableReason::Disabled + : CameraUnavailableReason::Error, + reason.empty() ? "error" : reason); + } +} + +void OrcaCloudSignalingChannel::send_json(const std::string& message) +{ + std::shared_ptr conn; + { + std::lock_guard lock(m_mutex); + conn = m_conn; + } + if (!conn || m_stop.load()) + return; + // Serialize the write onto the io_context thread (same thread that runs + // async_read), so reads and writes never touch the stream concurrently. + auto payload = std::make_shared(message); + boost::asio::post(conn->io_context, [this, conn, payload] { + if (m_stop.load()) + return; + boost::system::error_code ec; + conn->websocket.write(boost::asio::buffer(*payload), ec); + if (ec && !m_stop.load()) + unavailable(CameraUnavailableReason::Closed, ec.message()); + }); +} + +} // namespace Slic3r diff --git a/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp b/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp new file mode 100644 index 0000000000..b87b7400e2 --- /dev/null +++ b/src/slic3r/Utils/OrcaCloudSignalingChannel.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include "ICameraSignalingChannel.hpp" +#include "ICloudServiceAgent.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace Slic3r { + +class OrcaCloudSignalingChannel : public ICameraSignalingChannel { +public: + OrcaCloudSignalingChannel(std::shared_ptr cloud, std::string dev_id); + ~OrcaCloudSignalingChannel() override; + + void open() override; + void close() override; + void send_offer(std::string sdp) override; + void send_ice(std::string candidate, std::string mid) override; + +private: + using WebSocket = boost::beast::websocket::stream< + boost::beast::ssl_stream>; + + // The io_context and ssl_context must outlive the websocket stream that + // references them. Bundling them here with the stream declared last makes + // the destruction order correct (stream first, then contexts), and lets a + // single shared_ptr own the whole set. + struct Connection { + boost::asio::io_context io_context; + boost::asio::ssl::context ssl_context{boost::asio::ssl::context::tls_client}; + WebSocket websocket{io_context, ssl_context}; + }; + + void run(); + void do_read(std::shared_ptr conn); + void dispatch_message(const nlohmann::json& message, const std::string& raw); + void send_json(const std::string& message); + void unavailable(CameraUnavailableReason reason, std::string detail); + static std::string encode_path_component(const std::string& value); + static std::string host_without_scheme(std::string value); + + std::shared_ptr m_cloud; + std::string m_dev_id; + std::atomic m_stop{false}; + std::atomic m_open{false}; + std::thread m_thread; + mutable std::mutex m_mutex; + std::shared_ptr m_conn; +}; + +} // namespace Slic3r diff --git a/src/slic3r/Utils/OrcaPrinterAgent.cpp b/src/slic3r/Utils/OrcaPrinterAgent.cpp index 93cadd9abd..520686db3a 100644 --- a/src/slic3r/Utils/OrcaPrinterAgent.cpp +++ b/src/slic3r/Utils/OrcaPrinterAgent.cpp @@ -1,4 +1,5 @@ #include "OrcaPrinterAgent.hpp" +#include "OrcaCloudSignalingChannel.hpp" #include "NetworkAgentFactory.hpp" #include "OrcaCloudServiceAgent.hpp" #include @@ -47,6 +48,31 @@ void OrcaPrinterAgent::set_cloud_agent(std::shared_ptr cloud BOOST_LOG_TRIVIAL(info) << "OrcaPrinterAgent::set_cloud_agent: status callback result=" << callback_result; } +CameraStreamMode OrcaPrinterAgent::get_camera_stream_mode() const +{ + std::lock_guard lock(state_mutex); + if (m_lan_connected && !m_lan_rtsp_url.empty()) + return CameraStreamMode::rtsp; + if (m_cloud_agent && m_cloud_agent->is_user_login()) + return CameraStreamMode::webrtc; + return CameraStreamMode::none; +} + +std::string OrcaPrinterAgent::get_camera_url() const +{ + std::lock_guard lock(state_mutex); + return m_lan_connected ? m_lan_rtsp_url : std::string{}; +} + +std::unique_ptr +OrcaPrinterAgent::create_camera_signaling_channel(const std::string& dev_id) +{ + std::lock_guard lock(state_mutex); + if (!m_cloud_agent) + return nullptr; + return std::make_unique(m_cloud_agent, dev_id); +} + // ============================================================================ // Communication - All Stubs // ============================================================================ @@ -86,11 +112,20 @@ int OrcaPrinterAgent::send_message(std::string dev_id, std::string json_str, int int OrcaPrinterAgent::connect_printer(std::string dev_id, std::string dev_ip, std::string username, std::string password, bool use_ssl) { + std::lock_guard lock(state_mutex); + m_lan_connected = !dev_ip.empty(); + // OrcaPrinterAgent currently has no LAN status-push decoder. Preserve the + // standard OrcaSonar endpoint convention until the reported RTSP field is + // available, while keeping the URL behind the connection-aware mode API. + m_lan_rtsp_url = m_lan_connected ? "rtsp://" + dev_ip + ":8554/stream" : std::string{}; return BAMBU_NETWORK_SUCCESS; } int OrcaPrinterAgent::disconnect_printer() { + std::lock_guard lock(state_mutex); + m_lan_connected = false; + m_lan_rtsp_url.clear(); return BAMBU_NETWORK_SUCCESS; } diff --git a/src/slic3r/Utils/OrcaPrinterAgent.hpp b/src/slic3r/Utils/OrcaPrinterAgent.hpp index 9cf2b29638..00d7e372f6 100644 --- a/src/slic3r/Utils/OrcaPrinterAgent.hpp +++ b/src/slic3r/Utils/OrcaPrinterAgent.hpp @@ -27,6 +27,10 @@ public: // ======================================================================== void set_cloud_agent(std::shared_ptr cloud) override; + CameraStreamMode get_camera_stream_mode() const override; + std::string get_camera_url() const override; + std::unique_ptr + create_camera_signaling_channel(const std::string& dev_id) override; // Communication int send_message(std::string dev_id, std::string json_str, int qos, int flag) override; @@ -85,6 +89,9 @@ private: std::shared_ptr m_cloud_agent; OrcaCloudServiceAgent* m_orca_cloud = nullptr; // == m_cloud_agent.get() when the Orca provider is active + bool m_lan_connected = false; + std::string m_lan_rtsp_url; + // MOCK: OrcaCloud does not yet relay the printer's info.get_version reply, so // synthesize it and feed it through on_message_fn (same sink as real report // messages). Delete once the backend answers info.get_version. diff --git a/tests/slic3rutils/CMakeLists.txt b/tests/slic3rutils/CMakeLists.txt index edff97ff42..fb5b0bda1e 100644 --- a/tests/slic3rutils/CMakeLists.txt +++ b/tests/slic3rutils/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(${_TEST_NAME}_tests test_plugin_capabilities_in_use.cpp test_plugin_status.cpp test_printer_agent.cpp + test_webrtc_frame_assembler.cpp test_qidi_printer_agent.cpp test_plugin_install.cpp test_plugin_lifecycle.cpp diff --git a/tests/slic3rutils/test_webrtc_frame_assembler.cpp b/tests/slic3rutils/test_webrtc_frame_assembler.cpp new file mode 100644 index 0000000000..377d2931b7 --- /dev/null +++ b/tests/slic3rutils/test_webrtc_frame_assembler.cpp @@ -0,0 +1,67 @@ +#include + +#include + +#include +#include +#include +#include + +using Slic3r::GUI::WebRtcFrameAssembler; + +static std::vector make_chunk(std::uint32_t frame_id, + std::uint16_t index, + std::uint16_t count, + std::initializer_list payload) +{ + std::vector result(WebRtcFrameAssembler::HeaderSize + payload.size()); + result[0] = std::byte{1}; + result[1] = std::byte{0}; + result[2] = std::byte{static_cast(index >> 8)}; + result[3] = std::byte{static_cast(index)}; + result[4] = std::byte{static_cast(count >> 8)}; + result[5] = std::byte{static_cast(count)}; + result[6] = std::byte{static_cast(frame_id >> 24)}; + result[7] = std::byte{static_cast(frame_id >> 16)}; + result[8] = std::byte{static_cast(frame_id >> 8)}; + result[9] = std::byte{static_cast(frame_id)}; + for (std::size_t i = 0; i < payload.size(); ++i) + result[WebRtcFrameAssembler::HeaderSize + i] = std::byte{static_cast(payload.begin()[i])}; + return result; +} + +TEST_CASE("WebRTC frame assembler joins chunks in order", "[webrtc][unit]") +{ + WebRtcFrameAssembler assembler; + std::vector frame; + assembler.on_frame = [&frame](std::vector value) { frame = std::move(value); }; + + const auto second = make_chunk(7, 1, 2, {'C', 'D'}); + const auto first = make_chunk(7, 0, 2, {'A', 'B'}); + assembler.feed(second.data(), second.size()); + assembler.feed(first.data(), first.size()); + + REQUIRE(frame.size() == 4); + CHECK(std::to_integer(frame[0]) == 'A'); + CHECK(std::to_integer(frame[3]) == 'D'); +} + +TEST_CASE("WebRTC frame assembler discards stale and malformed chunks", "[webrtc][unit]") +{ + WebRtcFrameAssembler assembler; + int frames = 0; + assembler.on_frame = [&frames](std::vector) { ++frames; }; + + const auto stale = make_chunk(1, 0, 2, {'A'}); + const auto current = make_chunk(2, 0, 1, {'B'}); + const auto stale_tail = make_chunk(1, 1, 2, {'C'}); + assembler.feed(stale.data(), stale.size()); + assembler.feed(current.data(), current.size()); + assembler.feed(stale_tail.data(), stale_tail.size()); + + CHECK(frames == 1); + + auto malformed = make_chunk(3, 0, 0, {'X'}); + assembler.feed(malformed.data(), malformed.size()); + CHECK(frames == 1); +} From 2a4792e7626b03ef25b353a3bd02614b20f22bfa Mon Sep 17 00:00:00 2001 From: Ian Chua Date: Wed, 2 Sep 2026 15:59:13 +0800 Subject: [PATCH 5/5] feat: remove frame assembler and change config to set protocol --- src/slic3r/CMakeLists.txt | 2 - src/slic3r/GUI/WebRtcFrameAssembler.cpp | 86 ----------------- src/slic3r/GUI/WebRtcFrameAssembler.hpp | 39 -------- src/slic3r/GUI/WebRtcMediaController.cpp | 95 +++++-------------- src/slic3r/GUI/WebRtcMediaController.hpp | 10 +- tests/slic3rutils/CMakeLists.txt | 1 - .../test_webrtc_frame_assembler.cpp | 67 ------------- 7 files changed, 25 insertions(+), 275 deletions(-) delete mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.cpp delete mode 100644 src/slic3r/GUI/WebRtcFrameAssembler.hpp delete mode 100644 tests/slic3rutils/test_webrtc_frame_assembler.cpp diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 322b3b3f0e..c1ce3b22a6 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -344,8 +344,6 @@ set(SLIC3R_GUI_SOURCES GUI/MediaFilePanel.h GUI/MediaPlayCtrl.cpp GUI/MediaPlayCtrl.h - GUI/WebRtcFrameAssembler.cpp - GUI/WebRtcFrameAssembler.hpp GUI/WebRtcMediaController.cpp GUI/WebRtcMediaController.hpp GUI/MeshUtils.cpp diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.cpp b/src/slic3r/GUI/WebRtcFrameAssembler.cpp deleted file mode 100644 index 2771cc0887..0000000000 --- a/src/slic3r/GUI/WebRtcFrameAssembler.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "WebRtcFrameAssembler.hpp" - -#include -#include - -namespace Slic3r { namespace GUI { - -void WebRtcFrameAssembler::discard() -{ - m_active = false; - m_frame_id = 0; - m_chunk_count = 0; - m_received_chunks = 0; - m_total_size = 0; - m_chunks.clear(); - m_received.clear(); -} - -void WebRtcFrameAssembler::reset() -{ - discard(); -} - -void WebRtcFrameAssembler::feed(const std::byte* data, std::size_t len) -{ - if (data == nullptr || len < HeaderSize) - return; - - if (std::to_integer(data[0]) != 1) - return; - - const std::uint16_t chunk_index = static_cast((std::to_integer(data[2]) << 8) | - std::to_integer(data[3])); - const std::uint16_t chunk_count = static_cast((std::to_integer(data[4]) << 8) | - std::to_integer(data[5])); - const std::uint32_t frame_id = (static_cast(std::to_integer(data[6])) << 24) | - (static_cast(std::to_integer(data[7])) << 16) | - (static_cast(std::to_integer(data[8])) << 8) | - static_cast(std::to_integer(data[9])); - const std::size_t payload_size = len - HeaderSize; - - if (chunk_count == 0 || chunk_count > MaxChunkCount || chunk_index >= chunk_count || - payload_size > MaxChunkPayload) - return; - - if (!m_active || frame_id > m_frame_id) { - discard(); - m_active = true; - m_frame_id = frame_id; - m_chunk_count = chunk_count; - m_chunks.resize(chunk_count); - m_received.assign(chunk_count, false); - } else if (frame_id < m_frame_id) { - return; - } else if (chunk_count != m_chunk_count) { - discard(); - return; - } - - Frame& chunk = m_chunks[chunk_index]; - if (m_received[chunk_index]) - return; - - if (m_total_size > MaxFrameSize || payload_size > MaxFrameSize - m_total_size) { - discard(); - return; - } - - chunk.assign(data + HeaderSize, data + len); - m_received[chunk_index] = true; - m_total_size += payload_size; - ++m_received_chunks; - - if (m_received_chunks != m_chunk_count) - return; - - Frame frame; - frame.reserve(m_total_size); - for (const Frame& slice : m_chunks) - frame.insert(frame.end(), slice.begin(), slice.end()); - if (on_frame) - on_frame(std::move(frame)); - discard(); -} - -}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.hpp b/src/slic3r/GUI/WebRtcFrameAssembler.hpp deleted file mode 100644 index c526cad588..0000000000 --- a/src/slic3r/GUI/WebRtcFrameAssembler.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace Slic3r { namespace GUI { - -class WebRtcFrameAssembler { -public: - using Frame = std::vector; - - // The wire protocol uses a 10-byte header followed by one JPEG slice: - // version, flags, chunk index, chunk count, and frame id, all in network - // byte order where applicable. - static constexpr std::size_t HeaderSize = 10; - static constexpr std::size_t MaxChunkPayload = 16000; - static constexpr std::size_t MaxChunkCount = 4096; - static constexpr std::size_t MaxFrameSize = 8 * 1024 * 1024; - - std::function on_frame; - - void feed(const std::byte* data, std::size_t len); - void reset(); - -private: - void discard(); - - bool m_active = false; - std::uint32_t m_frame_id = 0; - std::uint16_t m_chunk_count = 0; - std::size_t m_received_chunks = 0; - std::size_t m_total_size = 0; - std::vector m_chunks; - std::vector m_received; -}; - -}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.cpp b/src/slic3r/GUI/WebRtcMediaController.cpp index 7eafbd4a3a..36a21c5c5d 100644 --- a/src/slic3r/GUI/WebRtcMediaController.cpp +++ b/src/slic3r/GUI/WebRtcMediaController.cpp @@ -1,18 +1,13 @@ #include "WebRtcMediaController.hpp" -#include "AVVideoDecoder.hpp" - #include #include #include -#include #include #include -#include - namespace { void init_rtc_logger_once() { @@ -25,10 +20,6 @@ void init_rtc_logger_once() } } // namespace -extern "C" { -#include -} - namespace Slic3r { namespace GUI { WebRtcMediaController::WebRtcMediaController(std::function frame_sink, @@ -75,8 +66,7 @@ void WebRtcMediaController::StartSession(std::unique_ptr lock(m_mutex); m_signaling = std::move(channel); - m_chunk_queue.clear(); - m_nal_queue.clear(); + m_jpeg_queue.clear(); m_pending_candidates.clear(); m_remote_description_set = false; m_video_size = wxDefaultSize; @@ -130,7 +120,6 @@ void WebRtcMediaController::teardown(bool notify) signaling = std::move(m_signaling); peer_connection = std::move(m_peer_connection); m_data_channel.reset(); - m_video_track.reset(); } if (signaling) signaling->close(); @@ -140,8 +129,7 @@ void WebRtcMediaController::teardown(bool notify) m_decode_thread.join(); { std::lock_guard lock(m_mutex); - m_chunk_queue.clear(); - m_nal_queue.clear(); + m_jpeg_queue.clear(); } if (was_alive && notify) report({Status::Stopped}); @@ -170,7 +158,7 @@ void WebRtcMediaController::bind_data_channel(const std::shared_ptronMessage( [this](rtc::binary data) { if (m_alive.load()) - enqueue_chunk(std::vector(data.begin(), data.end())); + enqueue_jpeg(std::vector(data.begin(), data.end())); }, [](rtc::string) {}); } @@ -179,6 +167,10 @@ void WebRtcMediaController::on_ready(std::vector servers) { init_rtc_logger_once(); rtc::Configuration configuration; + // Allow complete-JPEG DataChannel messages up to 1 MiB. This value is + // advertised in SDP and becomes the upper bound for frames OrcaSonar can + // send to OrcaSlicer. + configuration.maxMessageSize = 1024 * 1024; for (const CameraIceServer& server : servers) { try { rtc::IceServer ice_server(server.urls); @@ -234,16 +226,16 @@ void WebRtcMediaController::on_ready(std::vector servers) rtc::DataChannelInit init; init.reliability.unordered = true; - init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + // init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + // Request one complete JPEG frame per DataChannel message. OrcaSonar + // keeps the legacy chunked protocol for clients that omit this property. + init.protocol = "orca-jpeg"; auto data_channel = peer_connection->createDataChannel("camera", init); if (data_channel) bind_data_channel(data_channel); - // NOTE: the H.264 RTP RecvOnly track is intentionally NOT added to the offer - // yet. OrcaSonar answers application-only (no m=video), which makes - // libdatachannel renegotiate and send a second offer that OrcaSonar rejects - // with "webrtc.unavailable: error". Re-add the video m-line (enqueue_nal / - // the decode_loop NAL branch are already in place) once OrcaSonar answers it. + // Camera media is carried as one complete JPEG per DataChannel message; + // no RTP video track or application-level framing is required. std::shared_ptr peer_for_description; { @@ -327,21 +319,12 @@ void WebRtcMediaController::on_unavailable(CameraUnavailableReason reason, std:: report({Status::Failed, code}); } -void WebRtcMediaController::enqueue_chunk(std::vector chunk) +void WebRtcMediaController::enqueue_jpeg(std::vector jpeg) { std::lock_guard lock(m_mutex); - if (m_chunk_queue.size() >= 8) - m_chunk_queue.pop_front(); - m_chunk_queue.emplace_back(std::move(chunk)); - m_cond.notify_one(); -} - -void WebRtcMediaController::enqueue_nal(std::vector nal) -{ - std::lock_guard lock(m_mutex); - if (m_nal_queue.size() >= 4) - m_nal_queue.pop_front(); - m_nal_queue.emplace_back(std::move(nal)); + if (m_jpeg_queue.size() >= 4) + m_jpeg_queue.pop_front(); + m_jpeg_queue.emplace_back(std::move(jpeg)); m_cond.notify_one(); } @@ -376,19 +359,11 @@ void WebRtcMediaController::deliver_jpeg(std::vector jpeg) void WebRtcMediaController::decode_loop() { - WebRtcFrameAssembler assembler; - assembler.on_frame = [this](std::vector jpeg) { deliver_jpeg(std::move(jpeg)); }; - AVCodecParameters parameters{}; - parameters.codec_type = AVMEDIA_TYPE_VIDEO; - parameters.codec_id = AV_CODEC_ID_H264; - AVVideoDecoder decoder; - bool decoder_open = false; - int stall_polls = 0; std::unique_lock lock(m_mutex); while (m_alive.load()) { const bool woke = m_cond.wait_for(lock, std::chrono::seconds(2), [this] { - return !m_alive.load() || !m_chunk_queue.empty() || !m_nal_queue.empty(); + return !m_alive.load() || !m_jpeg_queue.empty(); }); if (!m_alive.load()) break; @@ -410,37 +385,11 @@ void WebRtcMediaController::decode_loop() continue; } stall_polls = 0; - if (!m_chunk_queue.empty()) { - auto chunk = std::move(m_chunk_queue.front()); - m_chunk_queue.pop_front(); + if (!m_jpeg_queue.empty()) { + auto jpeg = std::move(m_jpeg_queue.front()); + m_jpeg_queue.pop_front(); lock.unlock(); - assembler.feed(chunk.data(), chunk.size()); - lock.lock(); - } else if (!m_nal_queue.empty()) { - auto nal = std::move(m_nal_queue.front()); - m_nal_queue.pop_front(); - lock.unlock(); - if (!decoder_open) - decoder_open = decoder.open(parameters) == 0; - if (decoder_open) { - AVPacket* packet = av_packet_alloc(); - if (packet && av_new_packet(packet, static_cast(nal.size())) == 0) { - std::memcpy(packet->data, nal.data(), nal.size()); - if (decoder.decode(*packet) == 0) { - wxImage image; - if (decoder.toWxImage(image, wxDefaultSize)) { - { - std::lock_guard frame_lock(m_mutex); - m_video_size = image.GetSize(); - } - if (m_frame_sink) - m_frame_sink(image, image.GetSize()); - report({Status::Playing}); - } - } - } - av_packet_free(&packet); - } + deliver_jpeg(std::move(jpeg)); lock.lock(); } } diff --git a/src/slic3r/GUI/WebRtcMediaController.hpp b/src/slic3r/GUI/WebRtcMediaController.hpp index 1cbd05fcd7..5b06b8e695 100644 --- a/src/slic3r/GUI/WebRtcMediaController.hpp +++ b/src/slic3r/GUI/WebRtcMediaController.hpp @@ -1,12 +1,12 @@ #pragma once #include "IMediaController.hpp" -#include "WebRtcFrameAssembler.hpp" #include #include #include +#include #include #include #include @@ -19,7 +19,6 @@ namespace rtc { class DataChannel; class PeerConnection; -class Track; } namespace Slic3r { namespace GUI { @@ -66,14 +65,12 @@ private: void on_ice(std::string candidate, std::string mid); void on_unavailable(CameraUnavailableReason reason, std::string detail); void decode_loop(); - void enqueue_chunk(std::vector chunk); - void enqueue_nal(std::vector nal); + void enqueue_jpeg(std::vector jpeg); void deliver_jpeg(std::vector jpeg); mutable std::mutex m_mutex; std::condition_variable m_cond; - std::deque> m_chunk_queue; - std::deque> m_nal_queue; + std::deque> m_jpeg_queue; // Remote candidates can arrive before the answer; libdatachannel rejects // addRemoteCandidate until a remote description is set, so buffer them. std::vector> m_pending_candidates; @@ -81,7 +78,6 @@ private: std::unique_ptr m_signaling; std::shared_ptr m_peer_connection; std::shared_ptr m_data_channel; - std::shared_ptr m_video_track; std::thread m_decode_thread; std::atomic m_alive{false}; std::atomic m_epoch{0}; diff --git a/tests/slic3rutils/CMakeLists.txt b/tests/slic3rutils/CMakeLists.txt index fb5b0bda1e..edff97ff42 100644 --- a/tests/slic3rutils/CMakeLists.txt +++ b/tests/slic3rutils/CMakeLists.txt @@ -12,7 +12,6 @@ add_executable(${_TEST_NAME}_tests test_plugin_capabilities_in_use.cpp test_plugin_status.cpp test_printer_agent.cpp - test_webrtc_frame_assembler.cpp test_qidi_printer_agent.cpp test_plugin_install.cpp test_plugin_lifecycle.cpp diff --git a/tests/slic3rutils/test_webrtc_frame_assembler.cpp b/tests/slic3rutils/test_webrtc_frame_assembler.cpp deleted file mode 100644 index 377d2931b7..0000000000 --- a/tests/slic3rutils/test_webrtc_frame_assembler.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -using Slic3r::GUI::WebRtcFrameAssembler; - -static std::vector make_chunk(std::uint32_t frame_id, - std::uint16_t index, - std::uint16_t count, - std::initializer_list payload) -{ - std::vector result(WebRtcFrameAssembler::HeaderSize + payload.size()); - result[0] = std::byte{1}; - result[1] = std::byte{0}; - result[2] = std::byte{static_cast(index >> 8)}; - result[3] = std::byte{static_cast(index)}; - result[4] = std::byte{static_cast(count >> 8)}; - result[5] = std::byte{static_cast(count)}; - result[6] = std::byte{static_cast(frame_id >> 24)}; - result[7] = std::byte{static_cast(frame_id >> 16)}; - result[8] = std::byte{static_cast(frame_id >> 8)}; - result[9] = std::byte{static_cast(frame_id)}; - for (std::size_t i = 0; i < payload.size(); ++i) - result[WebRtcFrameAssembler::HeaderSize + i] = std::byte{static_cast(payload.begin()[i])}; - return result; -} - -TEST_CASE("WebRTC frame assembler joins chunks in order", "[webrtc][unit]") -{ - WebRtcFrameAssembler assembler; - std::vector frame; - assembler.on_frame = [&frame](std::vector value) { frame = std::move(value); }; - - const auto second = make_chunk(7, 1, 2, {'C', 'D'}); - const auto first = make_chunk(7, 0, 2, {'A', 'B'}); - assembler.feed(second.data(), second.size()); - assembler.feed(first.data(), first.size()); - - REQUIRE(frame.size() == 4); - CHECK(std::to_integer(frame[0]) == 'A'); - CHECK(std::to_integer(frame[3]) == 'D'); -} - -TEST_CASE("WebRTC frame assembler discards stale and malformed chunks", "[webrtc][unit]") -{ - WebRtcFrameAssembler assembler; - int frames = 0; - assembler.on_frame = [&frames](std::vector) { ++frames; }; - - const auto stale = make_chunk(1, 0, 2, {'A'}); - const auto current = make_chunk(2, 0, 1, {'B'}); - const auto stale_tail = make_chunk(1, 1, 2, {'C'}); - assembler.feed(stale.data(), stale.size()); - assembler.feed(current.data(), current.size()); - assembler.feed(stale_tail.data(), stale_tail.size()); - - CHECK(frames == 1); - - auto malformed = make_chunk(3, 0, 0, {'X'}); - assembler.feed(malformed.data(), malformed.size()); - CHECK(frames == 1); -}