feat: remove frame assembler and change config to set protocol

src/slic3r/CMakeLists.txt

@@ -344,8 +344,6 @@ set(SLIC3R_GUI_SOURCES
     GUI/MediaFilePanel.h
     GUI/MediaPlayCtrl.cpp
     GUI/MediaPlayCtrl.h
-    GUI/WebRtcFrameAssembler.cpp
-    GUI/WebRtcFrameAssembler.hpp
     GUI/WebRtcMediaController.cpp
     GUI/WebRtcMediaController.hpp
     GUI/MeshUtils.cpp

src/slic3r/GUI/WebRtcFrameAssembler.cpp

@@ -1,86 +0,0 @@
-#include "WebRtcFrameAssembler.hpp"
-
-#include <algorithm>
-#include <limits>
-
-namespace Slic3r { namespace GUI {
-
-void WebRtcFrameAssembler::discard()
-{
-    m_active = false;
-    m_frame_id = 0;
-    m_chunk_count = 0;
-    m_received_chunks = 0;
-    m_total_size = 0;
-    m_chunks.clear();
-    m_received.clear();
-}
-
-void WebRtcFrameAssembler::reset()
-{
-    discard();
-}
-
-void WebRtcFrameAssembler::feed(const std::byte* data, std::size_t len)
-{
-    if (data == nullptr || len < HeaderSize)
-        return;
-
-    if (std::to_integer<unsigned int>(data[0]) != 1)
-        return;
-
-    const std::uint16_t chunk_index = static_cast<std::uint16_t>((std::to_integer<unsigned int>(data[2]) << 8) |
-                                                                  std::to_integer<unsigned int>(data[3]));
-    const std::uint16_t chunk_count = static_cast<std::uint16_t>((std::to_integer<unsigned int>(data[4]) << 8) |
-                                                                  std::to_integer<unsigned int>(data[5]));
-    const std::uint32_t frame_id = (static_cast<std::uint32_t>(std::to_integer<unsigned int>(data[6])) << 24) |
-                                   (static_cast<std::uint32_t>(std::to_integer<unsigned int>(data[7])) << 16) |
-                                   (static_cast<std::uint32_t>(std::to_integer<unsigned int>(data[8])) << 8) |
-                                   static_cast<std::uint32_t>(std::to_integer<unsigned int>(data[9]));
-    const std::size_t payload_size = len - HeaderSize;
-
-    if (chunk_count == 0 || chunk_count > MaxChunkCount || chunk_index >= chunk_count ||
-        payload_size > MaxChunkPayload)
-        return;
-
-    if (!m_active || frame_id > m_frame_id) {
-        discard();
-        m_active = true;
-        m_frame_id = frame_id;
-        m_chunk_count = chunk_count;
-        m_chunks.resize(chunk_count);
-        m_received.assign(chunk_count, false);
-    } else if (frame_id < m_frame_id) {
-        return;
-    } else if (chunk_count != m_chunk_count) {
-        discard();
-        return;
-    }
-
-    Frame& chunk = m_chunks[chunk_index];
-    if (m_received[chunk_index])
-        return;
-
-    if (m_total_size > MaxFrameSize || payload_size > MaxFrameSize - m_total_size) {
-        discard();
-        return;
-    }
-
-    chunk.assign(data + HeaderSize, data + len);
-    m_received[chunk_index] = true;
-    m_total_size += payload_size;
-    ++m_received_chunks;
-
-    if (m_received_chunks != m_chunk_count)
-        return;
-
-    Frame frame;
-    frame.reserve(m_total_size);
-    for (const Frame& slice : m_chunks)
-        frame.insert(frame.end(), slice.begin(), slice.end());
-    if (on_frame)
-        on_frame(std::move(frame));
-    discard();
-}
-
-}} // namespace Slic3r::GUI

src/slic3r/GUI/WebRtcFrameAssembler.hpp

@@ -1,39 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-#include <functional>
-#include <vector>
-
-namespace Slic3r { namespace GUI {
-
-class WebRtcFrameAssembler {
-public:
-    using Frame = std::vector<std::byte>;
-
-    // The wire protocol uses a 10-byte header followed by one JPEG slice:
-    // version, flags, chunk index, chunk count, and frame id, all in network
-    // byte order where applicable.
-    static constexpr std::size_t HeaderSize = 10;
-    static constexpr std::size_t MaxChunkPayload = 16000;
-    static constexpr std::size_t MaxChunkCount = 4096;
-    static constexpr std::size_t MaxFrameSize = 8 * 1024 * 1024;
-
-    std::function<void(Frame)> on_frame;
-
-    void feed(const std::byte* data, std::size_t len);
-    void reset();
-
-private:
-    void discard();
-
-    bool m_active = false;
-    std::uint32_t m_frame_id = 0;
-    std::uint16_t m_chunk_count = 0;
-    std::size_t m_received_chunks = 0;
-    std::size_t m_total_size = 0;
-    std::vector<Frame> m_chunks;
-    std::vector<bool> m_received;
-};
-
-}} // namespace Slic3r::GUI

src/slic3r/GUI/WebRtcMediaController.cpp

@@ -1,18 +1,13 @@
 #include "WebRtcMediaController.hpp"
 
-#include "AVVideoDecoder.hpp"
-
 #include <rtc/common.hpp>
 #include <rtc/rtc.hpp>
 
 #include <mutex>
-#include <variant>
 #include <wx/mstream.h>
 
 #include <boost/log/trivial.hpp>
 
-#include <cstring>
-
 namespace {
 void init_rtc_logger_once()
 {
@@ -25,10 +20,6 @@ void init_rtc_logger_once()
 }
 } // namespace
 
-extern "C" {
-#include <libavcodec/avcodec.h>
-}
-
 namespace Slic3r { namespace GUI {
 
 WebRtcMediaController::WebRtcMediaController(std::function<void(const wxImage&, wxSize)> frame_sink,
@@ -75,8 +66,7 @@ void WebRtcMediaController::StartSession(std::unique_ptr<ICameraSignalingChannel
     {
         std::lock_guard<std::mutex> lock(m_mutex);
         m_signaling = std::move(channel);
-        m_chunk_queue.clear();
+        m_jpeg_queue.clear();
-        m_nal_queue.clear();
         m_pending_candidates.clear();
         m_remote_description_set = false;
         m_video_size = wxDefaultSize;
@@ -130,7 +120,6 @@ void WebRtcMediaController::teardown(bool notify)
         signaling = std::move(m_signaling);
         peer_connection = std::move(m_peer_connection);
         m_data_channel.reset();
-        m_video_track.reset();
     }
     if (signaling)
         signaling->close();
@@ -140,8 +129,7 @@ void WebRtcMediaController::teardown(bool notify)
         m_decode_thread.join();
     {
         std::lock_guard<std::mutex> lock(m_mutex);
-        m_chunk_queue.clear();
+        m_jpeg_queue.clear();
-        m_nal_queue.clear();
     }
     if (was_alive && notify)
         report({Status::Stopped});
@@ -170,7 +158,7 @@ void WebRtcMediaController::bind_data_channel(const std::shared_ptr<rtc::DataCha
     dc->onMessage(
         [this](rtc::binary data) {
             if (m_alive.load())
-                enqueue_chunk(std::vector<std::byte>(data.begin(), data.end()));
+                enqueue_jpeg(std::vector<std::byte>(data.begin(), data.end()));
         },
         [](rtc::string) {});
 }
@@ -179,6 +167,10 @@ void WebRtcMediaController::on_ready(std::vector<CameraIceServer> servers)
 {
     init_rtc_logger_once();
     rtc::Configuration configuration;
+    // Allow complete-JPEG DataChannel messages up to 1 MiB. This value is
+    // advertised in SDP and becomes the upper bound for frames OrcaSonar can
+    // send to OrcaSlicer.
+    configuration.maxMessageSize = 1024 * 1024;
     for (const CameraIceServer& server : servers) {
         try {
             rtc::IceServer ice_server(server.urls);
@@ -234,16 +226,16 @@ void WebRtcMediaController::on_ready(std::vector<CameraIceServer> servers)
 
     rtc::DataChannelInit init;
     init.reliability.unordered = true;
-    init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350);
+    // init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350);
+    // Request one complete JPEG frame per DataChannel message. OrcaSonar
+    // keeps the legacy chunked protocol for clients that omit this property.
+    init.protocol = "orca-jpeg";
     auto data_channel = peer_connection->createDataChannel("camera", init);
     if (data_channel)
         bind_data_channel(data_channel);
 
-    // NOTE: the H.264 RTP RecvOnly track is intentionally NOT added to the offer
+    // Camera media is carried as one complete JPEG per DataChannel message;
-    // yet. OrcaSonar answers application-only (no m=video), which makes
+    // no RTP video track or application-level framing is required.
-    // libdatachannel renegotiate and send a second offer that OrcaSonar rejects
-    // with "webrtc.unavailable: error". Re-add the video m-line (enqueue_nal /
-    // the decode_loop NAL branch are already in place) once OrcaSonar answers it.
 
     std::shared_ptr<rtc::PeerConnection> peer_for_description;
     {
@@ -327,21 +319,12 @@ void WebRtcMediaController::on_unavailable(CameraUnavailableReason reason, std::
     report({Status::Failed, code});
 }
 
-void WebRtcMediaController::enqueue_chunk(std::vector<std::byte> chunk)
+void WebRtcMediaController::enqueue_jpeg(std::vector<std::byte> jpeg)
 {
     std::lock_guard<std::mutex> lock(m_mutex);
-    if (m_chunk_queue.size() >= 8)
+    if (m_jpeg_queue.size() >= 4)
-        m_chunk_queue.pop_front();
+        m_jpeg_queue.pop_front();
-    m_chunk_queue.emplace_back(std::move(chunk));
+    m_jpeg_queue.emplace_back(std::move(jpeg));
-    m_cond.notify_one();
-}
-
-void WebRtcMediaController::enqueue_nal(std::vector<std::byte> nal)
-{
-    std::lock_guard<std::mutex> lock(m_mutex);
-    if (m_nal_queue.size() >= 4)
-        m_nal_queue.pop_front();
-    m_nal_queue.emplace_back(std::move(nal));
     m_cond.notify_one();
 }
 
@@ -376,19 +359,11 @@ void WebRtcMediaController::deliver_jpeg(std::vector<std::byte> jpeg)
 
 void WebRtcMediaController::decode_loop()
 {
-    WebRtcFrameAssembler assembler;
-    assembler.on_frame = [this](std::vector<std::byte> jpeg) { deliver_jpeg(std::move(jpeg)); };
-    AVCodecParameters parameters{};
-    parameters.codec_type = AVMEDIA_TYPE_VIDEO;
-    parameters.codec_id = AV_CODEC_ID_H264;
-    AVVideoDecoder decoder;
-    bool decoder_open = false;
-
     int stall_polls = 0;
     std::unique_lock<std::mutex> lock(m_mutex);
     while (m_alive.load()) {
         const bool woke = m_cond.wait_for(lock, std::chrono::seconds(2), [this] {
-            return !m_alive.load() || !m_chunk_queue.empty() || !m_nal_queue.empty();
+            return !m_alive.load() || !m_jpeg_queue.empty();
         });
         if (!m_alive.load())
             break;
@@ -410,37 +385,11 @@ void WebRtcMediaController::decode_loop()
             continue;
         }
         stall_polls = 0;
-        if (!m_chunk_queue.empty()) {
+        if (!m_jpeg_queue.empty()) {
-            auto chunk = std::move(m_chunk_queue.front());
+            auto jpeg = std::move(m_jpeg_queue.front());
-            m_chunk_queue.pop_front();
+            m_jpeg_queue.pop_front();
             lock.unlock();
-            assembler.feed(chunk.data(), chunk.size());
+            deliver_jpeg(std::move(jpeg));
-            lock.lock();
-        } else if (!m_nal_queue.empty()) {
-            auto nal = std::move(m_nal_queue.front());
-            m_nal_queue.pop_front();
-            lock.unlock();
-            if (!decoder_open)
-                decoder_open = decoder.open(parameters) == 0;
-            if (decoder_open) {
-                AVPacket* packet = av_packet_alloc();
-                if (packet && av_new_packet(packet, static_cast<int>(nal.size())) == 0) {
-                    std::memcpy(packet->data, nal.data(), nal.size());
-                    if (decoder.decode(*packet) == 0) {
-                        wxImage image;
-                        if (decoder.toWxImage(image, wxDefaultSize)) {
-                            {
-                                std::lock_guard<std::mutex> frame_lock(m_mutex);
-                                m_video_size = image.GetSize();
-                            }
-                            if (m_frame_sink)
-                                m_frame_sink(image, image.GetSize());
-                            report({Status::Playing});
-                        }
-                    }
-                }
-                av_packet_free(&packet);
-            }
             lock.lock();
         }
     }

src/slic3r/GUI/WebRtcMediaController.hpp

@@ -1,12 +1,12 @@
 #pragma once
 
 #include "IMediaController.hpp"
-#include "WebRtcFrameAssembler.hpp"
 
 #include <wx/image.h>
 
 #include <atomic>
 #include <chrono>
+#include <cstddef>
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
@@ -19,7 +19,6 @@
 namespace rtc {
 class DataChannel;
 class PeerConnection;
-class Track;
 }
 
 namespace Slic3r { namespace GUI {
@@ -66,14 +65,12 @@ private:
     void on_ice(std::string candidate, std::string mid);
     void on_unavailable(CameraUnavailableReason reason, std::string detail);
     void decode_loop();
-    void enqueue_chunk(std::vector<std::byte> chunk);
+    void enqueue_jpeg(std::vector<std::byte> jpeg);
-    void enqueue_nal(std::vector<std::byte> nal);
     void deliver_jpeg(std::vector<std::byte> jpeg);
 
     mutable std::mutex m_mutex;
     std::condition_variable m_cond;
-    std::deque<std::vector<std::byte>> m_chunk_queue;
+    std::deque<std::vector<std::byte>> m_jpeg_queue;
-    std::deque<std::vector<std::byte>> m_nal_queue;
     // Remote candidates can arrive before the answer; libdatachannel rejects
     // addRemoteCandidate until a remote description is set, so buffer them.
     std::vector<std::pair<std::string, std::string>> m_pending_candidates;
@@ -81,7 +78,6 @@ private:
     std::unique_ptr<ICameraSignalingChannel> m_signaling;
     std::shared_ptr<rtc::PeerConnection> m_peer_connection;
     std::shared_ptr<rtc::DataChannel> m_data_channel;
-    std::shared_ptr<rtc::Track> m_video_track;
     std::thread m_decode_thread;
     std::atomic<bool> m_alive{false};
     std::atomic<std::uint64_t> m_epoch{0};

tests/slic3rutils/CMakeLists.txt

@@ -12,7 +12,6 @@ add_executable(${_TEST_NAME}_tests
     test_plugin_capabilities_in_use.cpp
     test_plugin_status.cpp
     test_printer_agent.cpp
-    test_webrtc_frame_assembler.cpp
     test_qidi_printer_agent.cpp
     test_plugin_install.cpp
     test_plugin_lifecycle.cpp

tests/slic3rutils/test_webrtc_frame_assembler.cpp

@@ -1,67 +0,0 @@
-#include <catch2/catch_all.hpp>
-
-#include <slic3r/GUI/WebRtcFrameAssembler.hpp>
-
-#include <cstdint>
-#include <cstddef>
-#include <initializer_list>
-#include <vector>
-
-using Slic3r::GUI::WebRtcFrameAssembler;
-
-static std::vector<std::byte> make_chunk(std::uint32_t frame_id,
-                                         std::uint16_t index,
-                                         std::uint16_t count,
-                                         std::initializer_list<unsigned int> payload)
-{
-    std::vector<std::byte> result(WebRtcFrameAssembler::HeaderSize + payload.size());
-    result[0] = std::byte{1};
-    result[1] = std::byte{0};
-    result[2] = std::byte{static_cast<unsigned char>(index >> 8)};
-    result[3] = std::byte{static_cast<unsigned char>(index)};
-    result[4] = std::byte{static_cast<unsigned char>(count >> 8)};
-    result[5] = std::byte{static_cast<unsigned char>(count)};
-    result[6] = std::byte{static_cast<unsigned char>(frame_id >> 24)};
-    result[7] = std::byte{static_cast<unsigned char>(frame_id >> 16)};
-    result[8] = std::byte{static_cast<unsigned char>(frame_id >> 8)};
-    result[9] = std::byte{static_cast<unsigned char>(frame_id)};
-    for (std::size_t i = 0; i < payload.size(); ++i)
-        result[WebRtcFrameAssembler::HeaderSize + i] = std::byte{static_cast<unsigned char>(payload.begin()[i])};
-    return result;
-}
-
-TEST_CASE("WebRTC frame assembler joins chunks in order", "[webrtc][unit]")
-{
-    WebRtcFrameAssembler assembler;
-    std::vector<std::byte> frame;
-    assembler.on_frame = [&frame](std::vector<std::byte> value) { frame = std::move(value); };
-
-    const auto second = make_chunk(7, 1, 2, {'C', 'D'});
-    const auto first = make_chunk(7, 0, 2, {'A', 'B'});
-    assembler.feed(second.data(), second.size());
-    assembler.feed(first.data(), first.size());
-
-    REQUIRE(frame.size() == 4);
-    CHECK(std::to_integer<char>(frame[0]) == 'A');
-    CHECK(std::to_integer<char>(frame[3]) == 'D');
-}
-
-TEST_CASE("WebRTC frame assembler discards stale and malformed chunks", "[webrtc][unit]")
-{
-    WebRtcFrameAssembler assembler;
-    int frames = 0;
-    assembler.on_frame = [&frames](std::vector<std::byte>) { ++frames; };
-
-    const auto stale = make_chunk(1, 0, 2, {'A'});
-    const auto current = make_chunk(2, 0, 1, {'B'});
-    const auto stale_tail = make_chunk(1, 1, 2, {'C'});
-    assembler.feed(stale.data(), stale.size());
-    assembler.feed(current.data(), current.size());
-    assembler.feed(stale_tail.data(), stale_tail.size());
-
-    CHECK(frames == 1);
-
-    auto malformed = make_chunk(3, 0, 0, {'X'});
-    assembler.feed(malformed.data(), malformed.size());
-    CHECK(frames == 1);
-}