diff --git a/src/slic3r/CMakeLists.txt b/src/slic3r/CMakeLists.txt index 322b3b3f0e..c1ce3b22a6 100644 --- a/src/slic3r/CMakeLists.txt +++ b/src/slic3r/CMakeLists.txt @@ -344,8 +344,6 @@ set(SLIC3R_GUI_SOURCES GUI/MediaFilePanel.h GUI/MediaPlayCtrl.cpp GUI/MediaPlayCtrl.h - GUI/WebRtcFrameAssembler.cpp - GUI/WebRtcFrameAssembler.hpp GUI/WebRtcMediaController.cpp GUI/WebRtcMediaController.hpp GUI/MeshUtils.cpp diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.cpp b/src/slic3r/GUI/WebRtcFrameAssembler.cpp deleted file mode 100644 index 2771cc0887..0000000000 --- a/src/slic3r/GUI/WebRtcFrameAssembler.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "WebRtcFrameAssembler.hpp" - -#include -#include - -namespace Slic3r { namespace GUI { - -void WebRtcFrameAssembler::discard() -{ - m_active = false; - m_frame_id = 0; - m_chunk_count = 0; - m_received_chunks = 0; - m_total_size = 0; - m_chunks.clear(); - m_received.clear(); -} - -void WebRtcFrameAssembler::reset() -{ - discard(); -} - -void WebRtcFrameAssembler::feed(const std::byte* data, std::size_t len) -{ - if (data == nullptr || len < HeaderSize) - return; - - if (std::to_integer(data[0]) != 1) - return; - - const std::uint16_t chunk_index = static_cast((std::to_integer(data[2]) << 8) | - std::to_integer(data[3])); - const std::uint16_t chunk_count = static_cast((std::to_integer(data[4]) << 8) | - std::to_integer(data[5])); - const std::uint32_t frame_id = (static_cast(std::to_integer(data[6])) << 24) | - (static_cast(std::to_integer(data[7])) << 16) | - (static_cast(std::to_integer(data[8])) << 8) | - static_cast(std::to_integer(data[9])); - const std::size_t payload_size = len - HeaderSize; - - if (chunk_count == 0 || chunk_count > MaxChunkCount || chunk_index >= chunk_count || - payload_size > MaxChunkPayload) - return; - - if (!m_active || frame_id > m_frame_id) { - discard(); - m_active = true; - m_frame_id = frame_id; - m_chunk_count = chunk_count; - m_chunks.resize(chunk_count); - m_received.assign(chunk_count, false); - } else if (frame_id < m_frame_id) { - return; - } else if (chunk_count != m_chunk_count) { - discard(); - return; - } - - Frame& chunk = m_chunks[chunk_index]; - if (m_received[chunk_index]) - return; - - if (m_total_size > MaxFrameSize || payload_size > MaxFrameSize - m_total_size) { - discard(); - return; - } - - chunk.assign(data + HeaderSize, data + len); - m_received[chunk_index] = true; - m_total_size += payload_size; - ++m_received_chunks; - - if (m_received_chunks != m_chunk_count) - return; - - Frame frame; - frame.reserve(m_total_size); - for (const Frame& slice : m_chunks) - frame.insert(frame.end(), slice.begin(), slice.end()); - if (on_frame) - on_frame(std::move(frame)); - discard(); -} - -}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcFrameAssembler.hpp b/src/slic3r/GUI/WebRtcFrameAssembler.hpp deleted file mode 100644 index c526cad588..0000000000 --- a/src/slic3r/GUI/WebRtcFrameAssembler.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace Slic3r { namespace GUI { - -class WebRtcFrameAssembler { -public: - using Frame = std::vector; - - // The wire protocol uses a 10-byte header followed by one JPEG slice: - // version, flags, chunk index, chunk count, and frame id, all in network - // byte order where applicable. - static constexpr std::size_t HeaderSize = 10; - static constexpr std::size_t MaxChunkPayload = 16000; - static constexpr std::size_t MaxChunkCount = 4096; - static constexpr std::size_t MaxFrameSize = 8 * 1024 * 1024; - - std::function on_frame; - - void feed(const std::byte* data, std::size_t len); - void reset(); - -private: - void discard(); - - bool m_active = false; - std::uint32_t m_frame_id = 0; - std::uint16_t m_chunk_count = 0; - std::size_t m_received_chunks = 0; - std::size_t m_total_size = 0; - std::vector m_chunks; - std::vector m_received; -}; - -}} // namespace Slic3r::GUI diff --git a/src/slic3r/GUI/WebRtcMediaController.cpp b/src/slic3r/GUI/WebRtcMediaController.cpp index 7eafbd4a3a..36a21c5c5d 100644 --- a/src/slic3r/GUI/WebRtcMediaController.cpp +++ b/src/slic3r/GUI/WebRtcMediaController.cpp @@ -1,18 +1,13 @@ #include "WebRtcMediaController.hpp" -#include "AVVideoDecoder.hpp" - #include #include #include -#include #include #include -#include - namespace { void init_rtc_logger_once() { @@ -25,10 +20,6 @@ void init_rtc_logger_once() } } // namespace -extern "C" { -#include -} - namespace Slic3r { namespace GUI { WebRtcMediaController::WebRtcMediaController(std::function frame_sink, @@ -75,8 +66,7 @@ void WebRtcMediaController::StartSession(std::unique_ptr lock(m_mutex); m_signaling = std::move(channel); - m_chunk_queue.clear(); - m_nal_queue.clear(); + m_jpeg_queue.clear(); m_pending_candidates.clear(); m_remote_description_set = false; m_video_size = wxDefaultSize; @@ -130,7 +120,6 @@ void WebRtcMediaController::teardown(bool notify) signaling = std::move(m_signaling); peer_connection = std::move(m_peer_connection); m_data_channel.reset(); - m_video_track.reset(); } if (signaling) signaling->close(); @@ -140,8 +129,7 @@ void WebRtcMediaController::teardown(bool notify) m_decode_thread.join(); { std::lock_guard lock(m_mutex); - m_chunk_queue.clear(); - m_nal_queue.clear(); + m_jpeg_queue.clear(); } if (was_alive && notify) report({Status::Stopped}); @@ -170,7 +158,7 @@ void WebRtcMediaController::bind_data_channel(const std::shared_ptronMessage( [this](rtc::binary data) { if (m_alive.load()) - enqueue_chunk(std::vector(data.begin(), data.end())); + enqueue_jpeg(std::vector(data.begin(), data.end())); }, [](rtc::string) {}); } @@ -179,6 +167,10 @@ void WebRtcMediaController::on_ready(std::vector servers) { init_rtc_logger_once(); rtc::Configuration configuration; + // Allow complete-JPEG DataChannel messages up to 1 MiB. This value is + // advertised in SDP and becomes the upper bound for frames OrcaSonar can + // send to OrcaSlicer. + configuration.maxMessageSize = 1024 * 1024; for (const CameraIceServer& server : servers) { try { rtc::IceServer ice_server(server.urls); @@ -234,16 +226,16 @@ void WebRtcMediaController::on_ready(std::vector servers) rtc::DataChannelInit init; init.reliability.unordered = true; - init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + // init.reliability.maxPacketLifeTime = std::chrono::milliseconds(350); + // Request one complete JPEG frame per DataChannel message. OrcaSonar + // keeps the legacy chunked protocol for clients that omit this property. + init.protocol = "orca-jpeg"; auto data_channel = peer_connection->createDataChannel("camera", init); if (data_channel) bind_data_channel(data_channel); - // NOTE: the H.264 RTP RecvOnly track is intentionally NOT added to the offer - // yet. OrcaSonar answers application-only (no m=video), which makes - // libdatachannel renegotiate and send a second offer that OrcaSonar rejects - // with "webrtc.unavailable: error". Re-add the video m-line (enqueue_nal / - // the decode_loop NAL branch are already in place) once OrcaSonar answers it. + // Camera media is carried as one complete JPEG per DataChannel message; + // no RTP video track or application-level framing is required. std::shared_ptr peer_for_description; { @@ -327,21 +319,12 @@ void WebRtcMediaController::on_unavailable(CameraUnavailableReason reason, std:: report({Status::Failed, code}); } -void WebRtcMediaController::enqueue_chunk(std::vector chunk) +void WebRtcMediaController::enqueue_jpeg(std::vector jpeg) { std::lock_guard lock(m_mutex); - if (m_chunk_queue.size() >= 8) - m_chunk_queue.pop_front(); - m_chunk_queue.emplace_back(std::move(chunk)); - m_cond.notify_one(); -} - -void WebRtcMediaController::enqueue_nal(std::vector nal) -{ - std::lock_guard lock(m_mutex); - if (m_nal_queue.size() >= 4) - m_nal_queue.pop_front(); - m_nal_queue.emplace_back(std::move(nal)); + if (m_jpeg_queue.size() >= 4) + m_jpeg_queue.pop_front(); + m_jpeg_queue.emplace_back(std::move(jpeg)); m_cond.notify_one(); } @@ -376,19 +359,11 @@ void WebRtcMediaController::deliver_jpeg(std::vector jpeg) void WebRtcMediaController::decode_loop() { - WebRtcFrameAssembler assembler; - assembler.on_frame = [this](std::vector jpeg) { deliver_jpeg(std::move(jpeg)); }; - AVCodecParameters parameters{}; - parameters.codec_type = AVMEDIA_TYPE_VIDEO; - parameters.codec_id = AV_CODEC_ID_H264; - AVVideoDecoder decoder; - bool decoder_open = false; - int stall_polls = 0; std::unique_lock lock(m_mutex); while (m_alive.load()) { const bool woke = m_cond.wait_for(lock, std::chrono::seconds(2), [this] { - return !m_alive.load() || !m_chunk_queue.empty() || !m_nal_queue.empty(); + return !m_alive.load() || !m_jpeg_queue.empty(); }); if (!m_alive.load()) break; @@ -410,37 +385,11 @@ void WebRtcMediaController::decode_loop() continue; } stall_polls = 0; - if (!m_chunk_queue.empty()) { - auto chunk = std::move(m_chunk_queue.front()); - m_chunk_queue.pop_front(); + if (!m_jpeg_queue.empty()) { + auto jpeg = std::move(m_jpeg_queue.front()); + m_jpeg_queue.pop_front(); lock.unlock(); - assembler.feed(chunk.data(), chunk.size()); - lock.lock(); - } else if (!m_nal_queue.empty()) { - auto nal = std::move(m_nal_queue.front()); - m_nal_queue.pop_front(); - lock.unlock(); - if (!decoder_open) - decoder_open = decoder.open(parameters) == 0; - if (decoder_open) { - AVPacket* packet = av_packet_alloc(); - if (packet && av_new_packet(packet, static_cast(nal.size())) == 0) { - std::memcpy(packet->data, nal.data(), nal.size()); - if (decoder.decode(*packet) == 0) { - wxImage image; - if (decoder.toWxImage(image, wxDefaultSize)) { - { - std::lock_guard frame_lock(m_mutex); - m_video_size = image.GetSize(); - } - if (m_frame_sink) - m_frame_sink(image, image.GetSize()); - report({Status::Playing}); - } - } - } - av_packet_free(&packet); - } + deliver_jpeg(std::move(jpeg)); lock.lock(); } } diff --git a/src/slic3r/GUI/WebRtcMediaController.hpp b/src/slic3r/GUI/WebRtcMediaController.hpp index 1cbd05fcd7..5b06b8e695 100644 --- a/src/slic3r/GUI/WebRtcMediaController.hpp +++ b/src/slic3r/GUI/WebRtcMediaController.hpp @@ -1,12 +1,12 @@ #pragma once #include "IMediaController.hpp" -#include "WebRtcFrameAssembler.hpp" #include #include #include +#include #include #include #include @@ -19,7 +19,6 @@ namespace rtc { class DataChannel; class PeerConnection; -class Track; } namespace Slic3r { namespace GUI { @@ -66,14 +65,12 @@ private: void on_ice(std::string candidate, std::string mid); void on_unavailable(CameraUnavailableReason reason, std::string detail); void decode_loop(); - void enqueue_chunk(std::vector chunk); - void enqueue_nal(std::vector nal); + void enqueue_jpeg(std::vector jpeg); void deliver_jpeg(std::vector jpeg); mutable std::mutex m_mutex; std::condition_variable m_cond; - std::deque> m_chunk_queue; - std::deque> m_nal_queue; + std::deque> m_jpeg_queue; // Remote candidates can arrive before the answer; libdatachannel rejects // addRemoteCandidate until a remote description is set, so buffer them. std::vector> m_pending_candidates; @@ -81,7 +78,6 @@ private: std::unique_ptr m_signaling; std::shared_ptr m_peer_connection; std::shared_ptr m_data_channel; - std::shared_ptr m_video_track; std::thread m_decode_thread; std::atomic m_alive{false}; std::atomic m_epoch{0}; diff --git a/tests/slic3rutils/CMakeLists.txt b/tests/slic3rutils/CMakeLists.txt index fb5b0bda1e..edff97ff42 100644 --- a/tests/slic3rutils/CMakeLists.txt +++ b/tests/slic3rutils/CMakeLists.txt @@ -12,7 +12,6 @@ add_executable(${_TEST_NAME}_tests test_plugin_capabilities_in_use.cpp test_plugin_status.cpp test_printer_agent.cpp - test_webrtc_frame_assembler.cpp test_qidi_printer_agent.cpp test_plugin_install.cpp test_plugin_lifecycle.cpp diff --git a/tests/slic3rutils/test_webrtc_frame_assembler.cpp b/tests/slic3rutils/test_webrtc_frame_assembler.cpp deleted file mode 100644 index 377d2931b7..0000000000 --- a/tests/slic3rutils/test_webrtc_frame_assembler.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -using Slic3r::GUI::WebRtcFrameAssembler; - -static std::vector make_chunk(std::uint32_t frame_id, - std::uint16_t index, - std::uint16_t count, - std::initializer_list payload) -{ - std::vector result(WebRtcFrameAssembler::HeaderSize + payload.size()); - result[0] = std::byte{1}; - result[1] = std::byte{0}; - result[2] = std::byte{static_cast(index >> 8)}; - result[3] = std::byte{static_cast(index)}; - result[4] = std::byte{static_cast(count >> 8)}; - result[5] = std::byte{static_cast(count)}; - result[6] = std::byte{static_cast(frame_id >> 24)}; - result[7] = std::byte{static_cast(frame_id >> 16)}; - result[8] = std::byte{static_cast(frame_id >> 8)}; - result[9] = std::byte{static_cast(frame_id)}; - for (std::size_t i = 0; i < payload.size(); ++i) - result[WebRtcFrameAssembler::HeaderSize + i] = std::byte{static_cast(payload.begin()[i])}; - return result; -} - -TEST_CASE("WebRTC frame assembler joins chunks in order", "[webrtc][unit]") -{ - WebRtcFrameAssembler assembler; - std::vector frame; - assembler.on_frame = [&frame](std::vector value) { frame = std::move(value); }; - - const auto second = make_chunk(7, 1, 2, {'C', 'D'}); - const auto first = make_chunk(7, 0, 2, {'A', 'B'}); - assembler.feed(second.data(), second.size()); - assembler.feed(first.data(), first.size()); - - REQUIRE(frame.size() == 4); - CHECK(std::to_integer(frame[0]) == 'A'); - CHECK(std::to_integer(frame[3]) == 'D'); -} - -TEST_CASE("WebRTC frame assembler discards stale and malformed chunks", "[webrtc][unit]") -{ - WebRtcFrameAssembler assembler; - int frames = 0; - assembler.on_frame = [&frames](std::vector) { ++frames; }; - - const auto stale = make_chunk(1, 0, 2, {'A'}); - const auto current = make_chunk(2, 0, 1, {'B'}); - const auto stale_tail = make_chunk(1, 1, 2, {'C'}); - assembler.feed(stale.data(), stale.size()); - assembler.feed(current.data(), current.size()); - assembler.feed(stale_tail.data(), stale_tail.size()); - - CHECK(frames == 1); - - auto malformed = make_chunk(3, 0, 0, {'X'}); - assembler.feed(malformed.data(), malformed.size()); - CHECK(frames == 1); -}