feat: support generic RTSP camera streams

CMakeLists.txt

@@ -1068,6 +1068,7 @@ function(orcaslicer_copy_dlls target config postfix output_dlls)
             ${CMAKE_PREFIX_PATH}/bin/occt/TKXDESTEP.dll
             ${CMAKE_PREFIX_PATH}/bin/occt/TKXSBase.dll
             ${CMAKE_PREFIX_PATH}/bin/freetype.dll
+            ${CMAKE_PREFIX_PATH}/bin/avformat-61.dll
             ${CMAKE_PREFIX_PATH}/bin/avcodec-61.dll
             ${CMAKE_PREFIX_PATH}/bin/swresample-5.dll
             ${CMAKE_PREFIX_PATH}/bin/swscale-8.dll
@@ -1107,6 +1108,7 @@ function(orcaslicer_copy_dlls target config postfix output_dlls)
         ${_out_dir}/TKXSBase.dll
 
         ${_out_dir}/freetype.dll
+        ${_out_dir}/avformat-61.dll
         ${_out_dir}/avcodec-61.dll
         ${_out_dir}/swresample-5.dll
         ${_out_dir}/swscale-8.dll
@@ -1129,7 +1131,10 @@ function(orcaslicer_copy_sos target config postfix output_sos)
         set(_out_dir "${CMAKE_CURRENT_BINARY_DIR}")
     endif ()
 
-    file(COPY ${CMAKE_PREFIX_PATH}/lib/libavcodec.so
+    file(COPY ${CMAKE_PREFIX_PATH}/lib/libavformat.so
+            ${CMAKE_PREFIX_PATH}/lib/libavformat.so.61
+            ${CMAKE_PREFIX_PATH}/lib/libavformat.so.61.1.100
+            ${CMAKE_PREFIX_PATH}/lib/libavcodec.so
             ${CMAKE_PREFIX_PATH}/lib/libavcodec.so.61
             ${CMAKE_PREFIX_PATH}/lib/libavcodec.so.61.3.100
             ${CMAKE_PREFIX_PATH}/lib/libavutil.so
@@ -1144,6 +1149,9 @@ function(orcaslicer_copy_sos target config postfix output_sos)
          DESTINATION ${_out_dir})
 
     set(${output_sos}
+        ${_out_dir}/libavformat.so
+        ${_out_dir}/libavformat.so.61
+        ${_out_dir}/libavformat.so.61.1.100
         ${_out_dir}/libavcodec.so
         ${_out_dir}/libavcodec.so.61
         ${_out_dir}/libavcodec.so.61.3.100
@@ -1272,6 +1280,8 @@ endif ()
 
 if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
     set(LIBRARY_FILES
+    ${LIBDIR_BIN}/libavformat.so.61
+    ${LIBDIR_BIN}/libavformat.so.61.1.100
     ${LIBDIR_BIN}/libavcodec.so.61
     ${LIBDIR_BIN}/libavcodec.so.61.3.100
     ${LIBDIR_BIN}/libavutil.so.59

deps/FFMPEG/FFMPEG.cmake

@@ -69,14 +69,14 @@ else ()
             --disable-filters
             --enable-filter=*null*,afade,*fifo,*format,*resample,aeval,allrgb,allyuv,atempo,pan,*bars,color,*key,crop,draw*,eq*,framerate,*_qsv,*_vaapi,*v4l2*,hw*,scale,volume,test*
             --disable-protocols
-            --enable-protocol=file,fd,pipe,rtp,udp
+            --enable-protocol=file,fd,pipe,rtp,tcp,udp
             --disable-muxers
             --enable-muxer=rtp
             --disable-encoders
             --disable-decoders
             --enable-decoder=*aac*,h264*,mp3*,mjpeg,rv*
             --disable-demuxers
-            --enable-demuxer=h264,mp3,mov
+            --enable-demuxer=h264,mp3,mov,rtsp,sdp
             --disable-zlib
             --disable-avdevice
         BUILD_IN_SOURCE ON

src/slic3r/CMakeLists.txt

@@ -909,17 +909,19 @@ endif ()
 
 if (APPLE)
     # Static FFmpeg from the deps install: nothing to bundle into the .app,
-    # no rpath/install_name handling. Order matters: avcodec -> swscale -> avutil.
+    # no rpath/install_name handling. Order matters: avformat -> avcodec -> swscale -> avutil.
+    find_library(LIBAVFORMAT_LIBRARY NAMES libavformat.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH)
     find_library(LIBAVCODEC_LIBRARY NAMES libavcodec.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH)
     find_library(LIBSWSCALE_LIBRARY NAMES libswscale.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH)
     find_library(LIBAVUTIL_LIBRARY NAMES libavutil.a PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH)
-    if (NOT LIBAVCODEC_LIBRARY OR NOT LIBSWSCALE_LIBRARY OR NOT LIBAVUTIL_LIBRARY)
+    if (NOT LIBAVFORMAT_LIBRARY OR NOT LIBAVCODEC_LIBRARY OR NOT LIBSWSCALE_LIBRARY OR NOT LIBAVUTIL_LIBRARY)
-        message(FATAL_ERROR "Static FFmpeg (libavcodec.a/libswscale.a/libavutil.a) not found under ${CMAKE_PREFIX_PATH}/lib. Rebuild the deps — FFMPEG builds static-only on macOS.")
+        message(FATAL_ERROR "Static FFmpeg (libavformat.a/libavcodec.a/libswscale.a/libavutil.a) not found under ${CMAKE_PREFIX_PATH}/lib. Rebuild the deps — FFMPEG builds static-only on macOS.")
     endif ()
-    target_link_libraries(libslic3r_gui ${LIBAVCODEC_LIBRARY} ${LIBSWSCALE_LIBRARY} ${LIBAVUTIL_LIBRARY})
+    target_link_libraries(libslic3r_gui ${LIBAVFORMAT_LIBRARY} ${LIBAVCODEC_LIBRARY} ${LIBSWSCALE_LIBRARY} ${LIBAVUTIL_LIBRARY})
     target_include_directories(libslic3r_gui SYSTEM PRIVATE ${CMAKE_PREFIX_PATH}/include)
 else ()
     pkg_check_modules(LIBAV REQUIRED IMPORTED_TARGET
+        libavformat
         libavcodec
         libswscale
         libavutil

src/slic3r/GUI/AVVideoDecoder.cpp

@@ -45,6 +45,25 @@ int AVVideoDecoder::open(Bambu_StreamInfo const &info)
     return 0;
 }
 
+int AVVideoDecoder::open(AVCodecParameters const &parameters)
+{
+    if (avcodec_parameters_to_context(codec_ctx_, &parameters) < 0)
+        return -1;
+
+    auto codec = avcodec_find_decoder(codec_ctx_->codec_id);
+    if (codec == nullptr) {
+        fprintf(stderr, "AVVideoDecoder: unsupported codec!\n");
+        return -1;
+    }
+    if (avcodec_open2(codec_ctx_, codec, nullptr) < 0) {
+        fprintf(stderr, "AVVideoDecoder: could not open codec\n");
+        return -1;
+    }
+
+    frame_ = av_frame_alloc();
+    return frame_ == nullptr ? -1 : 0;
+}
+
 int AVVideoDecoder::decode(const Bambu_Sample &sample)
 {
     int ret = -1;
@@ -71,6 +90,14 @@ int AVVideoDecoder::decode(const Bambu_Sample &sample)
     return ret;
 }
 
+int AVVideoDecoder::decode(const AVPacket &packet)
+{
+    int ret = avcodec_send_packet(codec_ctx_, &packet);
+    if (ret == 0)
+        got_frame_ = avcodec_receive_frame(codec_ctx_, frame_) == 0;
+    return ret;
+}
+
 int AVVideoDecoder::flush()
 {
     int ret = avcodec_send_packet(codec_ctx_, nullptr);

src/slic3r/GUI/AVVideoDecoder.hpp

@@ -23,8 +23,10 @@ public:
 
 public:
     int  open(Bambu_StreamInfo const &info);
+    int  open(AVCodecParameters const &parameters);
 
     int  decode(Bambu_Sample const &sample);
+    int  decode(AVPacket const &packet);
 
     int  flush();
 

src/slic3r/GUI/wxMediaCtrl3.cpp

@@ -4,6 +4,9 @@
 #include "libslic3r/Utils.hpp"
 #include <boost/log/trivial.hpp>
 #include <wx/dcclient.h>
+extern "C" {
+#include <libavformat/avformat.h>
+}
 #ifdef __WIN32__
 #include <versionhelpers.h>
 #include <wx/msw/registry.h>
@@ -185,6 +188,106 @@ void wxMediaCtrl3::bambu_log(void *ctx, int level, tchar const *msg2)
     BOOST_LOG_TRIVIAL(info) << msg.ToUTF8().data();
 }
 
+int wxMediaCtrl3::rtsp_interrupt_callback(void *opaque)
+{
+    auto *ctrl = static_cast<wxMediaCtrl3 *>(opaque);
+    std::lock_guard<std::mutex> lock(ctrl->m_mutex);
+    return ctrl->m_url != ctrl->m_active_url;
+}
+
+int wxMediaCtrl3::PlayRtsp(std::shared_ptr<wxURI> const &url, std::unique_lock<std::mutex> &lock)
+{
+    if (avformat_network_init() < 0)
+        return 2;
+
+    AVFormatContext *format_context = avformat_alloc_context();
+    if (!format_context) {
+        avformat_network_deinit();
+        return 2;
+    }
+
+    format_context->interrupt_callback = {&wxMediaCtrl3::rtsp_interrupt_callback, this};
+    m_active_url = url;
+
+    auto finish = [&](int error) {
+        lock.unlock();
+        avformat_close_input(&format_context);
+        avformat_network_deinit();
+        lock.lock();
+        m_active_url.reset();
+        return error;
+    };
+
+    const std::string uri = url->BuildURI().ToUTF8().data();
+    AVDictionary *options = nullptr;
+    av_dict_set(&options, "rtsp_transport", "tcp", 0);
+    lock.unlock();
+    int error = avformat_open_input(&format_context, uri.c_str(), nullptr, &options);
+    av_dict_free(&options);
+    lock.lock();
+    if (error < 0)
+        return finish(2);
+
+    lock.unlock();
+    error = avformat_find_stream_info(format_context, nullptr);
+    lock.lock();
+    if (error < 0)
+        return finish(2);
+
+    const int video_stream = av_find_best_stream(format_context, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+    if (video_stream < 0)
+        return finish(2);
+
+    AVVideoDecoder decoder;
+    if (decoder.open(*format_context->streams[video_stream]->codecpar) < 0)
+        return finish(2);
+
+    m_video_size = {format_context->streams[video_stream]->codecpar->width,
+                    format_context->streams[video_stream]->codecpar->height};
+    if (!m_video_size.IsFullySpecified() || m_video_size.x <= 0 || m_video_size.y <= 0)
+        return finish(2);
+    adjust_frame_size(m_frame_size, m_video_size, GetSize());
+    NotifyStopped();
+
+    AVPacket *packet = av_packet_alloc();
+    if (!packet)
+        return finish(2);
+
+    while (m_url == url) {
+        lock.unlock();
+        error = av_read_frame(format_context, packet);
+        lock.lock();
+        if (m_url != url)
+            break;
+        if (error < 0)
+            break;
+        if (packet->stream_index == video_stream) {
+            const int decode_error = decoder.decode(*packet);
+            if (decode_error == 0) {
+                auto frame_size = m_frame_size;
+                lock.unlock();
+#ifdef _WIN32
+                wxBitmap frame;
+                decoder.toWxBitmap(frame, frame_size);
+#else
+                wxImage frame;
+                decoder.toWxImage(frame, frame_size);
+#endif
+                lock.lock();
+                if (m_url != url)
+                    break;
+                if (frame.IsOk())
+                    m_frame = frame;
+                CallAfter([this] { Refresh(); });
+            }
+        }
+        av_packet_unref(packet);
+    }
+
+    av_packet_free(&packet);
+    return finish(m_url == url ? 2 : 1);
+}
+
 void wxMediaCtrl3::PlayThread()
 {
     using namespace std::chrono_literals;
@@ -197,42 +300,21 @@ void wxMediaCtrl3::PlayThread()
             continue;
         if (!url->HasScheme())
             break;
-        lk.unlock();
+        const wxString scheme = url->GetScheme();
-        Bambu_Tunnel tunnel = nullptr;
+        const bool generic_rtsp = scheme.CmpNoCase("rtsp") == 0 || scheme.CmpNoCase("rtsps") == 0;
-        int error = Bambu_Create(&tunnel, m_url->BuildURI().ToUTF8());
+        int error = 0;
-        if (error == 0) {
+        if (generic_rtsp) {
-            Bambu_SetLogger(tunnel, &wxMediaCtrl3::bambu_log, this);
+            error = PlayRtsp(url, lk);
-            error = Bambu_Open(tunnel);
+        } else {
-            if (error == 0)
+            lk.unlock();
-                error = Bambu_would_block;
+            Bambu_Tunnel tunnel = nullptr;
-        }
+            error = Bambu_Create(&tunnel, m_url->BuildURI().ToUTF8());
-        lk.lock();
+            if (error == 0) {
-        while (error == int(Bambu_would_block)) {
+                Bambu_SetLogger(tunnel, &wxMediaCtrl3::bambu_log, this);
-            m_cond.wait_for(lk, 100ms);
+                error = Bambu_Open(tunnel);
-            if (m_url != url) {
+                if (error == 0)
-                error = 1;
+                    error = Bambu_would_block;
-                break;
             }
-            lk.unlock();
-            error = Bambu_StartStream(tunnel, true);
-            lk.lock();
-        }
-        Bambu_StreamInfo info;
-        if (error == 0)
-            error = Bambu_GetStreamInfo(tunnel, 0, &info);
-        AVVideoDecoder decoder;
-        int minFrameDuration = 0;
-        if (error == 0) {
-            decoder.open(info);
-            m_video_size = { info.format.video.width, info.format.video.height };
-            adjust_frame_size(m_frame_size, m_video_size, GetSize());
-            minFrameDuration = 800 / info.format.video.frame_rate; // 80%
-            NotifyStopped();
-        }
-        Bambu_Sample sample;
-        while (error == 0) {
-            lk.unlock();
-            error = Bambu_ReadSample(tunnel, &sample);
             lk.lock();
             while (error == int(Bambu_would_block)) {
                 m_cond.wait_for(lk, 100ms);
@@ -240,59 +322,87 @@ void wxMediaCtrl3::PlayThread()
                     error = 1;
                     break;
                 }
+                lk.unlock();
+                error = Bambu_StartStream(tunnel, true);
+                lk.lock();
+            }
+            Bambu_StreamInfo info;
+            if (error == 0)
+                error = Bambu_GetStreamInfo(tunnel, 0, &info);
+            AVVideoDecoder decoder;
+            int minFrameDuration = 0;
+            if (error == 0) {
+                decoder.open(info);
+                m_video_size = { info.format.video.width, info.format.video.height };
+                adjust_frame_size(m_frame_size, m_video_size, GetSize());
+                minFrameDuration = 800 / info.format.video.frame_rate; // 80%
+                NotifyStopped();
+            }
+            Bambu_Sample sample;
+            while (error == 0) {
                 lk.unlock();
                 error = Bambu_ReadSample(tunnel, &sample);
                 lk.lock();
-            }
+                while (error == int(Bambu_would_block)) {
-            if (error == 0) {
+                    m_cond.wait_for(lk, 100ms);
-                auto frame_size = m_frame_size;
+                    if (m_url != url) {
-                lk.unlock();
+                        error = 1;
-                decoder.decode(sample);
+                        break;
-#ifdef _WIN32
-                wxBitmap bm;
-                decoder.toWxBitmap(bm, frame_size);
-#else
-                wxImage bm;
-                decoder.toWxImage(bm, frame_size);
-#endif
-                lk.lock();
-                if (m_url != url) {
-                    error = 1;
-                    break;
-                }
-                if (bm.IsOk()) {
-                    auto now = std::chrono::system_clock::now();
-                    if (m_last_PTS && (sample.decode_time - m_last_PTS) < 30000000ULL) { // 3s
-                        auto next_PTS_expected = m_last_PTS_expected + std::chrono::milliseconds((sample.decode_time - m_last_PTS) / 10000ULL);
-                        // The frame is late, catch up a little
-                        auto next_PTS_practical = m_last_PTS_practical + std::chrono::milliseconds(minFrameDuration);
-                        auto next_PTS = std::max(next_PTS_expected, next_PTS_practical);
-                        if(now < next_PTS)
-                            std::this_thread::sleep_until(next_PTS);
-                        else
-                            next_PTS = now;
-                        //auto text = wxString::Format(L"wxMediaCtrl3 pts diff %ld\n", std::chrono::duration_cast<std::chrono::milliseconds>(next_PTS - next_PTS_expected).count());
-                        //OutputDebugString(text);
-                        m_last_PTS = sample.decode_time;
-                        m_last_PTS_expected = next_PTS_expected;
-                        m_last_PTS_practical = next_PTS;
-                    } else {
-                        // Resync
-                        m_last_PTS           = sample.decode_time;
-                        m_last_PTS_expected  = now;
-                        m_last_PTS_practical = now;
                     }
-                    m_frame = bm;
+                    lk.unlock();
+                    error = Bambu_ReadSample(tunnel, &sample);
+                    lk.lock();
+                }
+                if (error == 0) {
+                    auto frame_size = m_frame_size;
+                    lk.unlock();
+                    decoder.decode(sample);
+#ifdef _WIN32
+                    wxBitmap bm;
+                    decoder.toWxBitmap(bm, frame_size);
+#else
+                    wxImage bm;
+                    decoder.toWxImage(bm, frame_size);
+#endif
+                    lk.lock();
+                    if (m_url != url) {
+                        error = 1;
+                        break;
+                    }
+                    if (bm.IsOk()) {
+                        auto now = std::chrono::system_clock::now();
+                        if (m_last_PTS && (sample.decode_time - m_last_PTS) < 30000000ULL) { // 3s
+                            auto next_PTS_expected = m_last_PTS_expected + std::chrono::milliseconds((sample.decode_time - m_last_PTS) / 10000ULL);
+                            // The frame is late, catch up a little
+                            auto next_PTS_practical = m_last_PTS_practical + std::chrono::milliseconds(minFrameDuration);
+                            auto next_PTS = std::max(next_PTS_expected, next_PTS_practical);
+                            if(now < next_PTS)
+                                std::this_thread::sleep_until(next_PTS);
+                            else
+                                next_PTS = now;
+                            //auto text = wxString::Format(L"wxMediaCtrl3 pts diff %ld\n", std::chrono::duration_cast<std::chrono::milliseconds>(next_PTS - next_PTS_expected).count());
+                            //OutputDebugString(text);
+                            m_last_PTS = sample.decode_time;
+                            m_last_PTS_expected = next_PTS_expected;
+                            m_last_PTS_practical = next_PTS;
+                        } else {
+                            // Resync
+                            m_last_PTS           = sample.decode_time;
+                            m_last_PTS_expected  = now;
+                            m_last_PTS_practical = now;
+                        }
+                        m_frame = bm;
+                    }
+                    CallAfter([this] { Refresh(); });
                 }
-                CallAfter([this] { Refresh(); });
             }
-        }
+            if (tunnel) {
-        if (tunnel) {
+                lk.unlock();
-            lk.unlock();
+                Bambu_Close(tunnel);
-            Bambu_Close(tunnel);
+                Bambu_Destroy(tunnel);
-            Bambu_Destroy(tunnel);
+                tunnel = nullptr;
-            tunnel = nullptr;
+                lk.lock();
-            lk.lock();
+            }
         }
         if (m_url == url)
             m_error = error;

src/slic3r/GUI/wxMediaCtrl3.h

@@ -56,8 +56,10 @@ protected:
     void DoSetSize(int x, int y, int width, int height, int sizeFlags) override;
 
     static void bambu_log(void *ctx, int level, tchar const *msg);
+    static int rtsp_interrupt_callback(void *opaque);
 
     void PlayThread();
+    int PlayRtsp(std::shared_ptr<wxURI> const &url, std::unique_lock<std::mutex> &lock);
 
     void NotifyStopped();
 
@@ -74,6 +76,7 @@ private:
 #endif
 
     std::shared_ptr<wxURI> m_url;
+    std::shared_ptr<wxURI> m_active_url;
     std::uint64_t m_last_PTS{0};
     std::chrono::system_clock::time_point m_last_PTS_expected;
     std::chrono::system_clock::time_point m_last_PTS_practical;