マルチコーデックサイマルキャストの送信に対応する

.github/workflows/build.yml

@@ -29,8 +29,11 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           sudo apt-get update
-          sudo apt-get install -y libpulse-dev
+          sudo apt-get install -y libpulse-dev yasm clang-12
         if: matrix.m.linux == true
+      - run: |
+          brew install yasm
+        if: matrix.m.linux == false
       - run: python3 run.py --sumomo --package ${{ matrix.m.name }}
       - name: Get package name
         run: |

.vscode/c_cpp_properties.json

@@ -5,21 +5,28 @@
             "includePath": [
                 "${workspaceFolder}/include",
                 "${workspaceFolder}/_source/libdatachannel/include",
+                "${workspaceFolder}/_source/libdatachannel/include/rtc",
+                "${workspaceFolder}/_source/libdatachannel/src",
                 "${workspaceFolder}/_source/libdatachannel/deps/json/include",
                 "${workspaceFolder}/_source/libdatachannel/deps/plog/include",
+                "${workspaceFolder}/_source/libdatachannel/deps/libjuice/include",
+                "${workspaceFolder}/_source/libdatachannel/deps/usrsctp/usrsctplib",
+                "${workspaceFolder}/_source/libdatachannel/deps/libsrtp/include",
                 "${workspaceFolder}/_source/opus/include",
 
                 "${workspaceFolder}/_build/ubuntu-20.04_x86_64/release/sorac",
                 "${workspaceFolder}/_build/ubuntu-20.04_x86_64/release/sorac/proto/sorac",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/mbedtls/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/openh264/include",
+                "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/aom/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/libjpeg-turbo/include",
                 "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/libyuv/include",
 
                 "${workspaceFolder}/_build/macos_arm64/release/sorac",
                 "${workspaceFolder}/_build/macos_arm64/release/sorac/proto/sorac",
                 "${workspaceFolder}/_install/macos_arm64/release/mbedtls/include",
                 "${workspaceFolder}/_install/macos_arm64/release/openh264/include",
+                "${workspaceFolder}/_install/macos_arm64/release/aom/include",
                 "${workspaceFolder}/_install/macos_arm64/release/libjpeg-turbo/include",
                 "${workspaceFolder}/_install/macos_arm64/release/libyuv/include"
             ],

.vscode/settings.json

@@ -103,6 +103,8 @@
         "recorder.h": "c",
         "__mutex_base": "cpp",
         "sorac.h": "c",
-        "soracp.json.c.h": "c"
+        "soracp.json.c.h": "c",
+        "complex": "cpp",
+        "cfenv": "cpp"
     }
 }

CHANGES.md

@@ -11,6 +11,13 @@
 
 ## develop
 
+- [ADD] マルチコーデックサイマルキャストの送信に対応する
+  - @melpon
+- [ADD] libaom を追加して AV1 エンコードできるようにする
+  - @melpon
+- [UPDATE] protoc-gen-jsonif をアップデートして optional 対応する
+  - @melpon
+
 ## 2024.1.0
 
 **祝いリリース**

CMakeLists.txt

@@ -92,8 +92,11 @@ add_custom_command(
 target_sources(sorac
   PRIVATE
     "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.cpp"
+    src/aom_av1_video_encoder.cpp
     src/current_time.cpp
     src/data_channel.cpp
+    src/default_encoder_adapter.cpp
+    src/h264_profile_level_id.cpp
     src/open_h264_video_encoder.cpp
     src/opus_audio_encoder.cpp
     src/signaling.cpp
@@ -112,9 +115,12 @@ target_sources(sorac
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.h"
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.h"
       "${CMAKE_CURRENT_BINARY_DIR}/proto/sorac/soracp.json.c.hpp"
+      include/sorac/aom_av1_video_encoder.hpp
       include/sorac/bitrate.hpp
       include/sorac/current_time.hpp
       include/sorac/data_channel.hpp
+      include/sorac/default_encoder_adapter.hpp
+      include/sorac/h264_profile_level_id.hpp
       include/sorac/open_h264_video_encoder.hpp
       include/sorac/opus_audio_encoder.hpp
       include/sorac/signaling.hpp
@@ -132,6 +138,9 @@ target_compile_definitions(sorac PRIVATE JSONIF_USE_NLOHMANN_JSON)
 # OpenH264
 target_include_directories(sorac PRIVATE ${OPENH264_ROOT_DIR}/include)
 
+# AOM
+target_include_directories(sorac PRIVATE ${AOM_ROOT_DIR}/include)
+
 set_target_properties(sorac PROPERTIES CXX_STANDARD 20 C_STANDARD 20)
 
 set(_LIBS

NOTICE.md

@@ -333,3 +333,68 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ```
+
+## AOM
+
+https://aomedia.googlesource.com/aom/
+
+```
+Copyright (c) 2016, Alliance for Open Media. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in
+   the documentation and/or other materials provided with the
+   distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+```
+
+## WebRTC
+
+https://webrtc.googlesource.com/src/
+
+```
+Copyright (c) 2011, The WebRTC project authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+  * Neither the name of Google nor the names of its contributors may
+    be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```

VERSION

@@ -1,8 +1,10 @@
 SORA_C_SDK_VERSION=2024.1.0
-LIBDATACHANNEL_VERSION=v0.20.1
+LIBDATACHANNEL_URL=https://github.com/melpon/libdatachannel.git
+LIBDATACHANNEL_VERSION=005820eb54585b57fe5567a7ad51b801598c159f
 OPUS_VERSION=v1.4
 CMAKE_VERSION=3.28.1
 OPENH264_VERSION=v2.4.0
+AOM_VERSION=v3.8.1
 MBEDTLS_VERSION=v3.5.1
 PROTOBUF_VERSION=21.1
 PROTOC_GEN_JSONIF_VERSION=0.12.1

doc/FAQ.md

@@ -80,6 +80,26 @@ libwebrtc ベースの [Sora C++ SDK](https://github.com/shiguredo/sora-cpp-sdk)
     --cacert cacert.pem
 ```
 
+### マルチコーデックサイマルキャスト
+
+マルチコーデックサイマルキャストを利用した配信を macOS arm64 で利用する例です。
+
+```bash
+./sumomo --signaling-url wss://sora.example.com/signaling \
+  --channel-id sora \
+  --simulcast true \
+  --simulcast-multicodec true \
+  --audio=true \
+  --audio-type=macos \
+  --capture-type mac \
+  --capture-device-name=OBS \
+  --video-codec-type=AV1 \
+  --video-bit-rate 5000 \
+  --h264-encoder-type videotoolbox \
+  --aom `pwd`/_install/macos_arm64/release/aom/lib/libaom.dylib
+~/shiguredo/sora-c-sdk
+```
+
 ## Sumomo のヘルプ
 
 ```bash

examples/sumomo/fake_capturer.cpp

@@ -13,7 +13,8 @@ namespace sumomo {
 
 class FakeCapturer : public SumomoCapturer {
  public:
-  FakeCapturer() {
+  FakeCapturer(int width, int height, int fps, SumomoFakeCapturerFormat format)
+      : width_(width), height_(height), fps_(fps), format_(format) {
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::FakeCapturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -42,26 +43,39 @@ class FakeCapturer : public SumomoCapturer {
       engine_ = std::make_unique<std::mt19937>(seed_gen());
       return nullptr;
     });
-    th_.Start(30, [this](std::chrono::microseconds timestamp,
-                         std::chrono::microseconds prev) {
-      std::uniform_int_distribution<int> dist(0, 640 * 480 - 1);
+    th_.Start(fps_, [this](std::chrono::microseconds timestamp,
+                           std::chrono::microseconds prev) {
+      std::uniform_int_distribution<int> dist(0, width_ * height_ - 1);
       sorac::VideoFrame frame;
       frame.timestamp = timestamp;
-      frame.i420_buffer = sorac::VideoFrameBufferI420::Create(640, 480);
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.base_width = 640;
-      frame.base_height = 480;
+      frame.frame_number = ++frame_number_;
+      if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_I420) {
+        frame.i420_buffer =
+            sorac::VideoFrameBufferI420::Create(width_, height_);
+        for (int i = 0; i < width_ / 10; i++) {
+          frame.i420_buffer->y[dist(*engine_)] = 0xff;
+        }
+      } else if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_NV12) {
+        frame.nv12_buffer =
+            sorac::VideoFrameBufferNV12::Create(width_, height_);
+        for (int i = 0; i < width_ / 10; i++) {
+          frame.nv12_buffer->y[dist(*engine_)] = 0xff;
+        }
+      }
+      frame.base_width = width_;
+      frame.base_height = height_;
       callback_(frame);
     });
     return 0;
   }
   void Stop() { th_.Stop(); }
 
  private:
+  int width_;
+  int height_;
+  int fps_;
+  int frame_number_ = 0;
+  SumomoFakeCapturerFormat format_;
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   SteadyFrameThread th_;
   std::unique_ptr<std::mt19937> engine_;
@@ -71,7 +85,10 @@ class FakeCapturer : public SumomoCapturer {
 
 extern "C" {
 
-SumomoCapturer* sumomo_fake_capturer_create() {
-  return new sumomo::FakeCapturer();
+SumomoCapturer* sumomo_fake_capturer_create(int width,
+                                            int height,
+                                            int fps,
+                                            SumomoFakeCapturerFormat format) {
+  return new sumomo::FakeCapturer(width, height, fps, format);
 }
 }

examples/sumomo/fake_capturer.h

@@ -9,7 +9,16 @@
 extern "C" {
 #endif
 
-extern SumomoCapturer* sumomo_fake_capturer_create();
+typedef enum {
+  SUMOMO_FAKE_CAPTURER_FORMAT_I420 = 0,
+  SUMOMO_FAKE_CAPTURER_FORMAT_NV12 = 1,
+} SumomoFakeCapturerFormat;
+
+extern SumomoCapturer* sumomo_fake_capturer_create(
+    int width,
+    int height,
+    int fps,
+    SumomoFakeCapturerFormat format);
 
 #ifdef __cplusplus
 }

examples/sumomo/mac_capturer.h

@@ -11,7 +11,8 @@ extern "C" {
 
 extern SumomoCapturer* sumomo_mac_capturer_create(const char* device,
                                                   int width,
-                                                  int height);
+                                                  int height,
+                                                  int fps);
 
 #ifdef __cplusplus
 }

examples/sumomo/mac_capturer.mm

@@ -48,6 +48,7 @@ @implementation SumomoMacCapturer {
   std::function<void(const sorac::VideoFrame&)> _callback;
   BOOL _willBeRunning;
   dispatch_queue_t _frameQueue;
+  int _frameNumber;
 }
 
 - (instancetype)initWithCallback:
@@ -64,6 +65,7 @@ - (instancetype)initWithCallback:
     _videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
     _willBeRunning = NO;
     _frameQueue = nil;
+    _frameNumber = 0;
 
     NSSet<NSNumber*>* supportedPixelFormats = [NSSet
         setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
@@ -283,6 +285,7 @@ - (void)captureOutput:(AVCaptureOutput*)captureOutput
       (int64_t)(CMTimeGetSeconds(
                     CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
                 kMicrosecondsPerSecond));
+  frame.frame_number = ++_frameNumber;
   frame.nv12_buffer = sorac::VideoFrameBufferNV12::Create(width, height);
   frame.base_width = width;
   frame.base_height = height;
@@ -368,10 +371,11 @@ - (dispatch_queue_t)frameQueue {
 
 class MacCapturer : public SumomoCapturer {
  public:
-  MacCapturer(const char* device, int width, int height) {
+  MacCapturer(const char* device, int width, int height, int fps) {
     this->device_ = device;
     this->width_ = width;
     this->height_ = height;
+    this->fps_ = fps;
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::MacCapturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -385,7 +389,7 @@ - (dispatch_queue_t)frameQueue {
     };
     this->start = [](SumomoCapturer* p) {
       auto q = (sumomo::MacCapturer*)p;
-      return q->Start(q->device_.c_str(), q->width_, q->height_);
+      return q->Start(q->device_.c_str(), q->width_, q->height_, q->fps_);
     };
     this->stop = [](SumomoCapturer* p) { ((sumomo::MacCapturer*)p)->Stop(); };
   }
@@ -396,14 +400,14 @@ void SetFrameCallback(
     callback_ = callback;
   }
 
-  int Start(const char* device, int width, int height) {
+  int Start(const char* device, int width, int height, int fps) {
     Stop();
 
     capturer_ = [[SumomoMacCapturer alloc] initWithCallback:callback_];
     [capturer_ startCaptureWithDeviceName:device
                                     width:width
                                    height:height
-                                      fps:30
+                                      fps:fps
                         completionHandler:[](NSError* _Nullable error) {
                           if (error) {
                             fprintf(stderr, "Failed to start capture: %s\n",
@@ -422,6 +426,7 @@ void Stop() {
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   int width_;
   int height_;
+  int fps_;
 
   SumomoMacCapturer* capturer_;
 };
@@ -432,7 +437,8 @@ void Stop() {
 
 SumomoCapturer* sumomo_mac_capturer_create(const char* device,
                                            int width,
-                                           int height) {
-  return new sumomo::MacCapturer(device, width, height);
+                                           int height,
+                                           int fps) {
+  return new sumomo::MacCapturer(device, width, height, fps);
 }
 }