Add NTP time ms to OnVideoFrameReceived.

Author: mamaheux

examples/cpp-camera-stream-client/main.cpp

@@ -240,7 +240,7 @@ int main(int argc, char* argv[])
             cout << "\tid=" << client.id() << ", name=" << client.name() << endl;
         });
     client.setOnVideoFrameReceived(
-        [](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs)
+        [](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs, uint64_t ntpTimeMs)
         {
             // This callback is called from a WebRTC processing thread.
             cv::imshow(client.id(), bgrImg);

examples/cpp-video-stream-client/main.cpp

@@ -118,7 +118,8 @@ class SinAudioSource : public AudioSource
 
             auto start = chrono::steady_clock::now();
             this_thread::sleep_for(SinAudioSourceFrameDuration - SinAudioSourceSleepBuffer);
-            while ((chrono::steady_clock::now() - start) < SinAudioSourceFrameDuration);
+            while ((chrono::steady_clock::now() - start) < SinAudioSourceFrameDuration)
+                ;
         }
     }
 };
@@ -232,7 +233,7 @@ int main(int argc, char* argv[])
             cout << "\tid=" << client.id() << ", name=" << client.name() << endl;
         });
     client.setOnVideoFrameReceived(
-        [](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs)
+        [](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs, uint64_t ntpTimeMs)
         {
             // This callback is called from a WebRTC processing thread.
             cout << "OnVideoFrameReceived:" << endl;

examples/python-stream-client/python_stream_client.py

@@ -67,7 +67,7 @@ def on_remove_remote_stream(client):
         client.id, client.name, client.data))
 
 
-def on_video_frame_received(client, image, timestampUs):
+def on_video_frame_received(client, image, timestamp_us, ntp_time_us):
     # This callback is called from a WebRTC processing thread.
     cv2.imshow(client.id, image)
     cv2.waitKey(1)

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/include/OpenteraWebrtcNativeClient/Handlers/StreamPeerConnectionHandler.h

@@ -10,7 +10,7 @@
 
 namespace opentera
 {
-    using VideoFrameReceivedCallback = std::function<void(const Client&, const cv::Mat&, uint64_t)>;
+    using VideoFrameReceivedCallback = std::function<void(const Client&, const cv::Mat&, uint64_t, uint64_t)>;
     using EncodedVideoFrameReceivedCallback = std::function<void(
         const Client& client,
         const uint8_t* data,

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/include/OpenteraWebrtcNativeClient/Sinks/VideoSink.h

@@ -9,7 +9,7 @@
 namespace opentera
 {
 
-    using VideoSinkCallback = std::function<void(const cv::Mat&, uint64_t)>;
+    using VideoSinkCallback = std::function<void(const cv::Mat&, uint64_t, uint64_t)>;
 
     /**
      * @brief Class that sinks frame from a webrtc stream
@@ -32,7 +32,10 @@ namespace opentera
      * @brief get frame requirements for this sink
      * @return frame requirements for this sink
      */
-    inline rtc::VideoSinkWants VideoSink::wants() const { return m_wants; }
+    inline rtc::VideoSinkWants VideoSink::wants() const
+    {
+        return m_wants;
+    }
 
 }
 

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/include/OpenteraWebrtcNativeClient/StreamClient.h

@@ -96,12 +96,18 @@ namespace opentera
     /**
      * @brief Mutes the local audio.
      */
-    inline void StreamClient::muteLocalAudio() { setLocalAudioMuted(true); }
+    inline void StreamClient::muteLocalAudio()
+    {
+        setLocalAudioMuted(true);
+    }
 
     /**
      * @brief Unmutes the local audio.
      */
-    inline void StreamClient::unmuteLocalAudio() { setLocalAudioMuted(false); }
+    inline void StreamClient::unmuteLocalAudio()
+    {
+        setLocalAudioMuted(false);
+    }
 
     /**
      * @brief Indicates if the remote audio is muted.
@@ -115,12 +121,18 @@ namespace opentera
     /**
      * @brief Mutes the remote audio.
      */
-    inline void StreamClient::muteRemoteAudio() { setRemoteAudioMuted(true); }
+    inline void StreamClient::muteRemoteAudio()
+    {
+        setRemoteAudioMuted(true);
+    }
 
     /**
      * @brief Unmutes the remote audio.
      */
-    inline void StreamClient::unmuteRemoteAudio() { setRemoteAudioMuted(false); }
+    inline void StreamClient::unmuteRemoteAudio()
+    {
+        setRemoteAudioMuted(false);
+    }
 
     /**
      * @brief Indicates if the local video is muted.
@@ -134,12 +146,18 @@ namespace opentera
     /**
      * @brief Mutes the local video.
      */
-    inline void StreamClient::muteLocalVideo() { setLocalVideoMuted(true); }
+    inline void StreamClient::muteLocalVideo()
+    {
+        setLocalVideoMuted(true);
+    }
 
     /**
      * @brief Unmutes the local video.
      */
-    inline void StreamClient::unmuteLocalVideo() { setLocalVideoMuted(false); }
+    inline void StreamClient::unmuteLocalVideo()
+    {
+        setLocalVideoMuted(false);
+    }
 
     /**
      * @brief Sets the callback that is called when a stream is added.
@@ -185,6 +203,7 @@ namespace opentera
      *  - client: The client of the stream frame
      *  - bgrImg: The BGR frame image
      *  - timestampUs The timestamp in microseconds
+     *  - ntpTimeMs The ntp time in milliseconds
      * @endparblock
      *
      * @param callback The callback

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/python/src/StreamClientPython.cpp

@@ -12,9 +12,9 @@ namespace py = pybind11;
 
 void setOnVideoFrameReceived(
     StreamClient& self,
-    const function<void(const Client&, const py::array_t<uint8_t>&, uint64_t)>& pythonCallback)
+    const function<void(const Client&, const py::array_t<uint8_t>&, uint64_t, uint64_t)>& pythonCallback)
 {
-    auto callback = [=](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs)
+    auto callback = [=](const Client& client, const cv::Mat& bgrImg, uint64_t timestampUs, uint64_t ntpTimeMs)
     {
         size_t height = bgrImg.rows;
         size_t width = bgrImg.cols;
@@ -32,7 +32,7 @@ void setOnVideoFrameReceived(
 
         py::gil_scoped_acquire acquire;
         py::array_t<uint8_t> numpyBgrImg(bufferInfo);
-        pythonCallback(client, numpyBgrImg, timestampUs);
+        pythonCallback(client, numpyBgrImg, timestampUs, ntpTimeMs);
     };
 
     self.setOnVideoFrameReceived(callback);
@@ -315,6 +315,7 @@ void opentera::initStreamClientPython(pybind11::module& m)
             " - client: The client of the stream frame\n"
             " - bgr_img: The BGR frame image (numpy.array[uint8])\n"
             " - timestamp_us The timestamp in microseconds\n"
+            " - ntp_time_ms The ntp time in milliseconds\n"
             "\n"
             ":param callback: The callback")
         .def_property(

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/python/test/stream_client_test.py

@@ -74,11 +74,11 @@ def on_add_remote_stream2(client):
         def on_remove_remote_stream(client):
             self.add_failure('on_remove_remote_stream')
 
-        def on_video_frame_received1(client, frame, timestamp):
+        def on_video_frame_received1(client, frame, timestamp_us, ntp_time_us):
             self._mean_color_1 = np.mean(frame, axis=(0, 1))
             on_video_frame_awaiter1.done()
 
-        def on_video_frame_received2(client, frame, timestamp):
+        def on_video_frame_received2(client, frame, timestamp_us, ntp_time_us):
             self._mean_color_2 = np.mean(frame, axis=(0, 1))
             on_video_frame_awaiter2.done()
 

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/src/Handlers/StreamPeerConnectionHandler.cpp

@@ -44,8 +44,8 @@ StreamPeerConnectionHandler::StreamPeerConnectionHandler(
 {
     if (onVideoFrameReceived)
     {
-        m_videoSink = make_unique<VideoSink>([=](const cv::Mat& bgrImg, uint64_t timestampUs)
-                                             { onVideoFrameReceived(m_peerClient, bgrImg, timestampUs); });
+        m_videoSink = make_unique<VideoSink>([=](const cv::Mat& bgrImg, uint64_t timestampUs, uint64_t ntpTimeMs)
+                                             { onVideoFrameReceived(m_peerClient, bgrImg, timestampUs, ntpTimeMs); });
     }
 
     if (onEncodedVideoFrameReceived)
@@ -57,7 +57,8 @@ StreamPeerConnectionHandler::StreamPeerConnectionHandler(
                 bool isKeyFrame,
                 uint32_t width,
                 uint32_t height,
-                uint64_t timestampUs) {
+                uint64_t timestampUs)
+            {
                 onEncodedVideoFrameReceived(
                     m_peerClient,
                     data,
@@ -77,7 +78,8 @@ StreamPeerConnectionHandler::StreamPeerConnectionHandler(
                 int bitsPerSample,
                 int sampleRate,
                 size_t numberOfChannels,
-                size_t numberOfFrames) {
+                size_t numberOfFrames)
+            {
                 onAudioFrameReceived(
                     m_peerClient,
                     audioData,

opentera-webrtc-native-client/OpenteraWebrtcNativeClient/src/Sinks/VideoSink.cpp

@@ -61,7 +61,7 @@ void VideoSink::OnFrame(const webrtc::VideoFrame& frame)
     switch (frame.rotation())
     {
         case webrtc::kVideoRotation_0:
-            m_onFrameReceived(m_bgrImg, frame.timestamp_us());
+            m_onFrameReceived(m_bgrImg, frame.timestamp_us(), frame.ntp_time_ms());
             return;
         case webrtc::kVideoRotation_90:
             cv::rotate(m_bgrImg, m_bgrRotatedImg, cv::ROTATE_90_CLOCKWISE);
@@ -73,5 +73,5 @@ void VideoSink::OnFrame(const webrtc::VideoFrame& frame)
             cv::rotate(m_bgrImg, m_bgrRotatedImg, cv::ROTATE_90_COUNTERCLOCKWISE);
             break;
     }
-    m_onFrameReceived(m_bgrRotatedImg, frame.timestamp_us());
+    m_onFrameReceived(m_bgrRotatedImg, frame.timestamp_us(), frame.ntp_time_ms());
 }