ngx-rtmp-kmp: sync audio frames timestamps

to compensate for loss of precision due to RTMP timescale, causing
ticking noises in Apple players

nginx-kmp-out-module/src/ngx_kmp_out_track.c

@@ -124,6 +124,7 @@ ngx_kmp_out_track_init_conf(ngx_kmp_out_track_conf_t *conf)
     conf->buffer_bin_count = NGX_CONF_UNSET_UINT;
     conf->mem_high_watermark = NGX_CONF_UNSET_UINT;
     conf->mem_low_watermark = NGX_CONF_UNSET_UINT;
+    conf->audio_sync_margin = NGX_CONF_UNSET_MSEC;
     conf->flush_timeout = NGX_CONF_UNSET_MSEC;
     conf->keepalive_interval = NGX_CONF_UNSET_MSEC;
     conf->log_frames = NGX_CONF_UNSET_UINT;
@@ -198,6 +199,9 @@ ngx_kmp_out_track_merge_conf(ngx_conf_t *cf, ngx_kmp_out_track_conf_t *conf,
             ngx_kmp_out_track_default_mem_limit[media_type]);
     }
 
+    ngx_conf_merge_msec_value(conf->audio_sync_margin,
+                              prev->audio_sync_margin, 2);
+
     ngx_conf_merge_msec_value(conf->flush_timeout, prev->flush_timeout, 1000);
 
     ngx_conf_merge_msec_value(conf->keepalive_interval,

nginx-kmp-out-module/src/ngx_kmp_out_track.h

@@ -30,6 +30,7 @@ typedef struct {
     size_t           buffer_size[KMP_MEDIA_COUNT];
     size_t           mem_limit[KMP_MEDIA_COUNT];
     ngx_lba_t       *lba[KMP_MEDIA_COUNT];
+    ngx_msec_t       audio_sync_margin;
 
     ngx_msec_t       flush_timeout;
     ngx_msec_t       keepalive_interval;

nginx-rtmp-kmp-module/README.md

@@ -242,6 +242,16 @@ A large value can be more efficient, but increases the latency (a buffer is sent
 Sets the maximum total size of the buffers used to send audio data to the upstream server.
 If the limit is hit, the module drops the RTMP connection.
 
+#### kmp_audio_sync_margin
+* **syntax**: `kmp_audio_sync_margin msec;`
+* **default**: `2ms`
+* **context**: `rtmp`, `server`, `application`
+
+Sets the maximum correction value applied to the timestamps of audio frames.
+In order to overcome loss of precision in audio timestamps (RTMP uses millis timescale),
+the module extrapolates the audio timestamps using the actual duration of the audio frames.
+Frames that have a timestamp within kmp_audio_sync_margin from the extrapolated value, will use the extrapolated value.
+
 #### kmp_flush_timeout
 * **syntax**: `kmp_flush_timeout msec;`
 * **default**: `1s`

nginx-rtmp-kmp-module/src/ngx_rtmp_kmp_module.c

@@ -214,6 +214,13 @@ static ngx_command_t  ngx_rtmp_kmp_commands[] = {
       offsetof(ngx_rtmp_kmp_app_conf_t, t.mem_limit[KMP_MEDIA_AUDIO]),
       NULL },
 
+    { ngx_string("kmp_audio_sync_margin"),
+      NGX_RTMP_MAIN_CONF|NGX_RTMP_SRV_CONF|NGX_RTMP_APP_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_msec_slot,
+      NGX_RTMP_APP_CONF_OFFSET,
+      offsetof(ngx_rtmp_kmp_app_conf_t, t.audio_sync_margin),
+      NULL },
+
     { ngx_string("kmp_flush_timeout"),
       NGX_RTMP_MAIN_CONF|NGX_RTMP_SRV_CONF|NGX_RTMP_APP_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_msec_slot,

nginx-rtmp-kmp-module/src/ngx_rtmp_kmp_track.c

@@ -31,6 +31,8 @@ typedef struct {
     ngx_rtmp_session_t  *s;
     int64_t              timestamp;
     int32_t              last_timestamp;
+    int64_t              audio_base_dts;
+    int64_t              audio_samples;
     unsigned             timestamps_synced:1;
     unsigned             media_info_sent:1;
     unsigned             published:1;
@@ -212,6 +214,48 @@ ngx_rtmp_kmp_track_send_media_info(ngx_kmp_out_track_t *track,
 }
 
 
+static void
+ngx_rtmp_kmp_track_sync_audio(ngx_kmp_out_track_t *track,
+    kmp_frame_packet_t *frame)
+{
+    int64_t                    est_dts;
+    uint32_t                   margin;
+    uint32_t                   timescale;
+    uint32_t                   sample_rate;
+    ngx_rtmp_kmp_track_ctx_t  *ctx;
+
+    if (track->media_info.codec_id != KMP_CODEC_AUDIO_AAC) {
+        return;
+    }
+
+    sample_rate = track->media_info.u.audio.sample_rate;
+    if (sample_rate == 0) {
+        return;
+    }
+
+    timescale = track->media_info.timescale;
+    margin = track->conf->audio_sync_margin * timescale / NGX_RTMP_TIMESCALE;
+
+    ctx = track->ctx;
+    est_dts = ctx->audio_base_dts + ctx->audio_samples
+        * timescale / sample_rate;
+
+    if (frame->f.dts >= est_dts - margin
+        && frame->f.dts <= est_dts + margin)
+    {
+        frame->f.dts = est_dts;
+
+    } else {
+        ctx->audio_base_dts = frame->f.dts;
+        ctx->audio_samples = 0;
+    }
+
+    /* TODO: identify short frames (960) from extra data */
+
+    ctx->audio_samples += 1024;
+}
+
+
 static ngx_int_t
 ngx_rtmp_kmp_track_init_frame(ngx_kmp_out_track_t *track,
     kmp_frame_packet_t *frame, ngx_rtmp_header_t *h, ngx_chain_t **in,
@@ -390,6 +434,10 @@ ngx_rtmp_kmp_track_init_frame(ngx_kmp_out_track_t *track,
     track->stats.last_timestamp = ctx->timestamp;
     frame->f.dts = ctx->timestamp * rtmpscale;
 
+    if (h->type == NGX_RTMP_MSG_AUDIO) {
+        ngx_rtmp_kmp_track_sync_audio(track, frame);
+    }
+
     return NGX_OK;
 }
 
@@ -498,7 +546,7 @@ ngx_rtmp_kmp_track_av(ngx_kmp_out_track_t *track, ngx_rtmp_header_t *h,
 
 ngx_kmp_out_track_t *
 ngx_rtmp_kmp_track_create(ngx_kmp_out_track_conf_t *conf,
-    ngx_rtmp_session_t *s, ngx_rtmp_kmp_publish_t  *publish,
+    ngx_rtmp_session_t *s, ngx_rtmp_kmp_publish_t *publish,
     ngx_rtmp_header_t *h, ngx_chain_t *in)
 {
     u_char                    *p;