Multistream support for variable frame duration

Also fixes a bug with stereo streams where the initial memory was only
using the left channel.

3 files changed, 95 insertions, 31 deletions

diff --git a/src/opus_encoder.c b/src/opus_encoder.c
index c4a46557..4c0840ff 100644
--- a/src/opus_encoder.c
+++ b/src/opus_encoder.c
@@ -665,19 +665,21 @@ static int transient_viterbi(const float *E, const float *E_1, int N, int frame_
    return best_state;
 }
 
-static int optimize_framesize(const opus_val16 *x, int len, int C, opus_int32 Fs,
+int optimize_framesize(const opus_val16 *x, int len, int C, opus_int32 Fs,
                 int bitrate, opus_val16 tonality, opus_val32 *mem, int buffering)
 {
    int N;
-   int i;
+   int i, c;
    float e[MAX_DYNAMIC_FRAMESIZE+4];
    float e_1[MAX_DYNAMIC_FRAMESIZE+3];
    float memx;
    int bestLM=0;
    int subframe;
    int pos;
+   VARDECL(opus_val16, sub);
 
    subframe = Fs/400;
+   ALLOC(sub, subframe, opus_val16);
    e[0]=mem[0];
    e_1[0]=1./(EPSILON+mem[0]);
    if (buffering)
@@ -698,27 +700,26 @@ static int optimize_framesize(const opus_val16 *x, int len, int C, opus_int32 Fs
    }
    N=IMIN(len/subframe, MAX_DYNAMIC_FRAMESIZE);
    memx = x[0];
+   for (c=1;c<C;c++)
+      memx += x[c];
    for (i=0;i<N;i++)
    {
       float tmp;
       float tmpx;
       int j;
       tmp=EPSILON;
-      if (C==1)
-      {
-         for (j=0;j<subframe;j++)
-         {
-            tmpx = x[subframe*i+j];
-            tmp += (tmpx-memx)*(tmpx-memx);
-            memx = tmpx;
-         }
-      } else {
+
+      for (j=0;j<subframe;j++)
+         sub[j] = x[(subframe*i+j)*C];
+      for (c=1;c<C;c++)
          for (j=0;j<subframe;j++)
-         {
-            tmpx = x[(subframe*i+j)*2]+x[(subframe*i+j)*2+1];
-            tmp += (tmpx-memx)*(tmpx-memx);
-            memx = tmpx;
-         }
+            sub[j] += x[(subframe*i+j)*C+c];
+
+      for (j=0;j<subframe;j++)
+      {
+         tmpx = sub[j];
+         tmp += (tmpx-memx)*(tmpx-memx);
+         memx = tmpx;
       }
       e[i+pos] = tmp;
       e_1[i+pos] = 1.f/tmp;
@@ -1425,7 +1426,7 @@ opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_
             {
                 opus_int32 bonus=0;
 #ifndef FIXED_POINT
-                if (orig_frame_size != frame_size)
+                if (st->variable_duration && orig_frame_size != frame_size)
                 {
                    bonus = (40*st->stream_channels+40)*(48000/frame_size-48000/orig_frame_size);
                    if (analysis_info.valid)
diff --git a/src/opus_multistream_encoder.c b/src/opus_multistream_encoder.c
index db9fc785..d048f535 100644
--- a/src/opus_multistream_encoder.c
+++ b/src/opus_multistream_encoder.c
@@ -40,6 +40,9 @@
 struct OpusMSEncoder {
    ChannelLayout layout;
    int bitrate;
+   int variable_duration;
+   opus_int32 bitrate_bps;
+   opus_val32 subframe_mem[3];
    /* Encoder states go here */
 };
 
@@ -193,10 +196,38 @@ static int opus_multistream_encode_native
    VARDECL(opus_val16, buf);
    unsigned char tmp_data[MS_FRAME_TMP];
    OpusRepacketizer rp;
+   int orig_frame_size;
+   int coded_channels;
+   opus_int32 channel_rate;
    ALLOC_STACK;
 
    ptr = (char*)st + align(sizeof(OpusMSEncoder));
    opus_encoder_ctl((OpusEncoder*)ptr, OPUS_GET_SAMPLE_RATE(&Fs));
+
+   if (400*frame_size < Fs)
+   {
+      RESTORE_STACK;
+      return OPUS_BAD_ARG;
+   }
+   orig_frame_size = IMIN(frame_size,Fs/50);
+   if (st->variable_duration)
+   {
+      int LM = 3;
+      int channels;
+      opus_int32 delay_compensation;
+
+      channels = st->layout.nb_streams + st->layout.nb_coupled_streams;
+      opus_encoder_ctl((OpusEncoder*)ptr, OPUS_GET_LOOKAHEAD(&delay_compensation));
+      delay_compensation -= Fs/400;
+#ifndef FIXED_POINT
+      LM = optimize_framesize(pcm, frame_size, channels, Fs, st->bitrate_bps,
+            0.f, st->subframe_mem, delay_compensation);
+#endif
+      while ((Fs/400<<LM)>frame_size)
+         LM--;
+      frame_size = (Fs/400<<LM);
+   }
+
    /* Validate frame_size before using it to allocate stack space.
       This mirrors the checks in opus_encode[_float](). */
    if (400*frame_size != Fs && 200*frame_size != Fs &&
@@ -215,6 +246,31 @@ static int opus_multistream_encode_native
       RESTORE_STACK;
       return OPUS_BUFFER_TOO_SMALL;
    }
+
+   /* Compute bitrate allocation between streams (this could be a lot better) */
+   coded_channels = st->layout.nb_streams + st->layout.nb_coupled_streams;
+   channel_rate = st->bitrate_bps / coded_channels;
+#ifndef FIXED_POINT
+   if (st->variable_duration && orig_frame_size != frame_size)
+   {
+      opus_int32 bonus;
+      bonus = 60*(48000/frame_size-48000/orig_frame_size);
+      channel_rate += bonus;
+   }
+#endif
+   ptr = (char*)st + align(sizeof(OpusMSEncoder));
+   for (s=0;s<st->layout.nb_streams;s++)
+   {
+      OpusEncoder *enc;
+      enc = (OpusEncoder*)ptr;
+      if (s < st->layout.nb_coupled_streams)
+         ptr += align(coupled_size);
+      else
+         ptr += align(mono_size);
+      opus_encoder_ctl(enc, OPUS_SET_BITRATE(channel_rate * (s < st->layout.nb_coupled_streams ? 2 : 1)));
+   }
+
+   ptr = (char*)st + align(sizeof(OpusMSEncoder));
    /* Counting ToC */
    tot_size = 0;
    for (s=0;s<st->layout.nb_streams;s++)
@@ -378,20 +434,8 @@ int opus_multistream_encoder_ctl(OpusMSEncoder *st, int request, ...)
    {
    case OPUS_SET_BITRATE_REQUEST:
    {
-      int chan, s;
       opus_int32 value = va_arg(ap, opus_int32);
-      chan = st->layout.nb_streams + st->layout.nb_coupled_streams;
-      value /= chan;
-      for (s=0;s<st->layout.nb_streams;s++)
-      {
-         OpusEncoder *enc;
-         enc = (OpusEncoder*)ptr;
-         if (s < st->layout.nb_coupled_streams)
-            ptr += align(coupled_size);
-         else
-            ptr += align(mono_size);
-         opus_encoder_ctl(enc, request, value * (s < st->layout.nb_coupled_streams ? 2 : 1));
-      }
+      st->bitrate_bps = value;
    }
    break;
    case OPUS_GET_BITRATE_REQUEST:
@@ -504,7 +548,21 @@ int opus_multistream_encoder_ctl(OpusMSEncoder *st, int request, ...)
       }
       *value = (OpusEncoder*)ptr;
    }
-      break;
+   break;
+   case OPUS_SET_EXPERT_VARIABLE_DURATION_REQUEST:
+   {
+       opus_int32 value = va_arg(ap, opus_int32);
+       if (value<0 || value>1)
+          goto bad_arg;
+       st->variable_duration = value;
+   }
+   break;
+   case OPUS_GET_EXPERT_VARIABLE_DURATION_REQUEST:
+   {
+       opus_int32 *value = va_arg(ap, opus_int32*);
+       *value = st->variable_duration;
+   }
+   break;
    default:
       ret = OPUS_UNIMPLEMENTED;
       break;
@@ -512,6 +570,9 @@ int opus_multistream_encoder_ctl(OpusMSEncoder *st, int request, ...)
 
    va_end(ap);
    return ret;
+bad_arg:
+   va_end(ap);
+   return OPUS_BAD_ARG;
 }
 
 void opus_multistream_encoder_destroy(OpusMSEncoder *st)
diff --git a/src/opus_private.h b/src/opus_private.h
index 977f4a25..94de0033 100644
--- a/src/opus_private.h
+++ b/src/opus_private.h
@@ -81,6 +81,8 @@ int get_mono_channel(const ChannelLayout *layout, int stream_id, int prev);
 #define OPUS_SET_FORCE_MODE_REQUEST    11002
 #define OPUS_SET_FORCE_MODE(x) OPUS_SET_FORCE_MODE_REQUEST, __opus_check_int(x)
 
+int optimize_framesize(const opus_val16 *x, int len, int C, opus_int32 Fs,
+                int bitrate, opus_val16 tonality, opus_val32 *mem, int buffering);
 
 int encode_size(int size, unsigned char *data);