Merge pull request #590 from shoamano83/bugfix/video-streaming-codec-specific-data

Bugfix/video streaming codec specific data

3 files changed, 256 insertions, 2 deletions

diff --git a/sdl_android/src/androidTest/java/com/smartdevicelink/test/encoder/EncoderUtilsTest.java b/sdl_android/src/androidTest/java/com/smartdevicelink/test/encoder/EncoderUtilsTest.java
new file mode 100644
index 000000000..083a5dd3f
--- /dev/null
+++ b/sdl_android/src/androidTest/java/com/smartdevicelink/test/encoder/EncoderUtilsTest.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2017, Xevo Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.smartdevicelink.test.encoder;
+
+import android.annotation.TargetApi;
+import android.media.MediaFormat;
+import android.os.Build;
+
+import com.smartdevicelink.encoder.EncoderUtils;
+
+import junit.framework.TestCase;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * This is a unit test class for the SmartDeviceLink library project class :
+ * {@link com.smartdevicelink.encoder.EncoderUtils}
+ */
+@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
+public class EncoderUtilsTest extends TestCase {
+    public void testGetCodecSpecificDataWithNull() {
+        byte[] result = EncoderUtils.getCodecSpecificData(null);
+        assertNull(result);
+    }
+
+    public void testGetCodecSpecificDataForAVC() {
+        // example of SPS NAL unit with 4-byte start code
+        byte[] sps = new byte[] {
+                0x00, 0x00, 0x00, 0x01,
+                0x67, 0x42, (byte)0xC0, 0x0A, (byte)0xA6, 0x11, 0x11, (byte)0xE8,
+                0x40, 0x00, 0x00, (byte)0xFA, 0x40, 0x00, 0x3A, (byte)0x98,
+                0x23, (byte)0xC4, (byte)0x89, (byte)0x84, 0x60
+        };
+        // example of PPS NAL unit with 4-byte start code
+        byte[] pps = new byte[] {
+                0x00, 0x00, 0x00, 0x01,
+                0x68, (byte)0xC8, 0x42, 0x0F, 0x13, 0x20
+        };
+
+        ByteBuffer spsByteBuffer = ByteBuffer.allocate(sps.length);
+        spsByteBuffer.put(sps);
+        spsByteBuffer.flip();
+
+        ByteBuffer ppsByteBuffer = ByteBuffer.allocate(pps.length);
+        ppsByteBuffer.put(pps);
+        ppsByteBuffer.flip();
+
+        MediaFormat format = MediaFormat.createVideoFormat("video/avc", 16, 16);
+        format.setByteBuffer("csd-0", spsByteBuffer);
+        format.setByteBuffer("csd-1", ppsByteBuffer);
+
+        byte[] result = EncoderUtils.getCodecSpecificData(format);
+        assertNotNull(result);
+
+        byte[] expected = new byte[sps.length + pps.length];
+        System.arraycopy(sps, 0, expected, 0, sps.length);
+        System.arraycopy(pps, 0, expected, sps.length, pps.length);
+        assertTrue("Output codec specific data doesn't match", Arrays.equals(expected, result));
+    }
+
+    public void testGetCodecSpecificDataWithInvalidAVCData() {
+        // testing an error case when the encoder emits SPS only (which should not happen)
+        byte[] sps = new byte[] {
+                0x00, 0x00, 0x00, 0x01,
+                0x67, 0x42, (byte)0xC0, 0x0A, (byte)0xA6, 0x11, 0x11, (byte)0xE8,
+                0x40, 0x00, 0x00, (byte)0xFA, 0x40, 0x00, 0x3A, (byte)0x98,
+                0x23, (byte)0xC4, (byte)0x89, (byte)0x84, 0x60
+        };
+
+        ByteBuffer spsByteBuffer = ByteBuffer.allocate(sps.length);
+        spsByteBuffer.put(sps);
+        spsByteBuffer.flip();
+
+        MediaFormat format = MediaFormat.createVideoFormat("video/avc", 16, 16);
+        format.setByteBuffer("csd-0", spsByteBuffer);
+        // no PPS
+
+        byte[] result = EncoderUtils.getCodecSpecificData(format);
+        assertNull(result);
+    }
+
+    public void testGetCodecSpecificDataForUnknownCodec() {
+        MediaFormat format = MediaFormat.createVideoFormat("video/raw", 16, 16);
+        byte[] result = EncoderUtils.getCodecSpecificData(format);
+        assertNull("For unsupported codec, getCodecSpecificData should return null", result);
+    }
+}
diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java
new file mode 100644
index 000000000..362564da8
--- /dev/null
+++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2017, Xevo Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.smartdevicelink.encoder;
+
+import android.annotation.TargetApi;
+import android.media.MediaFormat;
+import android.os.Build;
+import android.util.Log;
+
+import java.nio.ByteBuffer;
+
+@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
+public final class EncoderUtils {
+    private final static String TAG = "EncoderUtils";
+
+    /**
+     * Extracts codec-specific data from MediaFormat instance
+     *
+     * Currently, only AVC is supported.
+     *
+     * @param format MediaFormat instance retrieved from MediaCodec
+     * @return byte array containing codec-specific data, or null if an error occurred
+     */
+    public static byte[] getCodecSpecificData(MediaFormat format) {
+        if (format == null) {
+            return null;
+        }
+
+        String name = format.getString(MediaFormat.KEY_MIME);
+        if (name == null) {
+            return null;
+        }
+
+        // same as MediaFormat.MIMETYPE_VIDEO_AVC but it requires API level 21
+        if (name.equals("video/avc")) {
+            return getAVCCodecSpecificData(format);
+        } else {
+            Log.w(TAG, "Retrieving codec-specific data for " + name + " is not supported");
+            return null;
+        }
+    }
+
+    /**
+     * Extracts H.264 codec-specific data (SPS and PPS) from MediaFormat instance
+     *
+     * The codec-specific data is in byte-stream format; 4-byte start codes (0x00 0x00 0x00 0x01)
+     * are added in front of SPS and PPS NAL units.
+     *
+     * @param format MediaFormat instance retrieved from MediaCodec
+     * @return byte array containing codec-specific data, or null if an error occurred
+     */
+    private static byte[] getAVCCodecSpecificData(MediaFormat format) {
+        // For H.264, "csd-0" contains SPS and "csd-1" contains PPS. Refer to the documentation
+        // of MediaCodec.
+        if (!(format.containsKey("csd-0") && format.containsKey("csd-1"))) {
+            Log.w(TAG, "H264 codec specific data not found");
+            return null;
+        }
+
+        ByteBuffer sps = format.getByteBuffer("csd-0");
+        int spsLen = sps.remaining();
+        ByteBuffer pps = format.getByteBuffer("csd-1");
+        int ppsLen = pps.remaining();
+
+        byte[] output = new byte[spsLen + ppsLen];
+        try {
+            sps.get(output, 0, spsLen);
+            pps.get(output, spsLen, ppsLen);
+        } catch (Exception e) {
+            // should not happen
+            Log.w(TAG, "Error while copying H264 codec specific data: " + e);
+            return null;
+        }
+
+        return output;
+    }
+
+    private EncoderUtils() {}
+}
diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
index dccabff0b..196f5d98c 100644
--- a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
+++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
@@ -9,13 +9,16 @@ import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
 import android.os.Build;
+import android.util.Log;
 import android.view.Surface;
 
 import com.smartdevicelink.proxy.interfaces.IVideoStreamListener;
 
 @TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR2)
 public class SdlEncoder {
-	
+
+	private static final String TAG = "SdlEncoder";
+
 	// parameters for the encoder
 	private static final String _MIME_TYPE = "video/avc"; // H.264/AVC video
 	// private static final String MIME_TYPE = "video/mp4v-es"; //MPEG4 video
@@ -32,7 +35,9 @@ public class SdlEncoder {
 	
 	// allocate one of these up front so we don't need to do it every time
 	private MediaCodec.BufferInfo mBufferInfo;
-	
+
+	// Codec-specific data (SPS and PPS)
+	private byte[] mH264CodecSpecificData = null;
 
 	public SdlEncoder () {
 	}
@@ -120,6 +125,7 @@ public class SdlEncoder {
 			}
 			mOutputStream = null;
 		}
+		mH264CodecSpecificData = null;
 	}
 
 	/**
@@ -153,8 +159,33 @@ public class SdlEncoder {
 				// not expected for an encoder
 				encoderOutputBuffers = mEncoder.getOutputBuffers();
 			} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+				if (mH264CodecSpecificData == null) {
+					MediaFormat format = mEncoder.getOutputFormat();
+					mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format);
+				} else {
+					Log.w(TAG, "Output format change notified more than once, ignoring.");
+				}
 			} else if (encoderStatus < 0) {
 			} else {
+				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+					// If we already retrieve codec specific data via OUTPUT_FORMAT_CHANGED event,
+					// we do not need this data.
+					if (mH264CodecSpecificData != null) {
+						mBufferInfo.size = 0;
+					} else {
+						Log.i(TAG, "H264 codec specific data not retrieved yet.");
+					}
+				}
+				// append SPS and PPS in front of every IDR NAL unit
+				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0
+						&& mBufferInfo.size != 0
+						&& mH264CodecSpecificData != null) {
+					try {
+						mOutputStream.write(mH264CodecSpecificData, 0,
+								mH264CodecSpecificData.length);
+					} catch (Exception e) {}
+				}
+
 				if (mBufferInfo.size != 0) {
 					byte[] dataToWrite = new byte[mBufferInfo.size];
 					encoderOutputBuffers[encoderStatus].get(dataToWrite,