KVS - Add Support for More Resolution Widths (#3609)

* Add kvs producer audio, start fixing rotation issue (#1)

* MIcrophoneFrameSource builds, runs with "failed sampleRate 8000" exception

* Fix all runtime errors

* Started on 2-track streamInfo

* Construct TrackInfo, build errors fixed

* Progress, facing STATUS_MKV_INVALID_FRAME_DATA now if include audio with video

* Working, missing CPD error on playback though

* Works!! Todo: Should up the sample rate

* Begin modularizing code, not yet built nor test run

* Cleanup spaces, add to createFrameWithTrackID log

* Cleanup diffs

* Fix some importing and naming

* Rename AndroidAudioVideoMediaSource

* Add configuration class, control audio start/stop, working

* Saving attempted solutions to the issue

* Add TimeStampProvider to enforce unique timestamps

* Not cleaning junk, saving working camera 90deg rotation!

* Saving progress of fixing semi-planar case

* Fix convertByteArrayToPlanes, conversions now working perfect for semiplanar, rotation still not

* Rotation and colors work!

* Optimized rotation working, cleaned up comments.

* Turn off rotation, cleanup

* Revert EncoderFactory color format changes

* Revert "Revert EncoderFactory color format changes"

This reverts commit bfd9940.

* Revert "Add kvs producer audio, start fixing rotation issue (#1)"

This reverts commit 2355788.

* Initial fix-resolution commit

* Cleanup

* Add comments

* Apply suggestions from code review

Co-authored-by: sirknightj <sirknightj@gmail.com>
Co-authored-by: Matt Creaser <mattcreaser@gmail.com>

* Update aws-android-sdk-kinesisvideo/src/main/java/com/amazonaws/mobileconnectors/kinesisvideo/encoding/EncoderFrameSubmitter.java

Co-authored-by: sirknightj <sirknightj@gmail.com>

---------

Co-authored-by: sirknightj <sirknightj@gmail.com>
Co-authored-by: Matt Creaser <mattcreaser@gmail.com>
Co-authored-by: Matt Creaser <mattwcc@amazon.com>

Author: 3 people

aws-android-sdk-kinesisvideo/src/main/java/com/amazonaws/mobileconnectors/kinesisvideo/encoding/EncoderFrameSubmitter.java

@@ -137,10 +137,27 @@ private void copyCameraFrameIntoInputImage(final int inputBufferIndex,
                                                final Image cameraFrame) {
         final Image codecInputImage = mEncoder.getInputImage(inputBufferIndex);
 
+        final int imageWidth = cameraFrame.getWidth();
+        final int imageRowStride = cameraFrame.getPlanes()[0].getRowStride();
+        final int imageChromaPixelStride = cameraFrame.getPlanes()[1].getPixelStride();
+
+        final boolean isRowPaddingPresent = imageWidth != imageRowStride;
+        final boolean isSemiPlanar = imageChromaPixelStride != 1;
+
         for (int i = 0; i < cameraFrame.getPlanes().length; i++) {
             final ByteBuffer sourceImagePlane = cameraFrame.getPlanes()[i].getBuffer();
             final ByteBuffer destinationImagePlane = codecInputImage.getPlanes()[i].getBuffer();
-            copyBuffer(sourceImagePlane, destinationImagePlane);
+
+            final int pixelStride = codecInputImage.getPlanes()[i].getPixelStride();
+
+            // Accounting for horizontal padding that may have been added for camera optimization. For example, non-multiple-of-64-width
+            // resolutions may have had padding added to the rows to reach the nearest greater multiple of 64.
+            // This accounting for is only supported for SemiPlanar format at the moment.
+            if (isRowPaddingPresent && isSemiPlanar) {
+                copyPaddedBuffer(sourceImagePlane, destinationImagePlane, imageWidth, imageRowStride, pixelStride);
+            } else {
+                copyBuffer(sourceImagePlane, destinationImagePlane);
+            }
         }
     }
 
@@ -156,6 +173,58 @@ private int copyBuffer(final ByteBuffer sourceBuffer,
         return bytesToCopy;
     }
 
+    // This copy function will ignore the extra padding data when copying into the encoder's input buffer.
+    // This will only work for SemiPlanar format.
+    private int copyPaddedBuffer(final ByteBuffer sourceBuffer,
+                                 final ByteBuffer destinationBuffer,
+                                 final int imageWidth, 
+                                 final int sourceBufferRowStride, 
+                                 final int pixelStride) {
+        
+        // For an unknown reason, the sourceBuffer's remaining bytes are slightly less than expected, rounding up fixes this.
+        final int numRows = (int) Math.ceil((double) sourceBuffer.remaining() / sourceBufferRowStride);
+        
+        // Desired bytes might exceed destination buffer capacity.
+        final int desiredBytesToCopy = numRows * imageWidth;
+        final int bytesToCopy = Math.min(desiredBytesToCopy, destinationBuffer.capacity());
+        destinationBuffer.limit(bytesToCopy);
+
+        // If this is a chroma plane, then don't include the final row in the for loop - it is an edge case.
+        final int numRowsToIterate;
+        if (pixelStride == 2) {
+            numRowsToIterate = numRows - 1;
+        } else {
+            numRowsToIterate = numRows;
+        }
+        
+        for (int row = 0; row < numRowsToIterate; row++) {
+            final int startPos = row * sourceBufferRowStride;
+            final int endPos = startPos + imageWidth;
+
+            sourceBuffer.limit(endPos);
+            sourceBuffer.position(startPos);
+
+            // Put the valid row data into the destination buffer.
+            destinationBuffer.put(sourceBuffer);
+        }
+
+        // If this is a chroma frame, handle the final row which is missing one value, hence the "-1" to the endPos.
+        if (pixelStride == 2) {
+            final int startPos = numRowsToIterate * sourceBufferRowStride;
+            final int endPos = startPos + imageWidth - 1;
+
+            sourceBuffer.limit(endPos);
+            sourceBuffer.position(startPos);
+
+            // Put the valid row data into the destination buffer.
+            destinationBuffer.put(sourceBuffer);
+        }
+
+        destinationBuffer.rewind();
+
+        return bytesToCopy;
+    }
+
     private long nanosSinceFirstFrame() {
         final long currentTime = System.currentTimeMillis();
         if (mFirstFrameTimestamp < 0) {