Implement QAndroidImageCapture::captureToBuffer()

Implemented the captureToBuffer based on the current implementation.
Found that the image was not created on the right orientation. Fixed
that by correcting the orientation after the picture was taken.
Small refactor on cameraandroidsession because of different arguments.

Fixes: QTBUG-97587
Change-Id: Ieb0c07f3b397df789f3f43d033ced76c01e8f33f
Reviewed-by: Assam Boudjelthia <assam.boudjelthia@qt.io>
(cherry picked from commit 60f9c575c2e21f5dfc08386e537ea56d5759b826)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

6 files changed, 182 insertions, 67 deletions

diff --git a/src/android/jar/src/org/qtproject/qt/android/multimedia/QtCameraListener.java b/src/android/jar/src/org/qtproject/qt/android/multimedia/QtCameraListener.java
index ff26d90c3..a37544916 100644
--- a/src/android/jar/src/org/qtproject/qt/android/multimedia/QtCameraListener.java
+++ b/src/android/jar/src/org/qtproject/qt/android/multimedia/QtCameraListener.java
@@ -40,10 +40,20 @@
 package org.qtproject.qt.android.multimedia;
 
 import android.hardware.Camera;
+import android.hardware.Camera.CameraInfo;
+
 import android.graphics.ImageFormat;
 import android.graphics.SurfaceTexture;
 import android.util.Log;
 import java.lang.Math;
+import android.media.ExifInterface;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.lang.String;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Matrix;
 
 public class QtCameraListener implements Camera.ShutterCallback,
                                          Camera.PictureCallback,
@@ -198,6 +208,60 @@ public class QtCameraListener implements Camera.ShutterCallback,
     @Override
     public void onPictureTaken(byte[] data, Camera camera)
     {
+        try {
+            InputStream stream = new ByteArrayInputStream(data);
+
+            ExifInterface exif = new ExifInterface(stream);
+
+            int orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION,
+                                                   ExifInterface.ORIENTATION_UNDEFINED);
+
+            int degree = 0;
+
+            switch (orientation) {
+            case ExifInterface.ORIENTATION_ROTATE_90:
+                degree = 90;
+                break;
+            case ExifInterface.ORIENTATION_ROTATE_180:
+                degree = 180;
+                break;
+            case ExifInterface.ORIENTATION_ROTATE_270:
+                degree = 270;
+                break;
+            }
+
+            Camera.CameraInfo info = new Camera.CameraInfo();
+            Camera.getCameraInfo(m_cameraId, info);
+
+            int rotation = (info.orientation - degree + 360) % 360;
+
+            Bitmap source = BitmapFactory.decodeByteArray(data, 0, data.length);
+            Matrix matrix = new Matrix();
+            matrix.postRotate(rotation);
+
+            if (info.facing == CameraInfo.CAMERA_FACING_FRONT) {
+                matrix.postScale(-1, 1, source.getWidth() / 2.0f, source.getHeight() / 2.0f);
+            }
+
+            Bitmap rotatedBitmap = Bitmap.createBitmap(source, 0, 0, source.getWidth(),
+                                                       source.getHeight(), matrix, true);
+
+            ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+            rotatedBitmap.compress(Bitmap.CompressFormat.JPEG, 100, outputStream);
+            byte[] byteArray = outputStream.toByteArray();
+
+            rotatedBitmap.recycle();
+            source.recycle();
+
+            notifyPictureCaptured(m_cameraId, byteArray);
+
+            return;
+
+        } catch (Exception e) {
+            Log.w(TAG, "Error fixing bitmap orientation.");
+            e.printStackTrace();
+        }
+
         notifyPictureCaptured(m_cameraId, data);
     }
 
diff --git a/src/multimedia/platform/android/mediacapture/qandroidcamerasession.cpp b/src/multimedia/platform/android/mediacapture/qandroidcamerasession.cpp
index 8c8e2b121..258ac2ef6 100644
--- a/src/multimedia/platform/android/mediacapture/qandroidcamerasession.cpp
+++ b/src/multimedia/platform/android/mediacapture/qandroidcamerasession.cpp
@@ -55,6 +55,7 @@
 #include <private/qmemoryvideobuffer_p.h>
 #include <private/qcameradevice_p.h>
 #include <private/qmediastoragelocation_p.h>
+#include <QImageWriter>
 
 QT_BEGIN_NAMESPACE
 
@@ -67,7 +68,6 @@ QAndroidCameraSession::QAndroidCameraSession(QObject *parent)
     , m_videoOutput(0)
     , m_savedState(-1)
     , m_previewStarted(false)
-    , m_lastImageCaptureId(0)
     , m_readyForCapture(false)
     , m_currentImageCaptureId(-1)
     , m_previewCallback(0)
@@ -582,30 +582,39 @@ void QAndroidCameraSession::setReadyForCapture(bool ready)
     emit readyForCaptureChanged(ready);
 }
 
-int QAndroidCameraSession::capture(const QString &fileName)
+int QAndroidCameraSession::captureImage()
 {
-    ++m_lastImageCaptureId;
+    const int newImageCaptureId = m_currentImageCaptureId + 1;
 
     if (!isReadyForCapture()) {
-        emit imageCaptureError(m_lastImageCaptureId, QImageCapture::NotReadyError,
+        emit imageCaptureError(newImageCaptureId, QImageCapture::NotReadyError,
                                QPlatformImageCapture::msgCameraNotReady());
-        return m_lastImageCaptureId;
+        return newImageCaptureId;
     }
 
     setReadyForCapture(false);
 
-    m_currentImageCaptureId = m_lastImageCaptureId;
-    m_currentImageCaptureFileName = fileName;
+    m_currentImageCaptureId = newImageCaptureId;
 
     applyImageSettings();
     applyResolution(m_actualImageSettings.resolution());
+    m_camera->takePicture();
 
-    // adjust picture rotation depending on the device orientation
-    m_camera->setRotation(currentCameraRotation());
+    return m_currentImageCaptureId;
+}
 
-    m_camera->takePicture();
+int QAndroidCameraSession::capture(const QString &fileName)
+{
+    m_currentImageCaptureFileName = fileName;
+    m_imageCaptureToBuffer = false;
+    return captureImage();
+}
 
-    return m_lastImageCaptureId;
+int QAndroidCameraSession::captureToBuffer()
+{
+    m_currentImageCaptureFileName.clear();
+    m_imageCaptureToBuffer = true;
+    return captureImage();
 }
 
 void QAndroidCameraSession::onCameraTakePictureFailed()
@@ -631,10 +640,10 @@ void QAndroidCameraSession::onLastPreviewFrameFetched(const QVideoFrame &frame)
     if (!m_camera)
         return;
 
-    (void) QtConcurrent::run(&QAndroidCameraSession::processPreviewImage, this,
-                      m_currentImageCaptureId,
-                      frame,
-                      m_camera->getRotation());
+    updateOrientation();
+
+    (void)QtConcurrent::run(&QAndroidCameraSession::processPreviewImage, this,
+                            m_currentImageCaptureId, frame, currentCameraRotation());
 }
 
 void QAndroidCameraSession::processPreviewImage(int id, const QVideoFrame &frame, int rotation)
@@ -643,9 +652,10 @@ void QAndroidCameraSession::processPreviewImage(int id, const QVideoFrame &frame
     // we get here is not. Flip it ourselves if the camera is front-facing to match what the user
     // sees on the viewfinder.
     QTransform transform;
+    transform.rotate(rotation);
+
     if (m_camera->getFacing() == AndroidCamera::CameraFacingFront)
         transform.scale(-1, 1);
-    transform.rotate(rotation);
 
     emit imageCaptured(id, frame.toImage().transformed(transform));
 }
@@ -663,15 +673,12 @@ void QAndroidCameraSession::onNewPreviewFrame(const QVideoFrame &frame)
     m_videoFrameCallbackMutex.unlock();
 }
 
-void QAndroidCameraSession::onCameraPictureCaptured(const QByteArray &data)
+void QAndroidCameraSession::onCameraPictureCaptured(const QVideoFrame &frame)
 {
     // Loading and saving the captured image can be slow, do it in a separate thread
-    (void) QtConcurrent::run(&QAndroidCameraSession::processCapturedImage, this,
-                      m_currentImageCaptureId,
-                      data,
-                      m_actualImageSettings.resolution(),
-                      /* captureToBuffer = */ false,
-                      m_currentImageCaptureFileName);
+    (void)QtConcurrent::run(&QAndroidCameraSession::processCapturedImage, this,
+                            m_currentImageCaptureId, frame, m_imageCaptureToBuffer,
+                            m_currentImageCaptureFileName);
 
     // Preview needs to be restarted after taking a picture
     if (m_camera)
@@ -709,38 +716,35 @@ void QAndroidCameraSession::onCameraPreviewStopped()
     setReadyForCapture(false);
 }
 
-void QAndroidCameraSession::processCapturedImage(int id,
-                                                 const QByteArray &data,
-                                                 const QSize &resolution,
-                                                 bool captureToBuffer,
-                                                 const QString &fileName)
+void QAndroidCameraSession::processCapturedImage(int id, const QVideoFrame &frame,
+                                                 bool captureToBuffer, const QString &fileName)
 {
+    if (captureToBuffer) {
+        emit imageAvailable(id, frame);
+        return;
+    }
 
+    const QString actualFileName = QMediaStorageLocation::generateFileName(
+            fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg"));
+    QImageWriter writer(actualFileName);
 
-    if (!captureToBuffer) {
-        const QString actualFileName = QMediaStorageLocation::generateFileName(fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg"));
-
-        QFile file(actualFileName);
-        if (file.open(QFile::WriteOnly)) {
-            if (file.write(data) == data.size()) {
-                // if the picture is saved into the standard picture location, register it
-                // with the Android media scanner so it appears immediately in apps
-                // such as the gallery.
-                if (fileName.isEmpty() || QFileInfo(fileName).isRelative())
-                    AndroidMultimediaUtils::registerMediaFile(actualFileName);
+    if (!writer.canWrite()) {
+        const QString errorMessage = tr("File is not available: %1").arg(writer.errorString());
+        emit imageCaptureError(id, QImageCapture::Error::ResourceError, errorMessage);
+        return;
+    }
 
-                emit imageSaved(id, actualFileName);
-            } else {
-                emit imageCaptureError(id, QImageCapture::OutOfSpaceError, file.errorString());
-            }
-        } else {
-            const QString errorMessage = tr("Could not open destination file: %1").arg(actualFileName);
-            emit imageCaptureError(id, QImageCapture::ResourceError, errorMessage);
-        }
-    } else {
-        QVideoFrame frame(new QMemoryVideoBuffer(data, -1), QVideoFrameFormat(resolution, QVideoFrameFormat::Format_Jpeg));
-        emit imageAvailable(id, frame);
+    const bool written = writer.write(frame.toImage());
+    if (!written) {
+        const QString errorMessage = tr("Could not save to file: %1").arg(writer.errorString());
+        emit imageCaptureError(id, QImageCapture::Error::ResourceError, errorMessage);
+        return;
     }
+
+    if (fileName.isEmpty() || QFileInfo(fileName).isRelative())
+        AndroidMultimediaUtils::registerMediaFile(actualFileName);
+
+    emit imageSaved(id, actualFileName);
 }
 
 void QAndroidCameraSession::onVideoOutputReady(bool ready)
diff --git a/src/multimedia/platform/android/mediacapture/qandroidcamerasession_p.h b/src/multimedia/platform/android/mediacapture/qandroidcamerasession_p.h
index e6310efd9..a94920866 100644
--- a/src/multimedia/platform/android/mediacapture/qandroidcamerasession_p.h
+++ b/src/multimedia/platform/android/mediacapture/qandroidcamerasession_p.h
@@ -98,6 +98,7 @@ public:
     bool isReadyForCapture() const;
     void setReadyForCapture(bool ready);
     int capture(const QString &fileName);
+    int captureToBuffer();
 
     int currentCameraRotation() const;
 
@@ -137,7 +138,7 @@ private Q_SLOTS:
 
     void onCameraTakePictureFailed();
     void onCameraPictureExposed();
-    void onCameraPictureCaptured(const QByteArray &data);
+    void onCameraPictureCaptured(const QVideoFrame &frame);
     void onLastPreviewFrameFetched(const QVideoFrame &frame);
     void onNewPreviewFrame(const QVideoFrame &frame);
     void onCameraPreviewStarted();
@@ -156,14 +157,13 @@ private:
     void applyImageSettings();
 
     void processPreviewImage(int id, const QVideoFrame &frame, int rotation);
-    void processCapturedImage(int id,
-                              const QByteArray &data,
-                              const QSize &resolution,
-                              bool captureToBuffer,
+    void processCapturedImage(int id, const QVideoFrame &frame, bool captureToBuffer,
                               const QString &fileName);
 
     void setActiveHelper(bool active);
 
+    int captureImage();
+
     int m_selectedCamera;
     AndroidCamera *m_camera;
     QAndroidVideoOutput *m_videoOutput;
@@ -183,10 +183,10 @@ private:
     AndroidCamera::FpsRange m_requestedFpsRange;
     AndroidCamera::ImageFormat m_requestedPixelFromat;
 
-    int m_lastImageCaptureId;
     bool m_readyForCapture;
     int m_currentImageCaptureId;
     QString m_currentImageCaptureFileName;
+    bool m_imageCaptureToBuffer;
 
     QMutex m_videoFrameCallbackMutex;
     PreviewCallback *m_previewCallback;
diff --git a/src/multimedia/platform/android/mediacapture/qandroidimagecapture.cpp b/src/multimedia/platform/android/mediacapture/qandroidimagecapture.cpp
index 32e787290..a3fd5cd16 100644
--- a/src/multimedia/platform/android/mediacapture/qandroidimagecapture.cpp
+++ b/src/multimedia/platform/android/mediacapture/qandroidimagecapture.cpp
@@ -61,13 +61,7 @@ int QAndroidImageCapture::capture(const QString &fileName)
 
 int QAndroidImageCapture::captureToBuffer()
 {
-    // ### implement me!
-    const QLatin1String errorMessage("Capturing to buffer not supported.");
-    QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
-                              Q_ARG(int, -1),
-                              Q_ARG(int, QImageCapture::NotSupportedFeatureError),
-                              Q_ARG(QString, errorMessage));
-    return -1;
+    return m_session->captureToBuffer();
 }
 
 QImageEncoderSettings QAndroidImageCapture::imageSettings() const
diff --git a/src/multimedia/platform/android/wrappers/jni/androidcamera.cpp b/src/multimedia/platform/android/wrappers/jni/androidcamera.cpp
index a60eb6ed0..80830565e 100644
--- a/src/multimedia/platform/android/wrappers/jni/androidcamera.cpp
+++ b/src/multimedia/platform/android/wrappers/jni/androidcamera.cpp
@@ -56,6 +56,8 @@
 
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(lcAndroidCamera, "qt.multimedia.android.camera")
+
 static const char QtCameraListenerClassName[] = "org/qtproject/qt/android/multimedia/QtCameraListener";
 
 typedef QHash<int, AndroidCamera *> CameraMap;
@@ -111,13 +113,57 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data)
 {
     QReadLocker locker(rwLock);
     const auto it = cameras->constFind(id);
-    if (Q_UNLIKELY(it == cameras->cend()))
+    if (Q_UNLIKELY(it == cameras->cend())) {
+        qCWarning(lcAndroidCamera) << "Could not obtain camera!";
         return;
+    }
+
+    AndroidCamera *camera = (*it);
 
     const int arrayLength = env->GetArrayLength(data);
     QByteArray bytes(arrayLength, Qt::Uninitialized);
-    env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data());
-    Q_EMIT (*it)->pictureCaptured(bytes);
+    env->GetByteArrayRegion(data, 0, arrayLength, reinterpret_cast<jbyte *>(bytes.data()));
+
+    auto parameters = camera->getParametersObject();
+
+    QJniObject size =
+            parameters.callObjectMethod("getPictureSize", "()Landroid/hardware/Camera$Size;");
+
+    if (!size.isValid()) {
+        qCWarning(lcAndroidCamera) << "Picture Size is not valid!";
+        return;
+    }
+
+    QSize pictureSize(size.getField<jint>("width"), size.getField<jint>("height"));
+
+    auto format = AndroidCamera::ImageFormat(parameters.callMethod<jint>("getPictureFormat"));
+
+    if (format == AndroidCamera::ImageFormat::UnknownImageFormat) {
+        qCWarning(lcAndroidCamera) << "Android Camera Image Format is UnknownImageFormat!";
+        return;
+    }
+
+    int bytesPerLine = 0;
+
+    switch (format) {
+    case AndroidCamera::ImageFormat::YV12:
+        bytesPerLine = (pictureSize.width() + 15) & ~15;
+        break;
+    case AndroidCamera::ImageFormat::NV21:
+        bytesPerLine = pictureSize.width();
+        break;
+    case AndroidCamera::ImageFormat::RGB565:
+    case AndroidCamera::ImageFormat::YUY2:
+        bytesPerLine = pictureSize.width() * 2;
+        break;
+    default:
+        bytesPerLine = -1;
+    }
+
+    QVideoFrame frame(new QMemoryVideoBuffer(bytes, bytesPerLine),
+                      QVideoFrameFormat(pictureSize, qt_pixelFormatFromAndroidImageFormat(format)));
+
+    emit camera->pictureCaptured(frame);
 }
 
 static void notifyNewPreviewFrame(JNIEnv *env, jobject, int id, jbyteArray data,
@@ -860,6 +906,12 @@ void AndroidCamera::stopPreviewSynchronous()
     QMetaObject::invokeMethod(d, "stopPreview", Qt::BlockingQueuedConnection);
 }
 
+QJniObject AndroidCamera::getParametersObject()
+{
+    Q_D(AndroidCamera);
+    return d->m_parameters;
+}
+
 AndroidCameraPrivate::AndroidCameraPrivate()
     : QObject()
 {
diff --git a/src/multimedia/platform/android/wrappers/jni/androidcamera_p.h b/src/multimedia/platform/android/wrappers/jni/androidcamera_p.h
index d5bded8d3..ff018335b 100644
--- a/src/multimedia/platform/android/wrappers/jni/androidcamera_p.h
+++ b/src/multimedia/platform/android/wrappers/jni/androidcamera_p.h
@@ -203,6 +203,7 @@ public:
     void notifyNewFrames(bool notify);
     void fetchLastPreviewFrame();
     QJniObject getCameraObject();
+    QJniObject getParametersObject();
 
     static int getNumberOfCameras();
     static void getCameraInfo(int id, QCameraDevicePrivate *info);
@@ -224,7 +225,7 @@ Q_SIGNALS:
 
     void takePictureFailed();
     void pictureExposed();
-    void pictureCaptured(const QByteArray &data);
+    void pictureCaptured(const QVideoFrame &frame);
     void lastPreviewFrameFetched(const QVideoFrame &frame);
     void newPreviewFrame(const QVideoFrame &frame);