d3d11videosink: Add render-rectangle propertyHEADmain

... and resize HWND on GstVideoOverlay::set_render_rectangle even when
we are rendering without external HWND

Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/2563
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4653>

2 files changed, 37 insertions, 23 deletions

diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp
index 99d3bede3c..19cc0fd92a 100644
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp
@@ -65,6 +65,7 @@ enum
   PROP_PRIMARIES_MODE,
   PROP_DISPLAY_FORMAT,
   PROP_EMIT_PRESENT,
+  PROP_RENDER_RECTANGE,
 };
 
 #define DEFAULT_ADAPTER                   -1
@@ -399,6 +400,13 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
               G_PARAM_STATIC_STRINGS)));
 
   /**
+   * GstD3D11VideoSink:render-rectangle:
+   *
+   * Since: 1.24
+   */
+  gst_video_overlay_install_properties (gobject_class, PROP_RENDER_RECTANGE);
+
+  /**
    * GstD3D11VideoSink::begin-draw:
    * @videosink: the #d3d11videosink
    *
@@ -571,6 +579,10 @@ gst_d3d11_videosink_set_property (GObject * object, guint prop_id,
     case PROP_EMIT_PRESENT:
       self->emit_present = g_value_get_boolean (value);
       break;
+    case PROP_RENDER_RECTANGE:
+      gst_video_overlay_set_property (object, PROP_RENDER_RECTANGE,
+          PROP_RENDER_RECTANGE, value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp
index c773f1de7f..bd2eaa43c5 100644
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp
@@ -343,16 +343,10 @@ gst_d3d11_window_win32_set_render_rectangle (GstD3D11Window * window,
        * on message pumping thread is not a worst idea in generall */
       PostMessageA (self->internal_hwnd, WM_GST_D3D11_MOVE_WINDOW, 0, 0);
     }
-  } else {
-    /* XXX: Not sure what's expected behavior if we are drawing on internal
-     * HWND but user wants to specify rectangle.
-     *
-     * - Should we move window to corresponding desktop coordinates ?
-     * - Or should crop correspondingly by modifying viewport of
-     *   render target view of swapchian's backbuffer or so ?
-     * - Or should we ignore set_render_rectangle if we are drawing on
-     *   internal HWND without external HWND ?
-     */
+  } else if (!window->external_handle && self->internal_hwnd) {
+    MoveWindow (self->internal_hwnd,
+        self->render_rect.x, self->render_rect.y, self->render_rect.w,
+        self->render_rect.h, TRUE);
   }
 }
 
@@ -1189,22 +1183,30 @@ gst_d3d11_window_win32_show (GstD3D11Window * window)
     /* if no parent the real size has to be set now because this has not been done
      * when at window creation */
     if (!self->external_hwnd) {
-      RECT rect = { 0, };
-
-      rect.right = width;
-      rect.bottom = height;
-
-      if (AdjustWindowRect (&rect, WS_GST_D3D11, FALSE)) {
-        width = rect.right - rect.left;
-        height = rect.bottom - rect.top;
+      if (self->render_rect.x != 0 || self->render_rect.y != 0 ||
+          self->render_rect.w != 0 || self->render_rect.h != 0) {
+        MoveWindow (self->internal_hwnd,
+            self->render_rect.x, self->render_rect.y, self->render_rect.w,
+            self->render_rect.h, FALSE);
       } else {
-        width += 2 * GetSystemMetrics (SM_CXSIZEFRAME);
-        height +=
-            2 * GetSystemMetrics (SM_CYSIZEFRAME) +
-            GetSystemMetrics (SM_CYCAPTION);
+        RECT rect = { 0, };
+
+        rect.right = width;
+        rect.bottom = height;
+
+        if (AdjustWindowRect (&rect, WS_GST_D3D11, FALSE)) {
+          width = rect.right - rect.left;
+          height = rect.bottom - rect.top;
+        } else {
+          width += 2 * GetSystemMetrics (SM_CXSIZEFRAME);
+          height +=
+              2 * GetSystemMetrics (SM_CYSIZEFRAME) +
+              GetSystemMetrics (SM_CYCAPTION);
+        }
+
+        MoveWindow (self->internal_hwnd, 0, 0, width, height, FALSE);
       }
 
-      MoveWindow (self->internal_hwnd, 0, 0, width, height, FALSE);
       ShowWindow (self->internal_hwnd, SW_SHOW);
     } else if (self->internal_hwnd) {
       /* ShowWindow will throw message to message pumping thread (app thread)