/* * GStreamer * Copyright (C) 2007 David Schleef * Copyright (C) 2008 Julien Isorce * Copyright (C) 2008 Filippo Argiolas * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstglfilter.h" #if GST_GL_HAVE_PLATFORM_EGL #include "egl/gsteglimagememory.h" #endif #define GST_CAT_DEFAULT gst_gl_filter_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); static GstStaticPadTemplate gst_gl_filter_src_pad_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") "; " #if GST_GL_HAVE_PLATFORM_EGL GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, "RGBA") "; " #endif GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) ); static GstStaticPadTemplate gst_gl_filter_sink_pad_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") "; " #if GST_GL_HAVE_PLATFORM_EGL GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, "RGBA") "; " #endif GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) ); /* Properties */ enum { PROP_0, PROP_OTHER_CONTEXT }; #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_filter_debug, "glfilter", 0, "glfilter element"); #define gst_gl_filter_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLFilter, gst_gl_filter, GST_TYPE_BASE_TRANSFORM, DEBUG_INIT); static void gst_gl_filter_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_filter_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_filter_set_context (GstElement * element, GstContext * context); static gboolean gst_gl_filter_query (GstBaseTransform * trans, GstPadDirection direction, GstQuery * query); static GstCaps *gst_gl_filter_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter); static GstCaps *gst_gl_filter_fixate_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); static void gst_gl_filter_reset (GstGLFilter * filter); static gboolean gst_gl_filter_start (GstBaseTransform * bt); static gboolean gst_gl_filter_stop (GstBaseTransform * bt); static gboolean gst_gl_filter_get_unit_size (GstBaseTransform * trans, GstCaps * caps, gsize * size); static GstFlowReturn gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf); static gboolean gst_gl_filter_propose_allocation (GstBaseTransform * trans, GstQuery * decide_query, GstQuery * query); static gboolean gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query); static gboolean gst_gl_filter_set_caps (GstBaseTransform * bt, GstCaps * incaps, GstCaps * outcaps); /* GstGLContextThreadFunc */ static void gst_gl_filter_start_gl (GstGLContext * context, gpointer data); static void gst_gl_filter_stop_gl (GstGLContext * context, gpointer data); static void gst_gl_filter_class_init (GstGLFilterClass * klass) { GObjectClass *gobject_class; GstElementClass *element_class; gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); gobject_class->set_property = gst_gl_filter_set_property; gobject_class->get_property = gst_gl_filter_get_property; GST_BASE_TRANSFORM_CLASS (klass)->transform_caps = gst_gl_filter_transform_caps; GST_BASE_TRANSFORM_CLASS (klass)->fixate_caps = gst_gl_filter_fixate_caps; GST_BASE_TRANSFORM_CLASS (klass)->transform = gst_gl_filter_transform; GST_BASE_TRANSFORM_CLASS (klass)->query = gst_gl_filter_query; GST_BASE_TRANSFORM_CLASS (klass)->start = gst_gl_filter_start; GST_BASE_TRANSFORM_CLASS (klass)->stop = gst_gl_filter_stop; GST_BASE_TRANSFORM_CLASS (klass)->set_caps = gst_gl_filter_set_caps; GST_BASE_TRANSFORM_CLASS (klass)->propose_allocation = gst_gl_filter_propose_allocation; GST_BASE_TRANSFORM_CLASS (klass)->decide_allocation = gst_gl_filter_decide_allocation; GST_BASE_TRANSFORM_CLASS (klass)->get_unit_size = gst_gl_filter_get_unit_size; element_class->set_context = gst_gl_filter_set_context; g_object_class_install_property (gobject_class, PROP_OTHER_CONTEXT, g_param_spec_object ("other-context", "External OpenGL context", "Give an external OpenGL context with which to share textures", GST_GL_TYPE_CONTEXT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&gst_gl_filter_src_pad_template)); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&gst_gl_filter_sink_pad_template)); klass->set_caps = NULL; klass->filter = NULL; klass->display_init_cb = NULL; klass->display_reset_cb = NULL; klass->onInitFBO = NULL; klass->onStart = NULL; klass->onStop = NULL; klass->onReset = NULL; klass->filter_texture = NULL; } static void gst_gl_filter_init (GstGLFilter * filter) { gst_gl_filter_reset (filter); } static void gst_gl_filter_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstGLFilter *filter = GST_GL_FILTER (object); switch (prop_id) { case PROP_OTHER_CONTEXT: { if (filter->other_context) gst_object_unref (filter->other_context); filter->other_context = g_value_dup_object (value); break; } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static void gst_gl_filter_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstGLFilter *filter = GST_GL_FILTER (object); switch (prop_id) { case PROP_OTHER_CONTEXT: g_value_set_object (value, filter->other_context); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static void gst_gl_filter_set_context (GstElement * element, GstContext * context) { GstGLFilter *filter = GST_GL_FILTER (element); gst_gl_handle_set_context (element, context, &filter->display); } static gboolean gst_gl_filter_query (GstBaseTransform * trans, GstPadDirection direction, GstQuery * query) { GstGLFilter *filter; filter = GST_GL_FILTER (trans); switch (GST_QUERY_TYPE (query)) { case GST_QUERY_ALLOCATION: { if (direction == GST_PAD_SINK && gst_base_transform_is_passthrough (trans)) return gst_pad_peer_query (GST_BASE_TRANSFORM_SRC_PAD (trans), query); break; } case GST_QUERY_CONTEXT: { return gst_gl_handle_context_query ((GstElement *) filter, query, &filter->display); break; } default: break; } return GST_BASE_TRANSFORM_CLASS (parent_class)->query (trans, direction, query);; } static void gst_gl_filter_reset (GstGLFilter * filter) { GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); gst_caps_replace (&filter->out_caps, NULL); if (filter->upload) { gst_object_unref (filter->upload); filter->upload = NULL; } if (filter->download) { gst_object_unref (filter->download); filter->download = NULL; } if (filter->context) { if (filter_class->onReset) filter_class->onReset (filter); if (filter_class->display_reset_cb != NULL) { gst_gl_context_thread_add (filter->context, gst_gl_filter_stop_gl, filter); } //blocking call, delete the FBO if (filter->fbo != 0) { gst_gl_context_del_fbo (filter->context, filter->fbo, filter->depthbuffer); } if (filter->in_tex_id) { gst_gl_context_del_texture (filter->context, &filter->in_tex_id); filter->in_tex_id = 0; } if (filter->out_tex_id) { gst_gl_context_del_texture (filter->context, &filter->out_tex_id); filter->out_tex_id = 0; } gst_object_unref (filter->context); filter->context = NULL; } if (filter->display) { gst_object_unref (filter->display); filter->display = NULL; } filter->fbo = 0; filter->depthbuffer = 0; filter->default_shader = NULL; if (filter->other_context) gst_object_unref (filter->other_context); filter->other_context = NULL; if (filter->context) gst_object_unref (filter->context); filter->context = NULL; } static gboolean gst_gl_filter_start (GstBaseTransform * bt) { GstGLFilter *filter = GST_GL_FILTER (bt); GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); if (!gst_gl_ensure_display (filter, &filter->display)) return FALSE; if (filter_class->onStart) filter_class->onStart (filter); return TRUE; } static gboolean gst_gl_filter_stop (GstBaseTransform * bt) { GstGLFilter *filter = GST_GL_FILTER (bt); GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); if (filter_class->onStop) filter_class->onStop (filter); gst_gl_filter_reset (filter); return TRUE; } static void gst_gl_filter_start_gl (GstGLContext * context, gpointer data) { GstGLFilter *filter = GST_GL_FILTER (data); GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); filter_class->display_init_cb (filter); } static void gst_gl_filter_stop_gl (GstGLContext * context, gpointer data) { GstGLFilter *filter = GST_GL_FILTER (data); GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); filter_class->display_reset_cb (filter); } static GstCaps * gst_gl_filter_fixate_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) { GstStructure *ins, *outs; const GValue *from_par, *to_par; GValue fpar = { 0, }, tpar = { 0,}; othercaps = gst_caps_truncate (othercaps); othercaps = gst_caps_make_writable (othercaps); GST_DEBUG_OBJECT (bt, "trying to fixate othercaps %" GST_PTR_FORMAT " based on caps %" GST_PTR_FORMAT, othercaps, caps); ins = gst_caps_get_structure (caps, 0); outs = gst_caps_get_structure (othercaps, 0); from_par = gst_structure_get_value (ins, "pixel-aspect-ratio"); to_par = gst_structure_get_value (outs, "pixel-aspect-ratio"); /* If we're fixating from the sinkpad we always set the PAR and * assume that missing PAR on the sinkpad means 1/1 and * missing PAR on the srcpad means undefined */ if (direction == GST_PAD_SINK) { if (!from_par) { g_value_init (&fpar, GST_TYPE_FRACTION); gst_value_set_fraction (&fpar, 1, 1); from_par = &fpar; } if (!to_par) { g_value_init (&tpar, GST_TYPE_FRACTION); gst_value_set_fraction (&tpar, 1, 1); to_par = &tpar; } } else { if (!to_par) { g_value_init (&tpar, GST_TYPE_FRACTION); gst_value_set_fraction (&tpar, 1, 1); to_par = &tpar; gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); } if (!from_par) { g_value_init (&fpar, GST_TYPE_FRACTION); gst_value_set_fraction (&fpar, 1, 1); from_par = &fpar; } } /* we have both PAR but they might not be fixated */ { gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d; gint w = 0, h = 0; gint from_dar_n, from_dar_d; gint num, den; /* from_par should be fixed */ g_return_val_if_fail (gst_value_is_fixed (from_par), othercaps); from_par_n = gst_value_get_fraction_numerator (from_par); from_par_d = gst_value_get_fraction_denominator (from_par); gst_structure_get_int (ins, "width", &from_w); gst_structure_get_int (ins, "height", &from_h); gst_structure_get_int (outs, "width", &w); gst_structure_get_int (outs, "height", &h); /* if both width and height are already fixed, we can't do anything * about it anymore */ if (w && h) { gint n = 1, d = 1; GST_DEBUG_OBJECT (bt, "dimensions already set to %dx%d, not fixating", w, h); if (!gst_value_is_fixed (to_par)) { GST_DEBUG_OBJECT (bt, "fixating to_par to %dx%d", n, d); if (gst_structure_has_field (outs, "pixel-aspect-ratio")) gst_structure_fixate_field_nearest_fraction (outs, "pixel-aspect-ratio", 1, 1); else if (n != d) gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); } goto done; } /* Calculate input DAR */ if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d, &from_dar_n, &from_dar_d)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } GST_DEBUG_OBJECT (bt, "Input DAR is %d/%d", from_dar_n, from_dar_d); /* If either width or height are fixed there's not much we * can do either except choosing a height or width and PAR * that matches the DAR as good as possible */ if (h) { gint num, den; GST_DEBUG_OBJECT (bt, "height is fixed (%d)", h); if (!gst_value_is_fixed (to_par)) { gst_value_set_fraction (&tpar, 1, 1); } /* PAR is fixed, choose the height that is nearest to the * height with the same DAR */ to_par_n = gst_value_get_fraction_numerator (to_par); to_par_d = gst_value_get_fraction_denominator (to_par); GST_DEBUG_OBJECT (bt, "PAR is fixed %d/%d", to_par_n, to_par_d); if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d, to_par_n, &num, &den)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } w = (guint) gst_util_uint64_scale_int (h, num, den); gst_structure_fixate_field_nearest_int (outs, "width", w); goto done; } else if (w) { gint num, den; GST_DEBUG_OBJECT (bt, "width is fixed (%d)", w); if (!gst_value_is_fixed (to_par)) { gst_value_set_fraction (&tpar, 1, 1); } /* PAR is fixed, choose the height that is nearest to the * height with the same DAR */ to_par_n = gst_value_get_fraction_numerator (to_par); to_par_d = gst_value_get_fraction_denominator (to_par); GST_DEBUG_OBJECT (bt, "PAR is fixed %d/%d", to_par_n, to_par_d); if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d, to_par_n, &num, &den)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } h = (guint) gst_util_uint64_scale_int (w, den, num); gst_structure_fixate_field_nearest_int (outs, "height", h); goto done; } else if (gst_value_is_fixed (to_par)) { GstStructure *tmp; gint set_h, set_w, f_h, f_w; to_par_n = gst_value_get_fraction_numerator (to_par); to_par_d = gst_value_get_fraction_denominator (to_par); /* Calculate scale factor for the PAR change */ if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n, to_par_d, &num, &den)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } /* Try to keep the input height */ tmp = gst_structure_copy (outs); gst_structure_fixate_field_nearest_int (tmp, "height", from_h); gst_structure_get_int (tmp, "height", &set_h); /* This might have failed but try to scale the width * to keep the DAR nonetheless */ w = (guint) gst_util_uint64_scale_int (set_h, num, den); gst_structure_fixate_field_nearest_int (tmp, "width", w); gst_structure_get_int (tmp, "width", &set_w); gst_structure_free (tmp); /* We kept the DAR and the height is nearest to the original height */ if (set_w == w) { gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", G_TYPE_INT, set_h, NULL); goto done; } f_h = set_h; f_w = set_w; /* If the former failed, try to keep the input width at least */ tmp = gst_structure_copy (outs); gst_structure_fixate_field_nearest_int (tmp, "width", from_w); gst_structure_get_int (tmp, "width", &set_w); /* This might have failed but try to scale the width * to keep the DAR nonetheless */ h = (guint) gst_util_uint64_scale_int (set_w, den, num); gst_structure_fixate_field_nearest_int (tmp, "height", h); gst_structure_get_int (tmp, "height", &set_h); gst_structure_free (tmp); /* We kept the DAR and the width is nearest to the original width */ if (set_h == h) { gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", G_TYPE_INT, set_h, NULL); goto done; } /* If all this failed, keep the height that was nearest to the orignal * height and the nearest possible width. This changes the DAR but * there's not much else to do here. */ gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT, f_h, NULL); goto done; } else { GstStructure *tmp; gint set_h, set_w, set_par_n, set_par_d, tmp2; /* width, height and PAR are not fixed */ /* First try to keep the height and width as good as possible * and scale PAR */ tmp = gst_structure_copy (outs); gst_structure_fixate_field_nearest_int (tmp, "height", from_h); gst_structure_get_int (tmp, "height", &set_h); gst_structure_fixate_field_nearest_int (tmp, "width", from_w); gst_structure_get_int (tmp, "width", &set_w); if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w, &to_par_n, &to_par_d)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } if (!gst_structure_has_field (tmp, "pixel-aspect-ratio")) gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par); gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio", to_par_n, to_par_d); gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n, &set_par_d); gst_structure_free (tmp); if (set_par_n == to_par_n && set_par_d == to_par_d) { gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", G_TYPE_INT, set_h, NULL); if (gst_structure_has_field (outs, "pixel-aspect-ratio") || set_par_n != set_par_d) gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d, NULL); goto done; } /* Otherwise try to scale width to keep the DAR with the set * PAR and height */ if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d, set_par_n, &num, &den)) { GST_ELEMENT_ERROR (bt, CORE, NEGOTIATION, (NULL), ("Error calculating the output scaled size - integer overflow")); goto done; } w = (guint) gst_util_uint64_scale_int (set_h, num, den); tmp = gst_structure_copy (outs); gst_structure_fixate_field_nearest_int (tmp, "width", w); gst_structure_get_int (tmp, "width", &tmp2); gst_structure_free (tmp); if (tmp2 == w) { gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height", G_TYPE_INT, set_h, NULL); if (gst_structure_has_field (outs, "pixel-aspect-ratio") || set_par_n != set_par_d) gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d, NULL); goto done; } /* ... or try the same with the height */ h = (guint) gst_util_uint64_scale_int (set_w, den, num); tmp = gst_structure_copy (outs); gst_structure_fixate_field_nearest_int (tmp, "height", h); gst_structure_get_int (tmp, "height", &tmp2); gst_structure_free (tmp); if (tmp2 == h) { gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", G_TYPE_INT, tmp2, NULL); if (gst_structure_has_field (outs, "pixel-aspect-ratio") || set_par_n != set_par_d) gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d, NULL); goto done; } /* If all fails we can't keep the DAR and take the nearest values * for everything from the first try */ gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", G_TYPE_INT, set_h, NULL); if (gst_structure_has_field (outs, "pixel-aspect-ratio") || set_par_n != set_par_d) gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d, NULL); } } done: othercaps = gst_caps_fixate (othercaps); GST_DEBUG_OBJECT (bt, "fixated othercaps to %" GST_PTR_FORMAT, othercaps); if (from_par == &fpar) g_value_unset (&fpar); if (to_par == &tpar) g_value_unset (&tpar); return othercaps; } static GstCaps * gst_gl_filter_set_caps_features (const GstCaps * caps, const gchar * feature_name) { GstCaps *tmp = gst_caps_copy (caps); guint n = gst_caps_get_size (tmp); guint i = 0; for (i = 0; i < n; i++) { GstCapsFeatures *features = gst_caps_get_features (tmp, i); if (features) { guint n_f = gst_caps_features_get_size (features); guint j = 0; for (j = 0; j < n_f; j++) { gst_caps_features_remove_id (features, gst_caps_features_get_nth_id (features, j)); } } gst_caps_features_add (features, feature_name); gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); } return tmp; } /* copies the given caps */ static GstCaps * gst_gl_filter_caps_remove_format_info (GstCaps * caps) { GstStructure *st; GstCapsFeatures *f; gint i, n; GstCaps *res; res = gst_caps_new_empty (); n = gst_caps_get_size (caps); for (i = 0; i < n; i++) { st = gst_caps_get_structure (caps, i); f = gst_caps_get_features (caps, i); /* If this is already expressed by the existing caps * skip this structure */ if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) continue; st = gst_structure_copy (st); /* Only remove format info for the cases when we can actually convert */ if (!gst_caps_features_is_any (f) && gst_caps_features_is_equal (f, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", "width", "height", NULL); gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); } return res; } static GstCaps * gst_gl_filter_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter) { GstCaps *tmp = NULL; GstCaps *result = NULL; GstCaps *glcaps = gst_gl_filter_set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); #if GST_GL_HAVE_PLATFORM_EGL GstCaps *eglcaps = gst_gl_filter_set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); #endif GstCaps *uploadcaps = gst_gl_filter_set_caps_features (caps, GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); GstCaps *raw_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); GstCapsFeatures *f; tmp = gst_caps_new_empty (); tmp = gst_caps_merge (tmp, glcaps); #if GST_GL_HAVE_PLATFORM_EGL tmp = gst_caps_merge (tmp, eglcaps); #endif tmp = gst_caps_merge (tmp, uploadcaps); tmp = gst_caps_merge (tmp, raw_caps); tmp = gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (caps)); if (filter) { result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (tmp); } else { result = tmp; } /* if output still intersects input then prefer the intersection */ f = gst_caps_get_features (caps, 0); if (!gst_caps_features_is_any (f) && !gst_caps_features_is_equal (f, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) { tmp = gst_caps_intersect_full (result, caps, GST_CAPS_INTERSECT_FIRST); result = gst_caps_merge (tmp, result); } GST_DEBUG_OBJECT (bt, "returning caps: %" GST_PTR_FORMAT, result); return result; } static gboolean gst_gl_filter_get_unit_size (GstBaseTransform * trans, GstCaps * caps, gsize * size) { gboolean ret = FALSE; GstVideoInfo info; ret = gst_video_info_from_caps (&info, caps); if (ret) *size = GST_VIDEO_INFO_SIZE (&info); return TRUE; } static gboolean gst_gl_filter_set_caps (GstBaseTransform * bt, GstCaps * incaps, GstCaps * outcaps) { GstGLFilter *filter; GstGLFilterClass *filter_class; filter = GST_GL_FILTER (bt); filter_class = GST_GL_FILTER_GET_CLASS (filter); if (!gst_video_info_from_caps (&filter->in_info, incaps)) goto wrong_caps; if (!gst_video_info_from_caps (&filter->out_info, outcaps)) goto wrong_caps; if (filter_class->set_caps) { if (!filter_class->set_caps (filter, incaps, outcaps)) goto error; } gst_caps_replace (&filter->out_caps, outcaps); GST_DEBUG ("set_caps %dx%d", GST_VIDEO_INFO_WIDTH (&filter->out_info), GST_VIDEO_INFO_HEIGHT (&filter->out_info)); return TRUE; /* ERRORS */ wrong_caps: { GST_WARNING ("Wrong caps"); return FALSE; } error: { return FALSE; } } static gboolean gst_gl_filter_propose_allocation (GstBaseTransform * trans, GstQuery * decide_query, GstQuery * query) { GstGLFilter *filter = GST_GL_FILTER (trans); GstBufferPool *pool; GstStructure *config; GstCaps *caps, *decide_caps; guint size; gboolean need_pool; GError *error = NULL; GstStructure *gl_context; gchar *platform, *gl_apis; gpointer handle; GstAllocator *allocator = NULL; GstAllocationParams params; gst_query_parse_allocation (query, &caps, &need_pool); if (caps == NULL) goto no_caps; if ((pool = filter->pool)) gst_object_ref (pool); if (pool != NULL) { GstCaps *pcaps; /* we had a pool, check caps */ GST_DEBUG_OBJECT (filter, "check existing pool caps"); config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL); if (!gst_caps_is_equal (caps, pcaps)) { GST_DEBUG_OBJECT (filter, "pool has different caps"); /* different caps, we can't use this pool */ gst_object_unref (pool); pool = NULL; } gst_structure_free (config); } if (!gst_gl_ensure_display (filter, &filter->display)) return FALSE; if (!filter->context) { filter->context = gst_gl_context_new (filter->display); if (!gst_gl_context_create (filter->context, filter->other_context, &error)) goto context_error; } if (pool == NULL && need_pool) { GstVideoInfo info; GstBufferPool *decide_pool = NULL; if (!gst_video_info_from_caps (&info, caps)) goto invalid_caps; if (decide_query) { gst_query_parse_allocation (decide_query, &decide_caps, NULL); decide_pool = gst_base_transform_get_buffer_pool (trans); } if (decide_pool && GST_IS_GL_BUFFER_POOL (decide_pool) && gst_caps_is_equal_fixed (decide_caps, caps)) { pool = decide_pool; } else { GST_DEBUG_OBJECT (filter, "create new pool"); if (decide_pool) gst_object_unref (decide_pool); pool = gst_gl_buffer_pool_new (filter->context); /* the normal size of a frame */ size = info.size; config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, 0, 0); if (!gst_buffer_pool_set_config (pool, config)) goto config_failed; } } /* we need at least 2 buffer because we hold on to the last one */ if (pool) { gst_query_add_allocation_pool (query, pool, size, 1, 0); gst_object_unref (pool); } /* we also support various metadata */ gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (filter->context)); platform = gst_gl_platform_to_string (gst_gl_context_get_gl_platform (filter->context)); handle = (gpointer) gst_gl_context_get_gl_context (filter->context); gl_context = gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", GST_GL_TYPE_CONTEXT, filter->context, "gst.gl.context.handle", G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); gst_query_add_allocation_meta (query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); g_free (gl_apis); g_free (platform); gst_structure_free (gl_context); gst_allocation_params_init (¶ms); allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); gst_query_add_allocation_param (query, allocator, ¶ms); gst_object_unref (allocator); #if GST_GL_HAVE_PLATFORM_EGL if (gst_gl_context_check_feature (filter->context, "EGL_KHR_image_base")) { allocator = gst_allocator_find (GST_EGL_IMAGE_MEMORY_TYPE); gst_query_add_allocation_param (query, allocator, ¶ms); gst_object_unref (allocator); } #endif return TRUE; /* ERRORS */ no_caps: { GST_DEBUG_OBJECT (trans, "no caps specified"); return FALSE; } invalid_caps: { GST_DEBUG_OBJECT (trans, "invalid caps specified"); return FALSE; } config_failed: { GST_DEBUG_OBJECT (trans, "failed setting config"); return FALSE; } context_error: { GST_ELEMENT_ERROR (trans, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL)); return FALSE; } } static gboolean gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) { GstGLFilter *filter = GST_GL_FILTER (trans); GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint min, max, size; gboolean update_pool; guint idx; GError *error = NULL; guint in_width, in_height, out_width, out_height; GstGLContext *other_context = NULL; if (!gst_gl_ensure_display (filter, &filter->display)) return FALSE; if (gst_query_find_allocation_meta (query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { GstGLContext *context; const GstStructure *upload_meta_params; gpointer handle; gchar *type; gchar *apis; gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); if (upload_meta_params) { if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", GST_GL_TYPE_CONTEXT, &context, NULL) && context) { GstGLContext *old = filter->context; filter->context = context; if (old) gst_object_unref (old); } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) && handle) { GstGLPlatform platform = GST_GL_PLATFORM_NONE; GstGLAPI gl_apis; GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", handle, type, apis); platform = gst_gl_platform_from_string (type); gl_apis = gst_gl_api_from_string (apis); if (gl_apis && platform) other_context = gst_gl_context_new_wrapped (filter->display, (guintptr) handle, platform, gl_apis); } } } if (filter->other_context) { if (!other_context) { other_context = filter->other_context; } else { GST_ELEMENT_WARNING (filter, LIBRARY, SETTINGS, ("%s", "Cannot share with more than one GL context"), ("%s", "Cannot share with more than one GL context")); } } if (!filter->context) { filter->context = gst_gl_context_new (filter->display); if (!gst_gl_context_create (filter->context, other_context, &error)) goto context_error; } in_width = GST_VIDEO_INFO_WIDTH (&filter->in_info); in_height = GST_VIDEO_INFO_HEIGHT (&filter->in_info); out_width = GST_VIDEO_INFO_WIDTH (&filter->out_info); out_height = GST_VIDEO_INFO_HEIGHT (&filter->out_info); if (filter->fbo) { gst_gl_context_del_fbo (filter->context, filter->fbo, filter->depthbuffer); filter->fbo = 0; filter->depthbuffer = 0; } if (filter->in_tex_id) { gst_gl_context_del_texture (filter->context, &filter->in_tex_id); filter->in_tex_id = 0; } if (filter->out_tex_id) { gst_gl_context_del_texture (filter->context, &filter->out_tex_id); filter->out_tex_id = 0; } //blocking call, generate a FBO if (!gst_gl_context_gen_fbo (filter->context, out_width, out_height, &filter->fbo, &filter->depthbuffer)) goto context_error; gst_gl_context_gen_texture (filter->context, &filter->in_tex_id, GST_VIDEO_FORMAT_RGBA, in_width, in_height); gst_gl_context_gen_texture (filter->context, &filter->out_tex_id, GST_VIDEO_FORMAT_RGBA, out_width, out_height); if (filter_class->display_init_cb != NULL) { gst_gl_context_thread_add (filter->context, gst_gl_filter_start_gl, filter); } if (filter_class->onInitFBO) { if (!filter_class->onInitFBO (filter)) goto error; } gst_query_parse_allocation (query, &caps, NULL); if (gst_query_get_n_allocation_pools (query) > 0) { gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); update_pool = TRUE; } else { GstVideoInfo vinfo; gst_video_info_init (&vinfo); gst_video_info_from_caps (&vinfo, caps); size = vinfo.size; min = max = 0; update_pool = FALSE; } if (!pool) pool = gst_gl_buffer_pool_new (filter->context); config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); gst_buffer_pool_set_config (pool, config); if (filter->upload) { gst_object_unref (filter->upload); filter->upload = NULL; } if (update_pool) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); else gst_query_add_allocation_pool (query, pool, size, min, max); gst_object_unref (pool); return TRUE; context_error: { GST_ELEMENT_ERROR (trans, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL)); return FALSE; } error: { GST_ELEMENT_ERROR (trans, LIBRARY, INIT, ("Subclass failed to initialize."), (NULL)); return FALSE; } } /** * gst_gl_filter_filter_texture: * @filter: a #GstGLFilter * @inbuf: an input buffer * @outbuf: an output buffer * * Perform automatic upload if needed, call filter_texture vfunc and then an * automatic download if needed. * * Returns: whether the transformation succeeded */ gboolean gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf, GstBuffer * outbuf) { GstGLFilterClass *filter_class; guint in_tex, out_tex; GstVideoFrame out_frame; gboolean ret; gboolean to_download = gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY, gst_caps_get_features (filter->out_caps, 0)); GstMapFlags out_map_flags = GST_MAP_WRITE; filter_class = GST_GL_FILTER_GET_CLASS (filter); if (!gst_gl_upload_perform_with_buffer (filter->upload, inbuf, &in_tex)) return FALSE; to_download |= !gst_is_gl_memory (gst_buffer_peek_memory (outbuf, 0)); if (!to_download) out_map_flags |= GST_MAP_GL; if (!gst_video_frame_map (&out_frame, &filter->out_info, outbuf, out_map_flags)) { ret = FALSE; goto inbuf_error; } if (!to_download) { out_tex = *(guint *) out_frame.data[0]; } else { GST_LOG ("Output Buffer does not contain correct memory, " "attempting to wrap for download"); if (!filter->download) filter->download = gst_gl_download_new (filter->context); gst_gl_download_set_format (filter->download, &out_frame.info); out_tex = filter->out_tex_id; } GST_DEBUG ("calling filter_texture with textures in:%i out:%i", in_tex, out_tex); g_assert (filter_class->filter_texture); ret = filter_class->filter_texture (filter, in_tex, out_tex); if (to_download) { if (!gst_gl_download_perform_with_data (filter->download, out_tex, out_frame.data)) { GST_ELEMENT_ERROR (filter, RESOURCE, NOT_FOUND, ("%s", "Failed to download video frame"), (NULL)); ret = FALSE; goto error; } } error: gst_video_frame_unmap (&out_frame); inbuf_error: gst_gl_upload_release_buffer (filter->upload); return ret; } static GstFlowReturn gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf) { GstGLFilter *filter; GstGLFilterClass *filter_class; filter = GST_GL_FILTER (bt); filter_class = GST_GL_FILTER_GET_CLASS (bt); if (!gst_gl_ensure_display (filter, &filter->display)) return GST_FLOW_NOT_NEGOTIATED; if (!filter->upload) { filter->upload = gst_gl_upload_new (filter->context); gst_gl_upload_set_format (filter->upload, &filter->in_info); } g_assert (filter_class->filter || filter_class->filter_texture); if (filter_class->filter) filter_class->filter (filter, inbuf, outbuf); else if (filter_class->filter_texture) gst_gl_filter_filter_texture (filter, inbuf, outbuf); return GST_FLOW_OK; } /* convenience functions to simplify filter development */ /** * gst_gl_filter_render_to_target: * @filter: a #GstGLFilter * @resize: whether to automatically resize the texture between the input size * and the output size * @input: the input texture * @target: the output texture * @func: the function to transform @input into @output. called with @data * @data: the data associated with @func * * Transforms @input into @output using @func on through FBO. @resize should * only ever be %TRUE whenever @input is the input texture of @filter. */ void gst_gl_filter_render_to_target (GstGLFilter * filter, gboolean resize, GLuint input, GLuint target, GLCB func, gpointer data) { guint in_width, in_height, out_width, out_height; out_width = GST_VIDEO_INFO_WIDTH (&filter->out_info); out_height = GST_VIDEO_INFO_HEIGHT (&filter->out_info); if (resize) { in_width = GST_VIDEO_INFO_WIDTH (&filter->in_info); in_height = GST_VIDEO_INFO_HEIGHT (&filter->in_info); } else { in_width = out_width; in_height = out_height; } GST_LOG ("rendering to target. in %u, %ux%u out %u, %ux%u", input, in_width, in_height, target, out_width, out_height); gst_gl_context_use_fbo (filter->context, out_width, out_height, filter->fbo, filter->depthbuffer, target, func, in_width, in_height, input, 0, in_width, 0, in_height, GST_GL_DISPLAY_PROJECTION_ORTHO2D, data); } static void _draw_with_shader_cb (gint width, gint height, guint texture, gpointer stuff) { GstGLFilter *filter = GST_GL_FILTER (stuff); GstGLFuncs *gl = filter->context->gl_vtable; #if GST_GL_HAVE_OPENGL if (gst_gl_context_get_gl_api (filter->context) & GST_GL_API_OPENGL) { gl->MatrixMode (GL_PROJECTION); gl->LoadIdentity (); } #endif gst_gl_shader_use (filter->default_shader); gl->ActiveTexture (GL_TEXTURE1); gl->BindTexture (GL_TEXTURE_2D, texture); gst_gl_shader_set_uniform_1i (filter->default_shader, "tex", 1); gst_gl_shader_set_uniform_1f (filter->default_shader, "width", width); gst_gl_shader_set_uniform_1f (filter->default_shader, "height", height); gst_gl_filter_draw_texture (filter, texture, width, height); } /** * gst_gl_filter_render_to_target_with_shader: * @filter: a #GstGLFilter * @resize: whether to automatically resize the texture between the input size * and the output size * @input: the input texture * @target: the output texture * @shader: the shader to use. * * Transforms @input into @output using @shader on FBO. @resize should * only ever be %TRUE whenever @input is the input texture of @filter. * * See also: gst_gl_filter_render_to_target() */ /* attach target to a FBO, use shader, pass input as "tex" uniform to * the shader, render input to a quad */ void gst_gl_filter_render_to_target_with_shader (GstGLFilter * filter, gboolean resize, GLuint input, GLuint target, GstGLShader * shader) { filter->default_shader = shader; gst_gl_filter_render_to_target (filter, resize, input, target, _draw_with_shader_cb, filter); } /** * gst_gl_filter_draw_texture: * @filter: a #GstGLFilter * @texture: the texture to draw * @width: width of @texture * @height: height of texture * * Draws @texture into the OpenGL scene at the specified @width and @height. */ void gst_gl_filter_draw_texture (GstGLFilter * filter, GLuint texture, guint width, guint height) { GstGLContext *context = filter->context; GstGLFuncs *gl = context->gl_vtable; GST_DEBUG ("drawing texture:%u dimensions:%ux%u", texture, width, height); #if GST_GL_HAVE_OPENGL if (gst_gl_context_get_gl_api (context) & GST_GL_API_OPENGL) { GLfloat verts[] = { -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f }; GLfloat texcoords[] = { 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 1.0f }; gl->ActiveTexture (GL_TEXTURE0); gl->Enable (GL_TEXTURE_2D); gl->BindTexture (GL_TEXTURE_2D, texture); gl->ClientActiveTexture (GL_TEXTURE0); gl->EnableClientState (GL_VERTEX_ARRAY); gl->EnableClientState (GL_TEXTURE_COORD_ARRAY); gl->VertexPointer (2, GL_FLOAT, 0, &verts); gl->TexCoordPointer (2, GL_FLOAT, 0, &texcoords); gl->DrawArrays (GL_TRIANGLE_FAN, 0, 4); gl->DisableClientState (GL_VERTEX_ARRAY); gl->DisableClientState (GL_TEXTURE_COORD_ARRAY); } #endif #if GST_GL_HAVE_GLES2 if (gst_gl_context_get_gl_api (context) & GST_GL_API_GLES2) { const GLfloat vVertices[] = { -1.0f, -1.0f, 0.0f, 0.0f, 0.0f, 1.0, -1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 1.0f, 1.0f, -1.0f, 1.0f, 0.0f, 0.0f, 1.0f }; GLushort indices[] = { 0, 1, 2, 0, 2, 3 }; /* glClear (GL_COLOR_BUFFER_BIT); */ /* Load the vertex position */ gl->VertexAttribPointer (filter->draw_attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), vVertices); /* Load the texture coordinate */ gl->VertexAttribPointer (filter->draw_attr_texture_loc, 2, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &vVertices[3]); gl->EnableVertexAttribArray (filter->draw_attr_position_loc); gl->EnableVertexAttribArray (filter->draw_attr_texture_loc); gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); gl->DisableVertexAttribArray (filter->draw_attr_position_loc); gl->DisableVertexAttribArray (filter->draw_attr_texture_loc); } #endif }