/* * Copyright (c) 2017 Rob Clark * Copyright (c) 2017 Carlos Rafael Giani * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sub license, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include "common.h" #include #include #include #include #include #include #include GST_DEBUG_CATEGORY_EXTERN(kmscube_debug); #define GST_CAT_DEFAULT kmscube_debug #define MAX_NUM_PLANES 3 inline static const char * yesno(int yes) { return yes ? "yes" : "no"; } struct decoder { GMainLoop *loop; GstElement *pipeline; GstElement *sink; pthread_t gst_thread; uint32_t format; GstVideoInfo info; const struct gbm *gbm; const struct egl *egl; unsigned frame; EGLImage last_frame; GstSample *last_samp; }; static GstPadProbeReturn pad_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { struct decoder *dec = user_data; GstEvent *event = GST_PAD_PROBE_INFO_EVENT(info); GstCaps *caps; (void)pad; if (GST_EVENT_TYPE(event) != GST_EVENT_CAPS) return GST_PAD_PROBE_OK; gst_event_parse_caps(event, &caps); if (!caps) { GST_ERROR("caps event without caps"); return GST_PAD_PROBE_OK; } if (!gst_video_info_from_caps(&dec->info, caps)) { GST_ERROR("caps event with invalid video caps"); return GST_PAD_PROBE_OK; } switch (GST_VIDEO_INFO_FORMAT(&(dec->info))) { case GST_VIDEO_FORMAT_I420: dec->format = DRM_FORMAT_YUV420; break; case GST_VIDEO_FORMAT_NV12: dec->format = DRM_FORMAT_NV12; break; case GST_VIDEO_FORMAT_YUY2: dec->format = DRM_FORMAT_YUYV; break; default: GST_ERROR("unknown format\n"); return GST_PAD_PROBE_OK; } return GST_PAD_PROBE_OK; } static void * gst_thread_func(void *args) { struct decoder *dec = args; g_main_loop_run(dec->loop); return NULL; } static void element_added_cb(GstBin *bin, GstElement *element, gpointer user_data) { GstElementFactory *elem_factory; gchar const *factory_name; (void)user_data; (void)bin; elem_factory = gst_element_get_factory(element); factory_name = gst_plugin_feature_get_name(elem_factory); GST_DEBUG("added element %s (created with factory %s)", GST_OBJECT_NAME(element), factory_name); /* v4l2 video decoder factories are generated by the GStreamer v4l probe. * The format is v4l2videoNdec, where N is an integer. So, check if the * element's factory name fits this pattern. */ if (g_str_has_prefix(factory_name, "v4l2video") && g_str_has_suffix(factory_name, "dec")) { /* yes, "capture" rather than "output" because v4l2 is bonkers */ gst_util_set_object_arg(G_OBJECT(element), "capture-io-mode", "dmabuf"); printf("found GStreamer V4L2 video decoder element with name \"%s\"\n", GST_OBJECT_NAME(element)); } } static gboolean bus_watch_cb(GstBus *bus, GstMessage *msg, gpointer user_data) { struct decoder *dec = (struct decoder *)user_data; (void)bus; switch (GST_MESSAGE_TYPE(msg)) { case GST_MESSAGE_STATE_CHANGED: { gchar *dotfilename; GstState old_gst_state, cur_gst_state, pending_gst_state; /* Only consider state change messages coming from * the toplevel element. */ if (GST_MESSAGE_SRC(msg) != GST_OBJECT(dec->pipeline)) break; gst_message_parse_state_changed(msg, &old_gst_state, &cur_gst_state, &pending_gst_state); printf( "GStreamer state change: old: %s current: %s pending: %s\n", gst_element_state_get_name(old_gst_state), gst_element_state_get_name(cur_gst_state), gst_element_state_get_name(pending_gst_state) ); dotfilename = g_strdup_printf( "statechange__old-%s__cur-%s__pending-%s", gst_element_state_get_name(old_gst_state), gst_element_state_get_name(cur_gst_state), gst_element_state_get_name(pending_gst_state) ); GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(dec->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, dotfilename); g_free(dotfilename); break; } case GST_MESSAGE_REQUEST_STATE: { GstState requested_state; gst_message_parse_request_state(msg, &requested_state); printf( "state change to %s was requested by %s\n", gst_element_state_get_name(requested_state), GST_MESSAGE_SRC_NAME(msg) ); gst_element_set_state(GST_ELEMENT(dec->pipeline), requested_state); break; } case GST_MESSAGE_LATENCY: { printf("redistributing latency\n"); gst_bin_recalculate_latency(GST_BIN(dec->pipeline)); break; } case GST_MESSAGE_INFO: case GST_MESSAGE_WARNING: case GST_MESSAGE_ERROR: { GError *error = NULL; gchar *debug_info = NULL; gchar const *prefix; switch (GST_MESSAGE_TYPE(msg)) { case GST_MESSAGE_INFO: gst_message_parse_info(msg, &error, &debug_info); prefix = "INFO"; break; case GST_MESSAGE_WARNING: gst_message_parse_warning(msg, &error, &debug_info); prefix = "WARNING"; break; case GST_MESSAGE_ERROR: gst_message_parse_error(msg, &error, &debug_info); prefix = "ERROR"; break; default: g_assert_not_reached(); } printf("GStreamer %s: %s; debug info: %s", prefix, error->message, debug_info); g_clear_error(&error); g_free(debug_info); if (GST_MESSAGE_TYPE(msg) == GST_MESSAGE_ERROR) { GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(dec->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "error"); } // TODO: stop mainloop in case of an error break; } default: break; } return TRUE; } static GstPadProbeReturn appsink_query_cb(GstPad *pad G_GNUC_UNUSED, GstPadProbeInfo *info, gpointer user_data G_GNUC_UNUSED) { GstQuery *query = info->data; if (GST_QUERY_TYPE (query) != GST_QUERY_ALLOCATION) return GST_PAD_PROBE_OK; gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); return GST_PAD_PROBE_HANDLED; } struct decoder * video_init(const struct egl *egl, const struct gbm *gbm, const char *filename) { struct decoder *dec; GstElement *src, *decodebin; GstPad *pad; GstBus *bus; if (egl_check(egl, eglCreateImageKHR) || egl_check(egl, eglDestroyImageKHR)) return NULL; dec = calloc(1, sizeof(*dec)); dec->loop = g_main_loop_new(NULL, FALSE); dec->gbm = gbm; dec->egl = egl; /* Setup pipeline: */ static const char *pipeline = "filesrc name=\"src\" ! decodebin name=\"decode\" ! video/x-raw ! appsink sync=false name=\"sink\""; dec->pipeline = gst_parse_launch(pipeline, NULL); dec->sink = gst_bin_get_by_name(GST_BIN(dec->pipeline), "sink"); /* Implement the allocation query using a pad probe. This probe will * adverstize support for GstVideoMeta, which avoid hardware accelerated * decoder that produce special strides and offsets from having to * copy the buffers. */ pad = gst_element_get_static_pad(dec->sink, "sink"); gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM, appsink_query_cb, NULL, NULL); gst_object_unref(pad); src = gst_bin_get_by_name(GST_BIN(dec->pipeline), "src"); g_object_set(G_OBJECT(src), "location", filename, NULL); gst_object_unref(src); /* Configure the sink like a video sink (mimic GstVideoSink) */ gst_base_sink_set_max_lateness(GST_BASE_SINK(dec->sink), 20 * GST_MSECOND); gst_base_sink_set_qos_enabled(GST_BASE_SINK(dec->sink), TRUE); /* if we don't limit max-buffers then we can let the decoder outrun * vsync and quickly chew up 100's of MB of buffers: */ g_object_set(G_OBJECT(dec->sink), "max-buffers", 2, NULL); gst_pad_add_probe(gst_element_get_static_pad(dec->sink, "sink"), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, pad_probe, dec, NULL); /* callback needed to make sure we get dmabuf's from v4l2videoNdec.. */ decodebin = gst_bin_get_by_name(GST_BIN(dec->pipeline), "decode"); g_signal_connect(decodebin, "element-added", G_CALLBACK(element_added_cb), dec); /* add bus to be able to receive error message, handle latency * requests, produce pipeline dumps, etc. */ bus = gst_pipeline_get_bus(GST_PIPELINE(dec->pipeline)); gst_bus_add_watch(bus, bus_watch_cb, dec); gst_object_unref(GST_OBJECT(bus)); /* let 'er rip! */ gst_element_set_state(dec->pipeline, GST_STATE_PLAYING); pthread_create(&dec->gst_thread, NULL, gst_thread_func, dec); return dec; } static void set_last_frame(struct decoder *dec, EGLImage frame, GstSample *samp) { if (dec->last_frame) dec->egl->eglDestroyImageKHR(dec->egl->display, dec->last_frame); dec->last_frame = frame; if (dec->last_samp) gst_sample_unref(dec->last_samp); dec->last_samp = samp; } // TODO this could probably be a helper re-used by cube-tex: static int buf_to_fd(const struct gbm *gbm, int size, void *ptr) { struct gbm_bo *bo; void *map, *map_data = NULL; uint32_t stride; int fd; /* NOTE: do not actually use GBM_BO_USE_WRITE since that gets us a dumb buffer: */ bo = gbm_bo_create(gbm->dev, size, 1, GBM_FORMAT_R8, GBM_BO_USE_LINEAR); map = gbm_bo_map(bo, 0, 0, size, 1, GBM_BO_TRANSFER_WRITE, &stride, &map_data); memcpy(map, ptr, size); gbm_bo_unmap(bo, map_data); fd = gbm_bo_get_fd(bo); /* we have the fd now, no longer need the bo: */ gbm_bo_destroy(bo); return fd; } static EGLImage buffer_to_image(struct decoder *dec, GstBuffer *buf) { struct { int fd, offset, stride; } planes[MAX_NUM_PLANES]; GstVideoMeta *meta = gst_buffer_get_video_meta(buf); EGLImage image; guint nmems = gst_buffer_n_memory(buf); guint nplanes = GST_VIDEO_INFO_N_PLANES(&(dec->info)); guint i; guint width, height; gboolean is_dmabuf_mem; GstMemory *mem; int dmabuf_fd = -1; static const EGLint egl_dmabuf_plane_fd_attr[MAX_NUM_PLANES] = { EGL_DMA_BUF_PLANE0_FD_EXT, EGL_DMA_BUF_PLANE1_FD_EXT, EGL_DMA_BUF_PLANE2_FD_EXT, }; static const EGLint egl_dmabuf_plane_offset_attr[MAX_NUM_PLANES] = { EGL_DMA_BUF_PLANE0_OFFSET_EXT, EGL_DMA_BUF_PLANE1_OFFSET_EXT, EGL_DMA_BUF_PLANE2_OFFSET_EXT, }; static const EGLint egl_dmabuf_plane_pitch_attr[MAX_NUM_PLANES] = { EGL_DMA_BUF_PLANE0_PITCH_EXT, EGL_DMA_BUF_PLANE1_PITCH_EXT, EGL_DMA_BUF_PLANE2_PITCH_EXT, }; /* Query gst_is_dmabuf_memory() here, since the gstmemory * block might get merged below by gst_buffer_map(), meaning * that the mem pointer would become invalid */ mem = gst_buffer_peek_memory(buf, 0); is_dmabuf_mem = gst_is_dmabuf_memory(mem); if (nmems > 1) { if (is_dmabuf_mem) { /* this case currently is not defined */ GST_FIXME("gstbuffers with multiple memory blocks and DMABUF " "memory currently are not supported"); return EGL_NO_IMAGE_KHR; } /* if this is not DMABUF memory, then the gst_buffer_map() * call below will automatically merge the memory blocks */ } if (is_dmabuf_mem) { dmabuf_fd = dup(gst_dmabuf_memory_get_fd(mem)); } else { GstMapInfo map_info; gst_buffer_map(buf, &map_info, GST_MAP_READ); dmabuf_fd = buf_to_fd(dec->gbm, map_info.size, map_info.data); gst_buffer_unmap(buf, &map_info); } if (dmabuf_fd < 0) { GST_ERROR("could not obtain DMABUF FD"); return EGL_NO_IMAGE_KHR; } /* Usually, a videometa should be present, since by using the internal kmscube * video_appsink element instead of the regular appsink, it is guaranteed that * video meta support is declared in the video_appsink's allocation query. * However, this assumes that upstream elements actually look at the allocation * query's contents properly, or that they even send a query at all. If this * is not the case, then upstream might decide to push frames without adding * a meta. It can happen, and in this case, look at the video info data as * a fallback (it is computed out of the input caps). */ if (meta) { for (i = 0; i < nplanes; i++) { planes[i].fd = dmabuf_fd; planes[i].offset = meta->offset[i]; planes[i].stride = meta->stride[i]; } } else { for (i = 0; i < nplanes; i++) { planes[i].fd = dmabuf_fd; planes[i].offset = GST_VIDEO_INFO_PLANE_OFFSET(&(dec->info), i); planes[i].stride = GST_VIDEO_INFO_PLANE_STRIDE(&(dec->info), i); } } width = GST_VIDEO_INFO_WIDTH(&(dec->info)); height = GST_VIDEO_INFO_HEIGHT(&(dec->info)); /* output some information at the beginning (= when the first frame is handled) */ if (dec->frame == 0) { GstVideoFormat pixfmt; const char *pixfmt_str; pixfmt = GST_VIDEO_INFO_FORMAT(&(dec->info)); pixfmt_str = gst_video_format_to_string(pixfmt); printf("===================================\n"); printf("GStreamer video stream information:\n"); printf(" size: %u x %u pixel\n", width, height); printf(" pixel format: %s number of planes: %u\n", pixfmt_str, nplanes); printf(" can use zero-copy: %s\n", yesno(is_dmabuf_mem)); printf(" video meta found: %s\n", yesno(meta != NULL)); printf("===================================\n"); } { /* Initialize the first 6 attributes with values that are * plane invariant (width, height, format) */ EGLint attr[6 + 6*(MAX_NUM_PLANES) + 1] = { EGL_WIDTH, width, EGL_HEIGHT, height, EGL_LINUX_DRM_FOURCC_EXT, dec->format }; for (i = 0; i < nplanes; i++) { attr[6 + 6*i + 0] = egl_dmabuf_plane_fd_attr[i]; attr[6 + 6*i + 1] = planes[i].fd; attr[6 + 6*i + 2] = egl_dmabuf_plane_offset_attr[i]; attr[6 + 6*i + 3] = planes[i].offset; attr[6 + 6*i + 4] = egl_dmabuf_plane_pitch_attr[i]; attr[6 + 6*i + 5] = planes[i].stride; } attr[6 + 6*nplanes] = EGL_NONE; image = dec->egl->eglCreateImageKHR(dec->egl->display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, attr); } /* Cleanup */ for (unsigned i = 0; i < nplanes; i++) close(planes[i].fd); return image; } EGLImage video_frame(struct decoder *dec) { GstSample *samp; GstBuffer *buf; EGLImage frame = NULL; samp = gst_app_sink_pull_sample(GST_APP_SINK(dec->sink)); if (!samp) { GST_DEBUG("got no appsink sample"); return NULL; } buf = gst_sample_get_buffer(samp); // TODO inline buffer_to_image?? frame = buffer_to_image(dec, buf); // TODO in the zero-copy dmabuf case it would be nice to associate // the eglimg w/ the buffer to avoid recreating it every frame.. set_last_frame(dec, frame, samp); dec->frame++; return frame; } void video_deinit(struct decoder *dec) { set_last_frame(dec, NULL, NULL); gst_element_set_state(dec->pipeline, GST_STATE_NULL); gst_object_unref(dec->sink); gst_object_unref(dec->pipeline); g_main_loop_quit(dec->loop); g_main_loop_unref(dec->loop); pthread_join(dec->gst_thread, 0); free(dec); }