ci: Port CI to the new monorepo

Main differences with previous setup are:
- No manifest creation
- gst-indent is executed only when the bot is assigned (instead of the manifest task)
- Cerbero jobs are triggered in the cerbero repo
- Remove cerbero and android related files as they now are in cerbero
  itself.
- Update `container.ps1` to the new file layout

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/891>

5 files changed, 595 insertions, 0 deletions

diff --git a/ci/fuzzing/README.txt b/ci/fuzzing/README.txt
new file mode 100644
index 0000000000..c9621a0455
--- /dev/null
+++ b/ci/fuzzing/README.txt
@@ -0,0 +1,80 @@
+Fuzzing GStreamer
+=================
+
+  This directory contains the various fuzzing targets and helper
+  scripts.
+
+* Fuzzing targets
+
+  Fuzzing targets as small applications where we can test a specific
+  element or API. The goal is to have them be as small/targetted as
+  possible.
+
+    ex: appsrc ! <some_element> ! fakesink num-buffers=<small>
+    
+  Not all components can be tested directly and therefore will be
+  indirectly tested via other targets (ex: libgstaudio will be tested
+  by targets/elements requiring it)
+
+  Anything that can process externally-provided data should be
+  covered, but there are cases where it might not make sense to use a
+  fuzzer (such as most elements processing raw audio/video).
+
+* build-oss-fuzz.sh
+
+  This is the script executed by the oss-fuzz project.
+
+  It builds glib, GStreamer, plugins and the fuzzing targets.
+
+* *.c
+
+  The fuzzing targets where the data to test will be provided to a
+  function whose signature follows the LibFuzzer signature:
+  https://llvm.org/docs/LibFuzzer.html
+
+* TODO
+
+  * Add a standalone build script
+
+    We need to be able to build and test the fuzzing targets outside
+    of the oss-fuzz infrastructure, and do that in our continous
+    integration system.
+
+    We need:
+
+    * A dummy fuzzing engine (given a directory, it opens all files and
+      calls the fuzzing targets with the content of those files.
+    * A script to be able to build those targets with that dummy engine
+    * A corpus of files to test those targets with.
+
+  * Build targets with dummy engine and run with existing tests.
+
+  * Create pull-based variants
+
+    Currently the existing targets are push-based only. Where
+    applicable we should make pull-based variants to test the other
+    code paths.
+
+  * Add more targets
+
+    core:
+      gst_parse fuzzer ?
+    base:
+      ext/
+        ogg
+	opus
+	pango
+	theora
+	vorbis
+      gst/
+        subparse
+	typefind : already covered in typefind target
+      gst-libs/gst/
+        sdp
+	other ones easily testable directly ?
+
+        	
+	 
+      
+      
+
diff --git a/ci/fuzzing/build-oss-fuzz.sh b/ci/fuzzing/build-oss-fuzz.sh
new file mode 100755
index 0000000000..297c322173
--- /dev/null
+++ b/ci/fuzzing/build-oss-fuzz.sh
@@ -0,0 +1,185 @@
+#!/bin/bash -eu
+
+# build-oss-fuzz.sh
+#
+# Build script which is executed by oss-fuzz build.sh
+#
+# $SRC: location of code checkouts
+# $OUT: location to put fuzzing targets and corpus
+# $WORK: writable directory where all compilation should be executed
+#
+# /!\ Do not override any CC, CXX, CFLAGS, ... variables
+#
+
+# This script is divided in two parts
+#
+# 1) Build all the dependencies statically
+#
+# 2) Build the fuzzing targets
+
+# Prefix where we will temporarily install everything
+PREFIX=$WORK/prefix
+mkdir -p $PREFIX
+# always try getting the arguments for static compilation/linking
+# Fixes GModule not being picked when gstreamer-1.0.pc is looked up by meson
+# more or less https://github.com/mesonbuild/meson/pull/6629
+export PKG_CONFIG="`which pkg-config` --static"
+export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig
+export PATH=$PREFIX/bin:$PATH
+
+# Minimize gst-debug level/code
+export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2"
+
+#
+echo "CFLAGS : " $CFLAGS
+echo "CXXFLAGS : " $CXXFLAGS
+PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0
+
+rm -rf $WORK/*
+
+# Switch to work directory
+cd $WORK
+
+# 1) BUILD GLIB AND GSTREAMER
+# Note: we build glib ourselves so that we get proper malloc/free backtraces
+tar xvJf $SRC/glib-2.64.2.tar.xz
+cd glib-2.64.2
+# options taken from glib's oss-fuzz build definition
+meson \
+    --prefix=$PREFIX \
+    --libdir=lib \
+    --default-library=static \
+    -Db_lundef=false \
+    -Doss_fuzz=enabled \
+    -Dlibmount=disabled \
+    -Dinternal_pcre=true \
+    _builddir
+ninja -C _builddir
+ninja -C _builddir install
+cd ..
+
+# Note: We don't use/build orc since it still seems to be problematic
+# with clang and the various sanitizers.
+
+# For now we only build core and base. Add other modules when/if needed
+for i in gstreamer gst-plugins-base;
+do
+    mkdir -p $i
+    cd $i
+    meson \
+        --prefix=$PREFIX \
+        --libdir=lib \
+        --default-library=static \
+        -Db_lundef=false \
+        -Ddoc=disabled \
+        -Dexamples=disabled \
+        -Dintrospection=disabled \
+        -Dtracer_hooks=false \
+        -Dregistry=false _builddir $SRC/$i
+    ninja -C _builddir
+    ninja -C _builddir install
+    cd ..
+done
+
+
+
+# 2) Build the target fuzzers
+
+# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the
+# actual fuzzing runner. Anything fuzzing engine can be used provided it calls
+# the same function as libfuzzer.
+
+# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX
+
+# We want to statically link everything, except for shared libraries
+# that are present on the base image. Those need to be specified
+# beforehand and explicitely linked dynamically If any of the static
+# dependencies require a pre-installed shared library, you need to add
+# that library to the following list
+PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread"
+
+# These are the basic .pc dependencies required to build any of the fuzzing targets
+# That is : glib, gstreamer core and gst-app
+# The extra target-specific dependencies are to be specified later
+COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0"
+
+# For each target, defined the following:
+# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS)
+#               All dependencies (including sub-dependencies) must be speecified
+# PLUGINS : .a of the plugins to link
+#           They must match the static plugins declared/registered in the target
+
+#
+# TARGET : push-based ogg/theora/vorbis discoverer
+#
+# FIXME : Rename to discoverer_push_oggtheoravorbis
+
+TARGET_DEPS=" gstreamer-pbutils-1.0 \
+	      gstreamer-video-1.0 \
+	      gstreamer-audio-1.0 \
+	      gstreamer-riff-1.0 \
+	      gstreamer-tag-1.0 \
+	      zlib ogg vorbis vorbisenc \
+	      theoraenc theoradec theora"
+
+PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
+       $PLUGIN_DIR/libgsttypefindfunctions.a \
+       $PLUGIN_DIR/libgstplayback.a \
+       $PLUGIN_DIR/libgstapp.a \
+       $PLUGIN_DIR/libgstvorbis.a \
+       $PLUGIN_DIR/libgsttheora.a \
+       $PLUGIN_DIR/libgstogg.a"
+
+echo
+echo ">>>> BUILDING gst-discoverer"
+echo
+BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
+BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
+
+$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/gst-discoverer.c -o $SRC/gst-ci/fuzzing/gst-discoverer.o
+$CXX $CXXFLAGS \
+      -o $OUT/gst-discoverer \
+      $PREDEPS_LDFLAGS \
+      $SRC/gst-ci/fuzzing/gst-discoverer.o \
+      $PLUGINS \
+      $BUILD_LDFLAGS \
+      $LIB_FUZZING_ENGINE \
+      -Wl,-Bdynamic
+
+#
+# TARGET : push-based typefind
+#
+
+# typefindfunction depends on pbutils which depends on gst{audio|video|tag}
+TARGET_DEPS=" gstreamer-pbutils-1.0 \
+	      gstreamer-video-1.0 \
+	      gstreamer-audio-1.0 \
+	      gstreamer-tag-1.0"
+
+PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
+       $PLUGIN_DIR/libgsttypefindfunctions.a \
+       $PLUGIN_DIR/libgstapp.a"
+
+echo
+echo ">>>> BUILDING typefind"
+echo
+BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
+BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
+
+$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/typefind.c -o $SRC/gst-ci/fuzzing/typefind.o
+$CXX $CXXFLAGS \
+      -o $OUT/typefind \
+      $PREDEPS_LDFLAGS \
+      $SRC/gst-ci/fuzzing/typefind.o \
+      $PLUGINS \
+      $BUILD_LDFLAGS \
+      $LIB_FUZZING_ENGINE \
+      -Wl,-Bdynamic
+
+echo
+echo ">>>> Installing seed corpus"
+echo
+# FIXME : Sadly we apparently need to have the corpus downloaded in the
+#         Dockerfile and not here.
+
+cp $SRC/*_seed_corpus.zip $OUT
diff --git a/ci/fuzzing/gst-discoverer.c b/ci/fuzzing/gst-discoverer.c
new file mode 100644
index 0000000000..0440606a4a
--- /dev/null
+++ b/ci/fuzzing/gst-discoverer.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2016 Google Inc.
+ * author: Edward Hervey <bilboed@bilboed.com>
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <locale.h>
+
+#include <stdlib.h>
+#include <glib.h>
+#include <gst/gst.h>
+#include <gst/pbutils/pbutils.h>
+
+#ifndef LOCAL_FUZZ_BUILD
+GST_PLUGIN_STATIC_DECLARE (coreelements);
+GST_PLUGIN_STATIC_DECLARE (playback);
+GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
+GST_PLUGIN_STATIC_DECLARE (app);
+GST_PLUGIN_STATIC_DECLARE (ogg);
+GST_PLUGIN_STATIC_DECLARE (theora);
+GST_PLUGIN_STATIC_DECLARE (vorbis);
+#endif
+
+/* push-based discoverer fuzzing target
+ *
+ * This application can be compiled with libFuzzer to simulate
+ * a push-based discoverer execution.
+ *
+ * To reproduce the failing behaviour, use:
+ * $ gst-discoverer-1.0 pushfile:///...
+ *
+ * The goal is to cover basic usage of demuxers, parsers and
+ * base decoder elements.
+ *
+ * When compiling, only link the required demuxer/parser/decoder
+ * plugins and keep it to a limited range (ex: ogg/theora/vorbis)
+ *
+ **/
+
+const guint8 *fuzztesting_data;
+size_t fuzztesting_size;
+
+static void
+appsrc_configuration (GstDiscoverer * dc, GstElement * source, gpointer data)
+{
+  GstBuffer *buf;
+  GstFlowReturn ret;
+
+  /* Create buffer from fuzztesting_data which shouldn't be freed */
+  buf =
+      gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data,
+      fuzztesting_size, 0, fuzztesting_size, NULL, NULL);
+  g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL);
+  g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &ret);
+  gst_buffer_unref (buf);
+}
+
+static void
+custom_logger (const gchar * log_domain,
+    GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
+{
+  if (log_level & G_LOG_LEVEL_CRITICAL) {
+    g_printerr ("CRITICAL ERROR : %s\n", message);
+    abort ();
+  } else if (log_level & G_LOG_LEVEL_WARNING) {
+    g_printerr ("WARNING : %s\n", message);
+  }
+}
+
+int
+LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
+{
+  GError *err = NULL;
+  GstDiscoverer *dc;
+  gint timeout = 10;
+  GstDiscovererInfo *info;
+  static gboolean initialized = FALSE;
+
+  if (!initialized) {
+    /* We want critical warnings to assert so we can fix them */
+    g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
+    g_log_set_default_handler (custom_logger, NULL);
+
+    /* Only initialize and register plugins once */
+    gst_init (NULL, NULL);
+
+#ifndef LOCAL_FUZZ_BUILD
+    GST_PLUGIN_STATIC_REGISTER (coreelements);
+    GST_PLUGIN_STATIC_REGISTER (playback);
+    GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
+    GST_PLUGIN_STATIC_REGISTER (app);
+    GST_PLUGIN_STATIC_REGISTER (ogg);
+    GST_PLUGIN_STATIC_REGISTER (theora);
+    GST_PLUGIN_STATIC_REGISTER (vorbis);
+#endif
+
+    initialized = TRUE;
+  }
+
+  dc = gst_discoverer_new (timeout * GST_SECOND, &err);
+  if (G_UNLIKELY (dc == NULL)) {
+    g_print ("Error initializing: %s\n", err->message);
+    g_clear_error (&err);
+    exit (1);
+  }
+
+  fuzztesting_data = data;
+  fuzztesting_size = size;
+
+  /* Connect to source-setup signal to give the data */
+  g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL);
+
+  info = gst_discoverer_discover_uri (dc, "appsrc://", &err);
+  g_clear_error (&err);
+  if (info)
+    gst_discoverer_info_unref (info);
+
+  g_object_unref (dc);
+
+  return 0;
+}
diff --git a/ci/fuzzing/localfuzzer.c b/ci/fuzzing/localfuzzer.c
new file mode 100644
index 0000000000..188d2461a8
--- /dev/null
+++ b/ci/fuzzing/localfuzzer.c
@@ -0,0 +1,78 @@
+/* GStreamer
+ * Copyright (C) 2017 Edward Hervey <bilboed@bilboed.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/* Local fuzzer runner */
+#include <glib.h>
+
+extern int LLVMFuzzerTestOneInput (const guint8 * data, size_t size);
+
+static void
+test_file (gchar * filename)
+{
+  GDir *dir;
+  gchar *path;
+  gchar *contents;
+  gsize length;
+
+  /* if filename is a directory, process the contents */
+  if ((dir = g_dir_open (filename, 0, NULL))) {
+    const gchar *entry;
+
+    while ((entry = g_dir_read_name (dir))) {
+      gchar *spath;
+
+      spath = g_strconcat (filename, G_DIR_SEPARATOR_S, entry, NULL);
+      test_file (spath);
+      g_free (spath);
+    }
+
+    g_dir_close (dir);
+    return;
+  }
+
+  /* Make sure path is absolute */
+  if (!g_path_is_absolute (filename)) {
+    gchar *curdir;
+
+    curdir = g_get_current_dir ();
+    path = g_build_filename (curdir, filename, NULL);
+    g_free (curdir);
+  } else
+    path = g_strdup (filename);
+
+  /* Check if path exists */
+  if (g_file_get_contents (path, &contents, &length, NULL)) {
+    g_print (">>> %s (%" G_GSIZE_FORMAT " bytes)\n", path, length);
+    LLVMFuzzerTestOneInput ((const guint8 *) contents, length);
+    g_free (contents);
+  }
+
+  g_free (path);
+}
+
+int
+main (int argc, gchar ** argv)
+{
+  gint i;
+
+  for (i = 1; i < argc; i++)
+    test_file (argv[i]);
+
+  return 0;
+}
diff --git a/ci/fuzzing/typefind.c b/ci/fuzzing/typefind.c
new file mode 100644
index 0000000000..999ba337fc
--- /dev/null
+++ b/ci/fuzzing/typefind.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2016 Google Inc.
+ * author: Edward Hervey <bilboed@bilboed.com>
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <locale.h>
+
+#include <stdlib.h>
+#include <glib.h>
+#include <gst/gst.h>
+
+#ifndef LOCAL_FUZZ_BUILD
+GST_PLUGIN_STATIC_DECLARE (coreelements);
+GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
+GST_PLUGIN_STATIC_DECLARE (app);
+#endif
+
+/* push-based typefind fuzzing target
+ *
+ * This application can be compiled with libFuzzer to simulate
+ * a push-based typefind execution.
+ *
+ * To reproduce the failing behaviour, use:
+ * $ gst-launch-1.0 pushfile:///.. ! typefind ! fakesink
+ *
+ * The goal is to cover typefind code and implementation.
+ *
+ **/
+static void
+custom_logger (const gchar * log_domain,
+    GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
+{
+  if (log_level & G_LOG_LEVEL_CRITICAL) {
+    g_printerr ("CRITICAL ERROR : %s\n", message);
+    abort ();
+  } else if (log_level & G_LOG_LEVEL_WARNING) {
+    g_printerr ("WARNING : %s\n", message);
+  }
+}
+
+int
+LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
+{
+  GError *err = NULL;
+  static gboolean initialized = FALSE;
+  GstElement *pipeline, *source, *typefind, *fakesink;
+  GstBuffer *buf;
+  GstFlowReturn flowret;
+  GstState state;
+
+  if (!initialized) {
+    /* We want critical warnings to assert so we can fix them */
+    g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
+    g_log_set_default_handler (custom_logger, NULL);
+
+    /* Only initialize and register plugins once */
+    gst_init (NULL, NULL);
+
+#ifndef LOCAL_FUZZ_BUILD
+    GST_PLUGIN_STATIC_REGISTER (coreelements);
+    GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
+    GST_PLUGIN_STATIC_REGISTER (app);
+#endif
+
+    initialized = TRUE;
+  }
+
+  /* Create the pipeline */
+  pipeline = gst_pipeline_new ("pipeline");
+  source = gst_element_factory_make ("appsrc", "source");
+  typefind = gst_element_factory_make ("typefind", "typefind");
+  fakesink = gst_element_factory_make ("fakesink", "fakesink");
+
+  gst_bin_add_many (GST_BIN (pipeline), source, typefind, fakesink, NULL);
+  gst_element_link_many (source, typefind, fakesink, NULL);
+
+  /* Set pipeline to READY so we can provide data to appsrc */
+  gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY);
+  buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size,
+      0, size, NULL, NULL);
+  g_object_set (G_OBJECT (source), "size", size, NULL);
+  g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &flowret);
+  gst_buffer_unref (buf);
+
+  /* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */
+  gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED);
+
+  /* wait until state change either completes or fails */
+  gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1);
+
+  /* Go back to NULL */
+  gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL);
+
+  /* And release the pipeline */
+  gst_object_unref (pipeline);
+
+  return 0;
+}