meson: Generate ChangeLog files for release tarballs on dist

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3521>

2 files changed, 293 insertions, 0 deletions

diff --git a/scripts/gen-changelog.py b/scripts/gen-changelog.py
new file mode 100755
index 0000000000..3924e6ea32
--- /dev/null
+++ b/scripts/gen-changelog.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+#
+# Makes a GNU-Style ChangeLog from a git repository
+import os
+import sys
+import subprocess
+import re
+
+meson_source_root = os.environ.get('MESON_SOURCE_ROOT')
+
+meson_dist_root = os.environ.get('MESON_DIST_ROOT')
+if meson_dist_root:
+    output_fn = os.path.join(meson_dist_root, 'ChangeLog')
+else:
+    output_fn = sys.stdout.fileno()
+
+# commit hash => release version tag string
+release_refs = {}
+
+# These are the pre-monorepo module beginnings
+changelog_starts = {
+    'gstreamer': '70521179a75db0c7230cc47c6d7f9d63cf73d351',
+    'gst-plugins-base': '68746a38d5e48e6f7c220663dcc2f175ff55cb3c',
+    'gst-plugins-good': '81f63142d65b62b0971c19ceb79956c49ffc2f06',
+    'gst-plugins-ugly': '7d7c3e478e32b7b66c44cc4442d571fbab534740',
+    'gst-plugins-bad': 'ea6821e2934fe8d356ea89d5610f0630b3446877',
+    'gst-libav': '3c440154c60d1ec0a54186f0fad4aebfd2ecc3ea',
+    'gst-rtsp-server': '5029c85a46a8c366c4bf272d503e22bbcd624ece',
+    'gst-editing-services': 'ee8bf88ebf131cf7c7161356540efc20bf411e14',
+    'gst-python': 'b3e564eff577e2f577d795051bbcca85d47c89dc',
+    'gstreamer-vaapi': 'c89e9afc5d43837c498a55f8f13ddf235442b83b',
+    'gst-omx': 'd2463b017f222e678978582544a9c9a80edfd330',
+    'gst-devtools': 'da962d096af9460502843e41b7d25fdece7ff1c2',
+    'gstreamer-sharp': 'b94528f8e7979df49fedf137dfa228d8fe475e1b',
+}
+
+
+def print_help():
+    print('', file=sys.stderr)
+    print('gen-changelog: generate GNU-style changelog from git history',
+          file=sys.stderr)
+    print('', file=sys.stderr)
+    print('Usage: {} [OPTIONS] GSTREAMER-MODULE [START-TAG] [HEAD-TAG]'.format(
+        sys.argv[0]), file=sys.stderr)
+    print('', file=sys.stderr)
+    sys.exit(1)
+
+
+if len(sys.argv) < 2 or len(sys.argv) > 4 or '--help' in sys.argv:
+    print_help()
+
+module = sys.argv[1]
+
+if len(sys.argv) > 2:
+    start_tag = sys.argv[2]
+else:
+    start_tag = None
+
+if len(sys.argv) > 3:
+    head_tag = sys.argv[3]
+else:
+    head_tag = None
+
+if module not in changelog_starts:
+    print(f'Unknown module {module}', file=sys.stderr)
+    print_help()
+
+
+def process_commit(lines, files, subtree_path=None):
+    # DATE NAME
+    # BLANK LINE
+    # Subject
+    # BLANK LINE
+    # ...
+    # FILES
+    fileincommit = False
+    lines = [x.strip() for x in lines if x.strip()
+             and not x.startswith('git-svn-id')]
+    files = [x.strip() for x in files if x.strip()]
+    for line in lines:
+        if line.startswith('* ') and ':' in line:
+            fileincommit = True
+            break
+
+    top_line = lines[0]
+    print(top_line.strip())
+    print()
+    if not fileincommit:
+        for f in files:
+            if subtree_path and f.startswith(subtree_path):
+                # requires Python 3.9
+                print('\t* %s:' % f.removeprefix(subtree_path))
+            else:
+                print('\t* %s:' % f)
+    for line in lines[1:]:
+        print('\t ', line)
+    print()
+
+
+def output_commits(module, start_tag, end_tag, subtree_path=None):
+    # retrieve commit date for start tag so we can filter the log for commits
+    # after that date. That way we don't include commits from merged-in
+    # plugin-move branches that go back to the beginning of time.
+    start_date = get_commit_date_for_ref(start_tag)
+
+    cmd = ['git', 'log',
+           '--pretty=format:--START-COMMIT--%H%n%ai  %an <%ae>%n%n%s%n%b%n--END-COMMIT--',
+           '--date=short',
+           '--name-only',
+           f'--since={start_date}',
+           f'{start_tag}..{end_tag}',
+           ]
+
+    if subtree_path:
+        cmd += ['--', '.']
+
+    p = subprocess.Popen(args=cmd, shell=False,
+                         stdout=subprocess.PIPE, cwd=meson_source_root)
+    buf = []
+    files = []
+    filemode = False
+    for lin in [x.decode('utf8', errors='replace') for x in p.stdout.readlines()]:
+        if lin.startswith("--START-COMMIT--"):
+            commit_hash = lin[16:].strip()
+            if buf != []:
+                process_commit(buf, files, subtree_path)
+
+            if commit_hash in release_refs:
+                version_str = release_refs[commit_hash]
+                print(f'=== release {version_str} ===\n')
+
+            buf = []
+            files = []
+            filemode = False
+        elif lin.startswith("--END-COMMIT--"):
+            filemode = True
+        elif filemode is True:
+            files.append(lin)
+        else:
+            buf.append(lin)
+    if buf != []:
+        process_commit(buf, files, subtree_path)
+
+
+def get_commit_date_for_ref(ref):
+    cmd = ['git', 'log', '--pretty=format:%cI', '-1', ref]
+    r = subprocess.run(cmd, capture_output=True, text=True,
+                       check=True, cwd=meson_source_root)
+    commit_date = r.stdout.strip()
+    return commit_date
+
+
+def populate_release_tags_for_premonorepo_module(module_tag_prefix):
+    if module_tag_prefix != '':
+        cmd = ['git', 'tag', '--list', f'{module_tag_prefix}*']
+    else:
+        cmd = ['git', 'tag', '--list', '1.*', 'RELEASE-*']
+
+    p = subprocess.Popen(args=cmd, shell=False,
+                         stdout=subprocess.PIPE, cwd=meson_source_root)
+    for line in [x.decode('utf8') for x in p.stdout.readlines()]:
+        git_tag = line.strip()
+        version_str = git_tag.removeprefix(module_tag_prefix).removeprefix('RELEASE-').split('-')[0].replace('_', '.')
+        # might have been populated with post-monorepo tags already for gstreamer core
+        if version_str not in release_refs:
+            # find last commit before tag in module subdirectory
+            cmd = ['git', 'log', '--pretty=format:%H', '-1', git_tag]
+            r = subprocess.run(cmd, capture_output=True,
+                               text=True, check=True, cwd=meson_source_root)
+            commit_hash = r.stdout.strip()
+            release_refs[commit_hash] = version_str
+
+            # print(f'{git_tag} => {version_str} => {commit_hash}')
+
+
+def populate_release_tags_for_monorepo_subproject():
+    cmd = ['git', 'tag', '--list', '1.*']
+    p = subprocess.Popen(args=cmd, shell=False,
+                         stdout=subprocess.PIPE, cwd=meson_source_root)
+    for line in [x.decode('utf8') for x in p.stdout.readlines()]:
+        version_str = line.strip()
+        version_arr = version_str.split('.')
+        major = int(version_arr[0])
+        minor = int(version_arr[1])
+        micro = int(version_arr[2])
+        # ignore pre-monorepo versions
+        if major < 1:
+            continue
+        if major == 1 and minor < 19:
+            continue
+        if major == 1 and minor == 19 and micro < 2:
+            continue
+        # find last commit before tag in module subdirectory
+        cmd = ['git', 'log', '--pretty=format:%H',
+               '-1', version_str, '--', '.']
+        r = subprocess.run(cmd, capture_output=True, text=True,
+                           check=True, cwd=meson_source_root)
+        commit_hash = r.stdout.strip()
+        release_refs[commit_hash] = version_str
+
+
+if __name__ == '__main__':
+    module_tag_prefix = '' if module == 'gstreamer' else f'{module}-'
+
+    populate_release_tags_for_monorepo_subproject()
+
+    with open(output_fn, 'w') as f:
+        sys.stdout = f
+
+        # Force writing of head tag
+        if head_tag and head_tag not in release_refs.values():
+            print(f'=== release {head_tag} ===\n')
+
+        # Output all commits from start_tag onwards, otherwise output full history.
+        # (We assume the start_tag is after the monorepo merge if it's specified.)
+        if start_tag and start_tag != 'start':
+            output_commits(module, start_tag, 'HEAD', f'subprojects/{module}/')
+        else:
+            # First output all post-monorepo commits or commits from start_tag if specified
+            output_commits(module, 'monorepo-start',
+                           'HEAD', f'subprojects/{module}/')
+
+            populate_release_tags_for_premonorepo_module(module_tag_prefix)
+
+            # Next output all pre-monorepo commits (modules have their own root)
+            if not start_tag:
+                module_start = f'{module_tag_prefix}1.0.0'
+            elif start_tag == 'start':
+                module_start = changelog_starts[module]
+            else:
+                module_start = f'{module_tag_prefix}{start_tag}'
+
+            output_commits(module, module_start,
+                           f'{module_tag_prefix}1.19.2', None)
+
+        # Write start tag at end for clarity
+        if not start_tag:
+            print(f'=== release 1.0.0 ===\n')
+        elif start_tag != 'start':
+            print(f'=== release {start_tag} ===\n')
diff --git a/scripts/meson.build b/scripts/meson.build
new file mode 100644
index 0000000000..0554b17b6e
--- /dev/null
+++ b/scripts/meson.build
@@ -0,0 +1,53 @@
+fs = import('fs')
+
+# scripts checks
+release_modules = [
+  'gstreamer',
+  'gst-plugins-base',
+  'gst-plugins-good',
+  'gst-plugins-ugly',
+  'gst-plugins-bad',
+  'gst-libav',
+  'gst-rtsp-server',
+  'gst-editing-services',
+  'gst-devtools',
+  'gst-python',
+  'gstreamer-vaapi',
+  'gst-omx',
+  'gst-docs',
+  'gstreamer-sharp',
+]
+
+# Make sure the files are all identical to avoid divergence
+gen_cl_hash = fs.hash(files('gen-changelog.py'), 'md5')
+
+out_of_sync_list = []
+
+foreach m : release_modules
+  module_gen_cl_hash = fs.hash(f'../subprojects/@m@/scripts/gen-changelog.py', 'md5')
+
+  if module_gen_cl_hash != gen_cl_hash
+    out_of_sync_list += [f'subprojects/@m@/scripts/gen-changelog.py']
+  endif
+endforeach
+
+if out_of_sync_list.length() > 0
+  module_list = ' '.join(release_modules)
+  out_of_sync_msg = '\n          '.join(out_of_sync_list)
+  error('''
+
+    ==============================================================================================================
+
+        The following subproject scripts are out of sync with scripts/gen-changelog.py:
+
+          @0@
+
+        Run
+
+          for m in @1@; do cp scripts/gen-changelog.py subprojects/$m/scripts/gen-changelog.py; done 
+
+        from the top-level git source directory to sync them up.
+
+    ==============================================================================================================
+    '''.format(out_of_sync_msg, module_list))
+endif