dpkg-build.py: Add a build element for debian sources

It produces an artifact, plus write public data to provide enough
information to build a debian package out of the artifact.

2 files changed, 280 insertions, 0 deletions

diff --git a/buildstream/plugins/elements/dpkg_build.py b/buildstream/plugins/elements/dpkg_build.py
new file mode 100644
index 000000000..ecbecc256
--- /dev/null
+++ b/buildstream/plugins/elements/dpkg_build.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+"""Dpkg build element
+
+A :mod:`BuildElement <buildstream.buildelement>` implementation for using
+dpkg elements
+
+Default Configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+The dpkg default configuration:
+  .. literalinclude:: ../../../buildstream/plugins/elements/dpkg_build.yaml
+     :language: yaml
+
+Public data
+~~~~~~~~~~~
+
+This plugin writes to an element's public data.
+
+split-rules
+-----------
+
+This plugin overwrites the element's split-rules with a list of its own
+creation, creating a split domain for every package that it detected.
+e.g.
+
+.. code:: yaml
+
+   public:
+     split-rules:
+       foo:
+       - /sbin/foo
+       - /usr/bin/bar
+       bar:
+       - /etc/quux
+
+dpkg-data
+---------
+
+control
+'''''''
+
+The control file will be written as raw text into the control field.
+e.g.
+
+.. code:: yaml
+
+   public:
+     dpkg-data:
+       foo:
+         control: |
+           Source: foo
+           Section: blah
+           Build-depends: bar (>= 1337), baz
+           ...
+
+name
+''''
+
+The name of the plugin will be written to the name field.
+e.g.
+
+.. code:: yaml
+
+   public:
+     dpkg-data:
+       foo:
+         name: foobar
+
+package-scripts
+---------------
+
+preinstall, postinstall, prerm and postrm scripts may be written to the
+package if they are detected. They are written as raw text. e.g.
+
+.. code:: yaml
+
+   public:
+     package-scripts:
+       foo:
+         preinstall: |
+           #!/usr/bin/bash
+           /sbin/ldconfig
+       bar:
+         postinstall: |
+           #!/usr/bin/bash
+           /usr/share/fonts/generate_fonts.sh
+
+"""
+
+import filecmp
+import os
+import re
+
+from buildstream import BuildElement, utils
+
+
+# Element implementation for the 'dpkg' kind.
+class DpkgElement(BuildElement):
+    def _get_packages(self, sandbox):
+        controlfile = os.path.join("debian", "control")
+        controlpath = os.path.join(
+            sandbox.get_directory(),
+            self.get_variable('build-root').lstrip(os.sep),
+            controlfile
+        )
+        with open(controlpath) as f:
+            return re.findall(r"Package:\s*(.+)\n", f.read())
+
+    def assemble(self, sandbox):
+        # Replace %{packages} if no variable was set
+        packages = self._get_packages(sandbox)
+        self.commands = dict([
+            (group, [
+                c.replace("<PACKAGES>", " ".join(packages)) for c in commands
+            ])
+            for group, commands in self.commands.items()
+        ])
+
+        collectdir = super().assemble(sandbox)
+
+        bad_overlaps = set()
+        new_split_rules = {}
+        new_dpkg_data = {}
+        new_package_scripts = {}
+        for package in packages:
+            package_path = os.path.join(sandbox.get_directory(),
+                                        self.get_variable('build-root').lstrip(os.sep),
+                                        'debian', package)
+
+            # Exclude DEBIAN files because they're pulled in as public metadata
+            contents = [x for x in utils.list_relative_paths(package_path)
+                        if x != "." and not x.startswith("DEBIAN")]
+            new_split_rules[package] = contents
+
+            # Check for any overlapping files that are different.
+            # Since we're storing all these files together, we need to warn
+            # because clobbering is bad!
+            for content_file in contents:
+                for split_package, split_contents in new_split_rules.items():
+                    for split_file in split_contents:
+                        content_file_path = os.path.join(package_path,
+                                                         content_file.lstrip(os.sep))
+                        split_file_path = os.path.join(os.path.dirname(package_path),
+                                                       split_package,
+                                                       split_file.lstrip(os.sep))
+                        if (content_file == split_file and
+                                os.path.isfile(content_file_path) and
+                                not filecmp.cmp(content_file_path, split_file_path)):
+                            bad_overlaps.add(content_file)
+
+            # Store /DEBIAN metadata for each package.
+            # DEBIAN/control goes into bst.dpkg-data.<package>.control
+            controlpath = os.path.join(package_path, "DEBIAN", "control")
+            if not os.path.exists(controlpath):
+                self.error("{}: package {} doesn't have a DEBIAN/control in {}!"
+                           .format(self.name, package, package_path))
+            with open(controlpath, "r") as f:
+                controldata = f.read()
+            new_dpkg_data[package] = {"control": controldata, "name": package}
+
+            # DEBIAN/{pre,post}{inst,rm} scripts go into bst.package-scripts.<package>.<script>
+            scriptfiles = ["preinst", "postinst", "prerm", "postrm"]
+            for s in scriptfiles:
+                path = os.path.join(package_path, "DEBIAN", s)
+                if os.path.exists(path):
+                    if package not in new_package_scripts:
+                        new_package_scripts[package] = {}
+                    with open(path, "r") as f:
+                        data = f.read()
+                    new_package_scripts[package][s] = data
+
+        bstdata = self.get_public_data("bst")
+        bstdata["split-rules"] = new_split_rules
+        bstdata["dpkg-data"] = new_dpkg_data
+        if new_package_scripts:
+            bstdata["package-scripts"] = new_package_scripts
+
+        self.set_public_data("bst", bstdata)
+
+        if bad_overlaps:
+            self.warn("Destructive overlaps found in some files!", "\n".join(bad_overlaps))
+
+        return collectdir
+
+
+# Plugin entry point
+def setup():
+    return DpkgElement
diff --git a/buildstream/plugins/elements/dpkg_build.yaml b/buildstream/plugins/elements/dpkg_build.yaml
new file mode 100644
index 000000000..5d7431811
--- /dev/null
+++ b/buildstream/plugins/elements/dpkg_build.yaml
@@ -0,0 +1,73 @@
+# Dpkg default configurations
+
+variables:
+
+  rulesfile: "debian/rules"
+  build: "%{rulesfile} build"
+  binary: "env DH_OPTIONS='--destdir=.' %{rulesfile} binary"
+
+  # packages' default value will be automatically replaced with
+  # defaults calculated from debian/control. Replace this with a
+  # space-separated list of packages to have more control over
+  # what gets generated.
+  #
+  # e.g.
+  # packages: "foo foo-dev foo-doc"
+  #
+  packages: <PACKAGES>
+  install-packages: |
+    for pkg in %{packages}; do
+      cp -a debian/${pkg}/* %{install-root}
+    done
+  patch: |
+    if grep -q "3.0 (quilt)" debian/source/format; then
+      quilt push -a
+    fi
+
+  # Set this if the sources cannot handle parallelization.
+  #
+  # notparallel: True
+
+config:
+
+  # Commands for configuring the software
+  #
+  configure-commands:
+  - |
+    %{patch}
+
+  # Commands for building the software
+  #
+  build-commands:
+  - |
+    %{build}
+  - |
+    %{binary}
+
+  # Commands for installing the software into a
+  # destination folder
+  #
+  install-commands:
+  - |
+    %{install-packages}
+
+  # Commands for stripping debugging information out of
+  # installed binaries
+  #
+  strip-commands:
+  - |
+    %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+  MAKEFLAGS: -j%{max-jobs}
+  V: 1
+  DH_VERBOSE: 1
+  QUILT_PATCHES: debian/patches
+
+# And dont consider MAKEFLAGS or V as something which may
+# effect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
+- DH_VERBOSE