cli: collect logs and apport subiquity support

cli/apport: collect-logs include subiquity logs config when present

Add support for both cloud-init collect-logs and apport to
include subiquity live installer artifacts if present to aid in
bug triage.

Apport integration to attach subiquity, curtin and
ubuntu-desktop-installer report keys when logs or
config files are present.

`ubuntu-bug cloud-init` will also automatically tag the
bug as curtin, subiquity or ubuntu-desktop-installer if
related logs are present in the bug report.

Additional collect-logs support to collect
/var/lib/cloud/data in the event that cloud-init is disabled
by systemd generator with /run/cloud-init/disabled flag.
In these situations we want to collect /var/lib/cloud/data
dir for more context on prior cloud-init behavior.

5 files changed, 295 insertions, 10 deletions

diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 7cba0bf5..7f3d6e5a 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -3,11 +3,17 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Cloud-init apport interface"""
+
 from cloudinit.cmd.devel import read_cfg_paths
+from cloudinit.cmd.devel.logs import (
+    INSTALLER_APPORT_FILES,
+    INSTALLER_APPORT_SENSITIVE_FILES,
+)
 
 try:
     from apport.hookutils import (
         attach_file,
+        attach_file_if_exists,
         attach_root_command_outputs,
         root_command_output,
     )
@@ -110,30 +116,57 @@ def attach_cloud_info(report, ui=None):
                 report["CloudName"] = "None"
 
 
+def attach_installer_files(report, ui=None):
+    """Attach any subiquity installer logs config.
+
+    To support decoupling apport integration from installer config/logs,
+    we eventually want to either source this function or APPORT_FILES
+    attribute from subiquity  and/or ubuntu-desktop-installer package-hooks
+    python modules.
+    """
+    for apport_file in INSTALLER_APPORT_FILES:
+        attach_file_if_exists(report, apport_file.path, apport_file.label)
+
+
 def attach_user_data(report, ui=None):
     """Optionally provide user-data if desired."""
     if ui:
         user_data_file = _get_user_data_file()
         prompt = (
-            "Your user-data or cloud-config file can optionally be provided"
-            " from {0} and could be useful to developers when addressing this"
-            " bug. Do you wish to attach user-data to this bug?".format(
-                user_data_file
-            )
+            "Your user-data, cloud-config or autoinstall files can optionally "
+            " be provided from {0} and could be useful to developers when"
+            " addressing this bug. Do you wish to attach user-data to this"
+            " bug?".format(user_data_file)
         )
         response = ui.yesno(prompt)
         if response is None:
             raise StopIteration  # User cancelled
         if response:
             attach_file(report, user_data_file, "user_data.txt")
+            for apport_file in INSTALLER_APPORT_SENSITIVE_FILES:
+                attach_file_if_exists(
+                    report, apport_file.path, apport_file.label
+                )
 
 
 def add_bug_tags(report):
     """Add any appropriate tags to the bug."""
+    new_tags = []
+    if report.get("CurtinError"):
+        new_tags.append("curtin")
+    if report.get("SubiquityLog"):
+        new_tags.append("subiquity")
     if "JournalErrors" in report.keys():
         errors = report["JournalErrors"]
         if "Breaking ordering cycle" in errors:
-            report["Tags"] = "systemd-ordering"
+            new_tags.append("systemd-ordering")
+    if report.get("UdiLog"):
+        new_tags.append("ubuntu-desktop-installer")
+    if new_tags:
+        report.setdefault("Tags", "")
+        if report["Tags"]:
+            report["Tags"] += " "
+        report["Tags"] += " ".join(new_tags)
 
 
 def add_info(report, ui):
@@ -151,6 +184,7 @@ def add_info(report, ui):
     attach_hwinfo(report, ui)
     attach_cloud_info(report, ui)
     attach_user_data(report, ui)
+    attach_installer_files(report, ui)
     add_bug_tags(report)
     return True
 
diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
index 69f4d931..385554d8 100755
--- a/cloudinit/cmd/devel/logs.py
+++ b/cloudinit/cmd/devel/logs.py
@@ -11,6 +11,8 @@ import os
 import shutil
 import sys
 from datetime import datetime
+from pathlib import Path
+from typing import NamedTuple
 
 from cloudinit.cmd.devel import read_cfg_paths
 from cloudinit.helpers import Paths
@@ -22,11 +24,46 @@ CLOUDINIT_LOGS = ["/var/log/cloud-init.log", "/var/log/cloud-init-output.log"]
 CLOUDINIT_RUN_DIR = "/run/cloud-init"
 
 
+class ApportFile(NamedTuple):
+    path: str
+    label: str
+
+
+INSTALLER_APPORT_SENSITIVE_FILES = [
+    ApportFile(
+        "/var/log/installer/autoinstall-user-data", "AutoInstallUserData"
+    ),
+    ApportFile("/autoinstall.yaml", "AutoInstallYAML"),
+    ApportFile("/etc/cloud/cloud.cfg.d/99-installer.cfg", "InstallerCloudCfg"),
+]
+
+INSTALLER_APPORT_FILES = [
+    ApportFile("/var/log/installer/ubuntu_desktop_installer.log", "UdiLog"),
+    ApportFile(
+        "/var/log/installer/subiquity-server-debug.log", "SubiquityLog"
+    ),
+    ApportFile(
+        "/var/log/installer/subiquity-client-debug.log", "SubiquityClientLog"
+    ),
+    ApportFile("/var/log/installer/curtin-install.log", "CurtinLog"),
+    ApportFile(
+        "/var/log/installer/subiquity-curtin-install.cfg", "CurtinConfig"
+    ),
+    ApportFile("/var/log/installer/curtin-error-logs.tar", "CurtinError"),
+    ApportFile("/var/log/installer/block/probe-data.json", "ProbeData"),
+]
+
+
 def _get_user_data_file() -> str:
     paths = read_cfg_paths()
     return paths.get_ipath_cur("userdata_raw")
 
 
+def _get_cloud_data_path() -> str:
+    paths = read_cfg_paths()
+    return paths.get_cpath("data")
+
+
 def get_parser(parser=None):
     """Build or extend and arg parser for collect-logs utility.
 
@@ -110,6 +147,21 @@ def _collect_file(path, out_dir, verbosity):
         _debug("file %s did not exist\n" % path, 2, verbosity)
 
 
+def collect_installer_logs(log_dir, include_userdata, verbosity):
+    """Obtain subiquity logs and config files."""
+    for src_file in INSTALLER_APPORT_FILES:
+        destination_dir = Path(log_dir + src_file.path).parent
+        if not destination_dir.exists():
+            ensure_dir(str(destination_dir))
+        _collect_file(src_file.path, str(destination_dir), verbosity)
+    if include_userdata:
+        for src_file in INSTALLER_APPORT_SENSITIVE_FILES:
+            destination_dir = Path(log_dir + src_file.path).parent
+            if not destination_dir.exists():
+                ensure_dir(str(destination_dir))
+            _collect_file(src_file.path, str(destination_dir), verbosity)
+
+
 def collect_logs(tarfile, include_userdata: bool, verbosity=0):
     """Collect all cloud-init logs and tar them up into the provided tarfile.
 
@@ -123,8 +175,7 @@ def collect_logs(tarfile, include_userdata: bool, verbosity=0):
         )
         return 1
     tarfile = os.path.abspath(tarfile)
-    date = datetime.utcnow().date().strftime("%Y-%m-%d")
-    log_dir = "cloud-init-logs-{0}".format(date)
+    log_dir = datetime.utcnow().date().strftime("cloud-init-logs-%Y-%m-%d")
     with tempdir(dir="/tmp") as tmp_dir:
         log_dir = os.path.join(tmp_dir, log_dir)
         version = _write_command_output_to_file(
@@ -160,6 +211,8 @@ def collect_logs(tarfile, include_userdata: bool, verbosity=0):
         if include_userdata:
             user_data_file = _get_user_data_file()
             _collect_file(user_data_file, log_dir, verbosity)
+        collect_installer_logs(log_dir, include_userdata, verbosity)
+
         run_dir = os.path.join(log_dir, "run")
         ensure_dir(run_dir)
         if os.path.exists(CLOUDINIT_RUN_DIR):
@@ -179,6 +232,14 @@ def collect_logs(tarfile, include_userdata: bool, verbosity=0):
                 1,
                 verbosity,
             )
+        if os.path.exists(os.path.join(CLOUDINIT_RUN_DIR, "disabled")):
+            # Fallback to grab previous cloud/data
+            cloud_data_dir = Path(_get_cloud_data_path())
+            if cloud_data_dir.exists():
+                shutil.copytree(
+                    str(cloud_data_dir),
+                    Path(log_dir + str(cloud_data_dir)),
+                )
         with chdir(tmp_dir):
             subp(["tar", "czvf", tarfile, log_dir.replace(tmp_dir + "/", "")])
     sys.stderr.write("Wrote %s\n" % tarfile)
diff --git a/tests/integration_tests/test_paths.py b/tests/integration_tests/test_paths.py
index 20392e35..14513c82 100644
--- a/tests/integration_tests/test_paths.py
+++ b/tests/integration_tests/test_paths.py
@@ -1,8 +1,14 @@
+import os
 import re
+from datetime import datetime
 from typing import Iterator
 
 import pytest
 
+from cloudinit.cmd.devel.logs import (
+    INSTALLER_APPORT_FILES,
+    INSTALLER_APPORT_SENSITIVE_FILES,
+)
 from tests.integration_tests.instances import IntegrationInstance
 from tests.integration_tests.util import verify_clean_log
 
@@ -43,12 +49,62 @@ class TestHonorCloudDir:
         assert f"{NEW_CLOUD_DIR}/instance/user-data.txt" in re.sub(
             r"\s+", "", help_result.stdout
         ), "user-data file not correctly render in collect-logs -h"
+
+        # Touch a couple of subiquity files to assert collected
+        installer_files = (
+            INSTALLER_APPORT_FILES[-1],
+            INSTALLER_APPORT_SENSITIVE_FILES[-1],
+        )
+
+        for apport_file in installer_files:
+            custom_client.execute(
+                f"mkdir -p {os.path.dirname(apport_file.path)}"
+            )
+            custom_client.execute(f"touch {apport_file.path}")
+
         collect_logs_result = custom_client.execute(
             "cloud-init collect-logs --include-userdata"
         )
         assert (
             collect_logs_result.ok
         ), f"collect-logs error: {collect_logs_result.stderr}"
+        found_logs = custom_client.execute(
+            "tar -tf cloud-init.tar.gz"
+        ).stdout.splitlines()
+        dirname = datetime.utcnow().date().strftime("cloud-init-logs-%Y-%m-%d")
+        expected_logs = [
+            f"{dirname}/",
+            f"{dirname}/cloud-init.log",
+            f"{dirname}/cloud-init-output.log",
+            f"{dirname}/dmesg.txt",
+            f"{dirname}/user-data.txt",
+            f"{dirname}/version",
+            f"{dirname}/dpkg-version",
+            f"{dirname}/journal.txt",
+            f"{dirname}/run/",
+            f"{dirname}/run/cloud-init/",
+            f"{dirname}/run/cloud-init/result.json",
+            f"{dirname}/run/cloud-init/.instance-id",
+            f"{dirname}/run/cloud-init/cloud-init-generator.log",
+            f"{dirname}/run/cloud-init/enabled",
+            f"{dirname}/run/cloud-init/cloud-id",
+            f"{dirname}/run/cloud-init/instance-data.json",
+            f"{dirname}/run/cloud-init/instance-data-sensitive.json",
+            f"{dirname}{installer_files[0].path}",
+            f"{dirname}{installer_files[1].path}",
+        ]
+        for log in expected_logs:
+            assert log in found_logs
+        # Assert disabled cloud-init collect-logs grabs /var/lib/cloud/data
+        custom_client.execute("touch /run/cloud-init/disabled")
+        assert custom_client.execute(
+            "cloud-init collect-logs --include-userdata"
+        ).ok
+        found_logs = custom_client.execute(
+            "tar -tf cloud-init.tar.gz"
+        ).stdout.splitlines()
+        dirname = datetime.utcnow().date().strftime("cloud-init-logs-%Y-%m-%d")
+        assert f"{dirname}/new-cloud-dir/data/result.json" in found_logs
 
     # LXD inserts some agent setup code into VMs on Bionic under
     # /var/lib/cloud. The inserted script will cause this test to fail
diff --git a/tests/unittests/cmd/devel/test_logs.py b/tests/unittests/cmd/devel/test_logs.py
index 0a418618..4e3f30d4 100644
--- a/tests/unittests/cmd/devel/test_logs.py
+++ b/tests/unittests/cmd/devel/test_logs.py
@@ -1,13 +1,17 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import glob
 import os
 import re
 from datetime import datetime
 from io import StringIO
 
+import pytest
+
 from cloudinit.cmd.devel import logs
+from cloudinit.cmd.devel.logs import ApportFile
 from cloudinit.subp import subp
-from cloudinit.util import load_file, write_file
+from cloudinit.util import ensure_dir, load_file, write_file
 from tests.unittests.helpers import mock
 
 M_PATH = "cloudinit.cmd.devel.logs."
@@ -81,6 +85,8 @@ class TestCollectLogs:
         mocker.patch(M_PATH + "sys.stderr", fake_stderr)
         mocker.patch(M_PATH + "CLOUDINIT_LOGS", [log1, log2])
         mocker.patch(M_PATH + "CLOUDINIT_RUN_DIR", run_dir)
+        mocker.patch(M_PATH + "INSTALLER_APPORT_FILES", [])
+        mocker.patch(M_PATH + "INSTALLER_APPORT_SENSITIVE_FILES", [])
         logs.collect_logs(output_tarfile, include_userdata=False)
         # unpack the tarfile and check file contents
         subp(["tar", "zxvf", output_tarfile, "-C", str(tmpdir)])
@@ -168,6 +174,8 @@ class TestCollectLogs:
         mocker.patch(M_PATH + "sys.stderr", fake_stderr)
         mocker.patch(M_PATH + "CLOUDINIT_LOGS", [log1, log2])
         mocker.patch(M_PATH + "CLOUDINIT_RUN_DIR", run_dir)
+        mocker.patch(M_PATH + "INSTALLER_APPORT_FILES", [])
+        mocker.patch(M_PATH + "INSTALLER_APPORT_SENSITIVE_FILES", [])
         mocker.patch(M_PATH + "_get_user_data_file", return_value=userdata)
         logs.collect_logs(output_tarfile, include_userdata=True)
         # unpack the tarfile and check file contents
@@ -187,6 +195,95 @@ class TestCollectLogs:
         fake_stderr.write.assert_any_call("Wrote %s\n" % output_tarfile)
 
 
+class TestCollectInstallerLogs:
+    @pytest.mark.parametrize(
+        "include_userdata, apport_files, apport_sensitive_files",
+        (
+            pytest.param(True, [], [], id="no_files_include_userdata"),
+            pytest.param(False, [], [], id="no_files_exclude_userdata"),
+            pytest.param(
+                True,
+                (ApportFile("log1", "Label1"), ApportFile("log2", "Label2")),
+                (
+                    ApportFile("private1", "LabelPrivate1"),
+                    ApportFile("private2", "PrivateLabel2"),
+                ),
+                id="files_and_dirs_include_userdata",
+            ),
+            pytest.param(
+                False,
+                (ApportFile("log1", "Label1"), ApportFile("log2", "Label2")),
+                (
+                    ApportFile("private1", "LabelPrivate1"),
+                    ApportFile("private2", "PrivateLabel2"),
+                ),
+                id="files_and_dirs_exclude_userdata",
+            ),
+        ),
+    )
+    def test_include_installer_logs_when_present(
+        self,
+        include_userdata,
+        apport_files,
+        apport_sensitive_files,
+        tmpdir,
+        mocker,
+    ):
+        src_dir = tmpdir.join("src")
+        ensure_dir(src_dir.strpath)
+        # collect-logs nests full directory path to file in the tarfile
+        destination_dir = tmpdir.join(src_dir)
+
+        # Create tmppath-based userdata_files, installer_logs, installer_dirs
+        expected_files = []
+        # Create last file in list to assert ignoring absent files
+        apport_files = [
+            logs.ApportFile(src_dir.join(apport.path).strpath, apport.label)
+            for apport in apport_files
+        ]
+        if apport_files:
+            write_file(apport_files[-1].path, apport_files[-1].label)
+            expected_files += [
+                destination_dir.join(
+                    os.path.basename(apport_files[-1].path)
+                ).strpath
+            ]
+        apport_sensitive_files = [
+            logs.ApportFile(src_dir.join(apport.path).strpath, apport.label)
+            for apport in apport_sensitive_files
+        ]
+        if apport_sensitive_files:
+            write_file(
+                apport_sensitive_files[-1].path,
+                apport_sensitive_files[-1].label,
+            )
+            if include_userdata:
+                expected_files += [
+                    destination_dir.join(
+                        os.path.basename(apport_sensitive_files[-1].path)
+                    ).strpath
+                ]
+        mocker.patch(M_PATH + "INSTALLER_APPORT_FILES", apport_files)
+        mocker.patch(
+            M_PATH + "INSTALLER_APPORT_SENSITIVE_FILES", apport_sensitive_files
+        )
+        logs.collect_installer_logs(
+            log_dir=tmpdir.strpath,
+            include_userdata=include_userdata,
+            verbosity=0,
+        )
+        expect_userdata = bool(include_userdata and apport_sensitive_files)
+        # when subiquity artifacts exist, and userdata set true, expect logs
+        expect_subiquity_logs = any([apport_files, expect_userdata])
+        if expect_subiquity_logs:
+            assert destination_dir.exists(), "Missing subiquity artifact dir"
+            assert sorted(expected_files) == sorted(
+                glob.glob(f"{destination_dir.strpath}/*")
+            )
+        else:
+            assert not destination_dir.exists(), "Unexpected subiquity dir"
+
+
 class TestParser:
     def test_parser_help_has_userdata_file(self, mocker, tmpdir):
         userdata = str(tmpdir.join("user-data.txt"))
diff --git a/tests/unittests/test_apport.py b/tests/unittests/test_apport.py
index a2c866b9..1876c1be 100644
--- a/tests/unittests/test_apport.py
+++ b/tests/unittests/test_apport.py
@@ -1,3 +1,5 @@
+import pytest
+
 from tests.unittests.helpers import mock
 
 M_PATH = "cloudinit.apport."
@@ -19,5 +21,40 @@ class TestApport:
         report = object()
         apport.attach_user_data(report, ui)
         assert [
-            mock.call(report, user_data_file, "user_data.txt")
+            mock.call(report, user_data_file, "user_data.txt"),
         ] == m_hookutils.attach_file.call_args_list
+        assert [
+            mock.call(
+                report,
+                "/var/log/installer/autoinstall-user-data",
+                "AutoInstallUserData",
+            ),
+            mock.call(report, "/autoinstall.yaml", "AutoInstallYAML"),
+            mock.call(
+                report,
+                "/etc/cloud/cloud.cfg.d/99-installer.cfg",
+                "InstallerCloudCfg",
+            ),
+        ] == m_hookutils.attach_file_if_exists.call_args_list
+
+    @pytest.mark.parametrize(
+        "report,tags",
+        (
+            ({"Irrelevant": "."}, ""),
+            ({"UdiLog": "."}, "ubuntu-desktop-installer"),
+            ({"CurtinError": ".", "SubiquityLog": "."}, "curtin subiquity"),
+            (
+                {
+                    "UdiLog": ".",
+                    "JournalErrors": "...Breaking ordering cycle...",
+                },
+                "systemd-ordering ubuntu-desktop-installer",
+            ),
+        ),
+    )
+    def test_add_bug_tags_assigns_proper_tags(self, report, tags):
+        """Tags are assigned based on non-empty project report key values."""
+        from cloudinit import apport
+
+        apport.add_bug_tags(report)
+        assert report.get("Tags", "") == tags