21.1-19-gbad84ad4-0ubuntu3 (patches unapplied)

Imported using git-ubuntu import.

4 files changed, 734 insertions, 0 deletions

diff --git a/debian/changelog b/debian/changelog
index a46efadd..008420de 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+cloud-init (21.1-19-gbad84ad4-0ubuntu3) hirsute; urgency=medium
+
+  * cherry-pick 83f6bbfb: Fix unpickle for source paths missing run_dir
+    (#863) (LP: #1899299)
+  * cherry-pick d132356c: fix error on upgrade caused by new vendordata2
+    attributes (LP: #1922739)
+
+ -- James Falcon <james.falcon@canonical.com>  Mon, 19 Apr 2021 13:41:21 -0500
+
 cloud-init (21.1-19-gbad84ad4-0ubuntu2) hirsute; urgency=medium
 
   * debian/cloud-init.templates: enable UpCloud by default
diff --git a/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863 b/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863
new file mode 100644
index 00000000..d80de236
--- /dev/null
+++ b/debian/patches/cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863
@@ -0,0 +1,576 @@
+From 83f6bbfbe5b924be61a3c098f4202377d69c8947 Mon Sep 17 00:00:00 2001
+From: lucasmoura <lucas.moura@canonical.com>
+Date: Mon, 12 Apr 2021 13:22:22 -0300
+Subject: [PATCH] Fix unpickle for source paths missing run_dir (#863)
+
+On the datasource class, we require the use of paths.run_dir to
+perform some operations. On older cloud-init version, the
+Paths class does not have the run_dir attribute. To fix that,
+we are now manually adding that attribute in the Paths
+object if doesn't exist in the unpickle operation.
+
+LP: #1899299
+---
+ cloudinit/helpers.py                          |  17 +-
+ cloudinit/tests/test_upgrade.py               |   3 +
+ .../data/old_pickles/trusty-14.04.1-0.7.5.pkl | 504 ++++++++++++++++++
+ 3 files changed, 523 insertions(+), 1 deletion(-)
+ create mode 100644 tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl
+
+--- a/cloudinit/helpers.py
++++ b/cloudinit/helpers.py
+@@ -20,6 +20,7 @@ from cloudinit.settings import (PER_INST
+ 
+ from cloudinit import log as logging
+ from cloudinit import type_utils
++from cloudinit import persistence
+ from cloudinit import util
+ 
+ LOG = logging.getLogger(__name__)
+@@ -317,7 +318,9 @@ class ContentHandlers(object):
+         return list(self.registered.items())
+ 
+ 
+-class Paths(object):
++class Paths(persistence.CloudInitPickleMixin):
++    _ci_pkl_version = 1
++
+     def __init__(self, path_cfgs, ds=None):
+         self.cfgs = path_cfgs
+         # Populate all the initial paths
+@@ -354,6 +357,18 @@ class Paths(object):
+         # Set when a datasource becomes active
+         self.datasource = ds
+ 
++    def _unpickle(self, ci_pkl_version: int) -> None:
++        """Perform deserialization fixes for Paths."""
++        if not hasattr(self, "run_dir"):
++            # On older versions of cloud-init the Paths class do not
++            # have the run_dir attribute. This is problematic because
++            # when loading the pickle object on newer versions of cloud-init
++            # we will rely on this attribute. To fix that, we are now
++            # manually adding that attribute here.
++            self.run_dir = Paths(
++                path_cfgs=self.cfgs,
++                ds=self.datasource).run_dir
++
+     # get_ipath_cur: get the current instance path for an item
+     def get_ipath_cur(self, name=None):
+         return self._get_path(self.instance_link, name)
+--- a/cloudinit/tests/test_upgrade.py
++++ b/cloudinit/tests/test_upgrade.py
+@@ -43,3 +43,6 @@ class TestUpgrade:
+     def test_blacklist_drivers_set_on_networking(self, previous_obj_pkl):
+         """We always expect Networking.blacklist_drivers to be initialised."""
+         assert previous_obj_pkl.distro.networking.blacklist_drivers is None
++
++    def test_paths_has_run_dir_attribute(self, previous_obj_pkl):
++        assert previous_obj_pkl.paths.run_dir is not None
+--- /dev/null
++++ b/tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl
+@@ -0,0 +1,504 @@
++ccopy_reg
++_reconstructor
++p1
++(ccloudinit.sources.DataSourceNoCloud
++DataSourceNoCloudNet
++p2
++c__builtin__
++object
++p3
++NtRp4
++(dp5
++S'paths'
++p6
++g1
++(ccloudinit.helpers
++Paths
++p7
++g3
++NtRp8
++(dp9
++S'lookups'
++p10
++(dp11
++S'cloud_config'
++p12
++S'cloud-config.txt'
++p13
++sS'userdata'
++p14
++S'user-data.txt.i'
++p15
++sS'vendordata'
++p16
++S'vendor-data.txt.i'
++p17
++sS'userdata_raw'
++p18
++S'user-data.txt'
++p19
++sS'boothooks'
++p20
++g20
++sS'scripts'
++p21
++g21
++sS'sem'
++p22
++g22
++sS'data'
++p23
++g23
++sS'vendor_scripts'
++p24
++S'scripts/vendor'
++p25
++sS'handlers'
++p26
++g26
++sS'obj_pkl'
++p27
++S'obj.pkl'
++p28
++sS'vendordata_raw'
++p29
++S'vendor-data.txt'
++p30
++sS'vendor_cloud_config'
++p31
++S'vendor-cloud-config.txt'
++p32
++ssS'template_tpl'
++p33
++S'/etc/cloud/templates/%s.tmpl'
++p34
++sS'cfgs'
++p35
++(dp36
++S'cloud_dir'
++p37
++S'/var/lib/cloud/'
++p38
++sS'templates_dir'
++p39
++S'/etc/cloud/templates/'
++p40
++sS'upstart_dir'
++p41
++S'/etc/init/'
++p42
++ssS'cloud_dir'
++p43
++g38
++sS'datasource'
++p44
++NsS'upstart_conf_d'
++p45
++g42
++sS'boot_finished'
++p46
++S'/var/lib/cloud/instance/boot-finished'
++p47
++sS'instance_link'
++p48
++S'/var/lib/cloud/instance'
++p49
++sS'seed_dir'
++p50
++S'/var/lib/cloud/seed'
++p51
++sbsS'supported_seed_starts'
++p52
++(S'http://'
++S'https://'
++S'ftp://'
++tp53
++sS'sys_cfg'
++p54
++(dp55
++S'output'
++p56
++(dp57
++S'all'
++p58
++S'| tee -a /var/log/cloud-init-output.log'
++p59
++ssS'users'
++p60
++(lp61
++S'default'
++p62
++asS'def_log_file'
++p63
++S'/var/log/cloud-init.log'
++p64
++sS'cloud_final_modules'
++p65
++(lp66
++S'rightscale_userdata'
++p67
++aS'scripts-vendor'
++p68
++aS'scripts-per-once'
++p69
++aS'scripts-per-boot'
++p70
++aS'scripts-per-instance'
++p71
++aS'scripts-user'
++p72
++aS'ssh-authkey-fingerprints'
++p73
++aS'keys-to-console'
++p74
++aS'phone-home'
++p75
++aS'final-message'
++p76
++aS'power-state-change'
++p77
++asS'disable_root'
++p78
++I01
++sS'syslog_fix_perms'
++p79
++S'syslog:adm'
++p80
++sS'log_cfgs'
++p81
++(lp82
++(lp83
++S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n'
++p84
++aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n'
++p85
++aa(lp86
++g84
++aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n"
++p87
++aasS'cloud_init_modules'
++p88
++(lp89
++S'migrator'
++p90
++aS'seed_random'
++p91
++aS'bootcmd'
++p92
++aS'write-files'
++p93
++aS'growpart'
++p94
++aS'resizefs'
++p95
++aS'set_hostname'
++p96
++aS'update_hostname'
++p97
++aS'update_etc_hosts'
++p98
++aS'ca-certs'
++p99
++aS'rsyslog'
++p100
++aS'users-groups'
++p101
++aS'ssh'
++p102
++asS'preserve_hostname'
++p103
++I00
++sS'_log'
++p104
++(lp105
++g84
++ag87
++ag85
++asS'datasource_list'
++p106
++(lp107
++S'NoCloud'
++p108
++aS'ConfigDrive'
++p109
++aS'OpenNebula'
++p110
++aS'Azure'
++p111
++aS'AltCloud'
++p112
++aS'OVF'
++p113
++aS'MAAS'
++p114
++aS'GCE'
++p115
++aS'OpenStack'
++p116
++aS'CloudSigma'
++p117
++aS'Ec2'
++p118
++aS'CloudStack'
++p119
++aS'SmartOS'
++p120
++aS'None'
++p121
++asS'vendor_data'
++p122
++(dp123
++S'prefix'
++p124
++(lp125
++sS'enabled'
++p126
++I01
++ssS'cloud_config_modules'
++p127
++(lp128
++S'emit_upstart'
++p129
++aS'disk_setup'
++p130
++aS'mounts'
++p131
++aS'ssh-import-id'
++p132
++aS'locale'
++p133
++aS'set-passwords'
++p134
++aS'grub-dpkg'
++p135
++aS'apt-pipelining'
++p136
++aS'apt-configure'
++p137
++aS'package-update-upgrade-install'
++p138
++aS'landscape'
++p139
++aS'timezone'
++p140
++aS'puppet'
++p141
++aS'chef'
++p142
++aS'salt-minion'
++p143
++aS'mcollective'
++p144
++aS'disable-ec2-metadata'
++p145
++aS'runcmd'
++p146
++aS'byobu'
++p147
++assg14
++Nsg16
++Nsg18
++S'#cloud-config\n{}\n\n'
++p148
++sg29
++S'#cloud-config\n{}\n\n'
++p149
++sS'dsmode'
++p150
++S'net'
++p151
++sS'seed'
++p152
++S'/var/lib/cloud/seed/nocloud-net'
++p153
++sS'cmdline_id'
++p154
++S'ds=nocloud-net'
++p155
++sS'ud_proc'
++p156
++g1
++(ccloudinit.user_data
++UserDataProcessor
++p157
++g3
++NtRp158
++(dp159
++g6
++g8
++sS'ssl_details'
++p160
++(dp161
++sbsg50
++g153
++sS'ds_cfg'
++p162
++(dp163
++sS'distro'
++p164
++g1
++(ccloudinit.distros.ubuntu
++Distro
++p165
++g3
++NtRp166
++(dp167
++S'osfamily'
++p168
++S'debian'
++p169
++sS'_paths'
++p170
++g8
++sS'name'
++p171
++S'ubuntu'
++p172
++sS'_runner'
++p173
++g1
++(ccloudinit.helpers
++Runners
++p174
++g3
++NtRp175
++(dp176
++g6
++g8
++sS'sems'
++p177
++(dp178
++sbsS'_cfg'
++p179
++(dp180
++S'paths'
++p181
++(dp182
++g37
++g38
++sg39
++g40
++sg41
++g42
++ssS'default_user'
++p183
++(dp184
++S'shell'
++p185
++S'/bin/bash'
++p186
++sS'name'
++p187
++S'ubuntu'
++p188
++sS'sudo'
++p189
++(lp190
++S'ALL=(ALL) NOPASSWD:ALL'
++p191
++asS'lock_passwd'
++p192
++I01
++sS'gecos'
++p193
++S'Ubuntu'
++p194
++sS'groups'
++p195
++(lp196
++S'adm'
++p197
++aS'audio'
++p198
++aS'cdrom'
++p199
++aS'dialout'
++p200
++aS'dip'
++p201
++aS'floppy'
++p202
++aS'netdev'
++p203
++aS'plugdev'
++p204
++aS'sudo'
++p205
++aS'video'
++p206
++assS'package_mirrors'
++p207
++(lp208
++(dp209
++S'arches'
++p210
++(lp211
++S'i386'
++p212
++aS'amd64'
++p213
++asS'failsafe'
++p214
++(dp215
++S'security'
++p216
++S'http://security.ubuntu.com/ubuntu'
++p217
++sS'primary'
++p218
++S'http://archive.ubuntu.com/ubuntu'
++p219
++ssS'search'
++p220
++(dp221
++S'security'
++p222
++(lp223
++sS'primary'
++p224
++(lp225
++S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/'
++p226
++aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/'
++p227
++aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/'
++p228
++assa(dp229
++S'arches'
++p230
++(lp231
++S'armhf'
++p232
++aS'armel'
++p233
++aS'default'
++p234
++asS'failsafe'
++p235
++(dp236
++S'security'
++p237
++S'http://ports.ubuntu.com/ubuntu-ports'
++p238
++sS'primary'
++p239
++S'http://ports.ubuntu.com/ubuntu-ports'
++p240
++ssasS'ssh_svcname'
++p241
++S'ssh'
++p242
++ssbsS'metadata'
++p243
++(dp244
++g150
++g151
++sS'local-hostname'
++p245
++S'trusty-upgrade2'
++p246
++sS'instance-id'
++p247
++S'trusty-upgrade2'
++p248
++ssb.
+\ No newline at end of file
diff --git a/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2 b/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2
new file mode 100644
index 00000000..7bd48fe7
--- /dev/null
+++ b/debian/patches/cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2
@@ -0,0 +1,147 @@
+From d132356cc361abef2d90d4073438f3ab759d5964 Mon Sep 17 00:00:00 2001
+From: James Falcon <TheRealFalcon@users.noreply.github.com>
+Date: Mon, 19 Apr 2021 11:31:28 -0500
+Subject: [PATCH] fix error on upgrade caused by new vendordata2 attributes
+ (#869)
+
+In #777, we added 'vendordata2' and 'vendordata2_raw' attributes to
+the DataSource class, but didn't use the upgrade framework to deal
+with an unpickle after upgrade. This commit adds the necessary
+upgrade code.
+
+Additionally, added a smaller-scope upgrade test to our integration
+tests that will be run on every CI run so we catch these issues
+immediately in the future.
+
+LP: #1922739
+---
+ cloudinit/sources/__init__.py           | 12 +++++++++++-
+ cloudinit/tests/test_upgrade.py         |  4 ++++
+ tests/integration_tests/clouds.py       |  4 ++--
+ tests/integration_tests/test_upgrade.py | 25 ++++++++++++++++++++++++-
+ 4 files changed, 41 insertions(+), 4 deletions(-)
+
+--- a/cloudinit/sources/__init__.py
++++ b/cloudinit/sources/__init__.py
+@@ -24,6 +24,7 @@ from cloudinit import util
+ from cloudinit.atomic_helper import write_json
+ from cloudinit.event import EventType
+ from cloudinit.filters import launch_index
++from cloudinit.persistence import CloudInitPickleMixin
+ from cloudinit.reporting import events
+ 
+ DSMODE_DISABLED = "disabled"
+@@ -134,7 +135,7 @@ URLParams = namedtuple(
+     'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries'])
+ 
+ 
+-class DataSource(metaclass=abc.ABCMeta):
++class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
+ 
+     dsmode = DSMODE_NETWORK
+     default_locale = 'en_US.UTF-8'
+@@ -196,6 +197,8 @@ class DataSource(metaclass=abc.ABCMeta):
+     # non-root users
+     sensitive_metadata_keys = ('merged_cfg', 'security-credentials',)
+ 
++    _ci_pkl_version = 1
++
+     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
+         self.sys_cfg = sys_cfg
+         self.distro = distro
+@@ -218,6 +221,13 @@ class DataSource(metaclass=abc.ABCMeta):
+         else:
+             self.ud_proc = ud_proc
+ 
++    def _unpickle(self, ci_pkl_version: int) -> None:
++        """Perform deserialization fixes for Paths."""
++        if not hasattr(self, 'vendordata2'):
++            self.vendordata2 = None
++        if not hasattr(self, 'vendordata2_raw'):
++            self.vendordata2_raw = None
++
+     def __str__(self):
+         return type_utils.obj_name(self)
+ 
+--- a/cloudinit/tests/test_upgrade.py
++++ b/cloudinit/tests/test_upgrade.py
+@@ -46,3 +46,7 @@ class TestUpgrade:
+ 
+     def test_paths_has_run_dir_attribute(self, previous_obj_pkl):
+         assert previous_obj_pkl.paths.run_dir is not None
++
++    def test_vendordata_exists(self, previous_obj_pkl):
++        assert previous_obj_pkl.vendordata2 is None
++        assert previous_obj_pkl.vendordata2_raw is None
+--- a/tests/integration_tests/clouds.py
++++ b/tests/integration_tests/clouds.py
+@@ -110,14 +110,14 @@ class IntegrationCloud(ABC):
+             # Even if we're using the default key, it may still have a
+             # different name in the clouds, so we need to set it separately.
+             self.cloud_instance.key_pair.name = settings.KEYPAIR_NAME
+-        self._released_image_id = self._get_initial_image()
++        self.released_image_id = self._get_initial_image()
+         self.snapshot_id = None
+ 
+     @property
+     def image_id(self):
+         if self.snapshot_id:
+             return self.snapshot_id
+-        return self._released_image_id
++        return self.released_image_id
+ 
+     def emit_settings_to_log(self) -> None:
+         log.info(
+--- a/tests/integration_tests/test_upgrade.py
++++ b/tests/integration_tests/test_upgrade.py
+@@ -1,4 +1,5 @@
+ import logging
++import os
+ import pytest
+ import time
+ from pathlib import Path
+@@ -8,6 +9,8 @@ from tests.integration_tests.conftest im
+     get_validated_source,
+     session_start_time,
+ )
++from tests.integration_tests.instances import CloudInitSource
++
+ 
+ log = logging.getLogger('integration_testing')
+ 
+@@ -63,7 +66,7 @@ def test_upgrade(session_cloud: Integrat
+         return  # type checking doesn't understand that skip raises
+ 
+     launch_kwargs = {
+-        'image_id': session_cloud._get_initial_image(),
++        'image_id': session_cloud.released_image_id,
+     }
+ 
+     image = ImageSpecification.from_os_image()
+@@ -93,6 +96,26 @@ def test_upgrade(session_cloud: Integrat
+         instance.install_new_cloud_init(source, take_snapshot=False)
+         instance.execute('hostname something-else')
+         _restart(instance)
++        assert instance.execute('cloud-init status --wait --long').ok
+         _output_to_compare(instance, after_path, netcfg_path)
+ 
+     log.info('Wrote upgrade test logs to %s and %s', before_path, after_path)
++
++
++@pytest.mark.ci
++@pytest.mark.ubuntu
++def test_upgrade_package(session_cloud: IntegrationCloud):
++    if get_validated_source(session_cloud) != CloudInitSource.DEB_PACKAGE:
++        not_run_message = 'Test only supports upgrading to build deb'
++        if os.environ.get('TRAVIS'):
++            # If this isn't running on CI, we should know
++            pytest.fail(not_run_message)
++        else:
++            pytest.skip(not_run_message)
++
++    launch_kwargs = {'image_id': session_cloud.released_image_id}
++
++    with session_cloud.launch(launch_kwargs=launch_kwargs) as instance:
++        instance.install_deb()
++        instance.restart()
++        assert instance.execute('cloud-init status --wait --long').ok
diff --git a/debian/patches/series b/debian/patches/series
new file mode 100644
index 00000000..70ce620b
--- /dev/null
+++ b/debian/patches/series
@@ -0,0 +1,2 @@
+cpick-83f6bbfb-Fix-unpickle-for-source-paths-missing-run_dir-863
+cpick-d132356c-fix-error-on-upgrade-caused-by-new-vendordata2