core: workaround configuring IPv6 routes with "src" (RTA_PREFSRC)

Kernel does not allow to add IPv6 routes with "src", as long as the
corresponding address is still tentative (related bug rh#1457196).

The workaround for this is cumbersome. First, when we fail to add such a
route with "pref_src", we guess that it happend due to this issue. In
that case, nm_ip6_config_commit() returns the list of routes that could
not be added for the moment (but hopefully can be added later).

We track this list in NMDevice, and keep trying to merge the routes
back into ip6_config. In order to not try indefinitely, keep track of a
timestamp when we tried to add this route for the first time.

Another uglyness is that pending tentative routes don't explicitly block
activation. In practice they may do, because for these routes we also have
an IPv6 address that is still doing DAD, so the IP configuration is
still pending due to that.

https://bugzilla.redhat.com/show_bug.cgi?id=1452684

9 files changed, 189 insertions, 10 deletions

diff --git a/shared/nm-utils/nm-shared-utils.h b/shared/nm-utils/nm-shared-utils.h
index 8d1085f6a5..065e629b4e 100644
--- a/shared/nm-utils/nm-shared-utils.h
+++ b/shared/nm-utils/nm-shared-utils.h
@@ -178,6 +178,7 @@ _nm_g_slice_free_fcn_define (1)
 _nm_g_slice_free_fcn_define (2)
 _nm_g_slice_free_fcn_define (4)
 _nm_g_slice_free_fcn_define (8)
+_nm_g_slice_free_fcn_define (12)
 _nm_g_slice_free_fcn_define (16)
 
 #define _nm_g_slice_free_fcn1(mem_size) \
@@ -192,6 +193,7 @@ _nm_g_slice_free_fcn_define (16)
 		case  2: _fcn = _nm_g_slice_free_fcn_2;  break; \
 		case  4: _fcn = _nm_g_slice_free_fcn_4;  break; \
 		case  8: _fcn = _nm_g_slice_free_fcn_8;  break; \
+		case 12: _fcn = _nm_g_slice_free_fcn_12;  break; \
 		case 16: _fcn = _nm_g_slice_free_fcn_16; break; \
 		default: g_assert_not_reached (); _fcn = NULL; break; \
 		} \
diff --git a/src/devices/nm-device.c b/src/devices/nm-device.c
index cd523b9dad..4043457bee 100644
--- a/src/devices/nm-device.c
+++ b/src/devices/nm-device.c
@@ -400,6 +400,7 @@ typedef struct _NMDevicePrivate {
 	/* IPv4LL stuff */
 	sd_ipv4ll *    ipv4ll;
 	guint          ipv4ll_timeout;
+	guint          rt6_temporary_not_available_id;
 
 	/* IPv4 DAD stuff */
 	struct {
@@ -421,6 +422,8 @@ typedef struct _NMDevicePrivate {
 	bool           nm_ipv6ll; /* TRUE if NM handles the device's IPv6LL address */
 	NMIP6Config *  dad6_ip6_config;
 
+	GHashTable *   rt6_temporary_not_available;
+
 	NMNDisc *      ndisc;
 	gulong         ndisc_changed_id;
 	gulong         ndisc_timeout_id;
@@ -6350,6 +6353,18 @@ ip6_config_merge_and_apply (NMDevice *self,
 		                     | (ignore_auto_dns ? NM_IP_CONFIG_MERGE_NO_DNS : 0));
 	}
 
+	if (priv->rt6_temporary_not_available) {
+		const NMPObject *o;
+		GHashTableIter hiter;
+
+		g_hash_table_iter_init (&hiter, priv->rt6_temporary_not_available);
+		while (g_hash_table_iter_next (&hiter, (gpointer *) &o, NULL)) {
+			nm_ip6_config_add_route (composite,
+			                         NMP_OBJECT_CAST_IP6_ROUTE (o),
+			                         NULL);
+		}
+	}
+
 	/* Merge user overrides into the composite config. For assumed connections,
 	 * con_ip6_config is empty. */
 	if (priv->con_ip6_config)
@@ -7452,6 +7467,9 @@ addrconf6_start (NMDevice *self, NMSettingIP6ConfigPrivacy use_tempaddr)
 		priv->ac_ip6_config = NULL;
 	}
 
+	g_clear_pointer (&priv->rt6_temporary_not_available, g_hash_table_unref);
+	nm_clear_g_source (&priv->rt6_temporary_not_available_id);
+
 	s_ip6 = NM_SETTING_IP6_CONFIG (nm_connection_get_setting_ip6_config (connection));
 	g_assert (s_ip6);
 
@@ -7505,6 +7523,8 @@ addrconf6_cleanup (NMDevice *self)
 	nm_device_remove_pending_action (self, NM_PENDING_ACTION_AUTOCONF6, FALSE);
 
 	g_clear_object (&priv->ac_ip6_config);
+	g_clear_pointer (&priv->rt6_temporary_not_available, g_hash_table_unref);
+	nm_clear_g_source (&priv->rt6_temporary_not_available_id);
 	g_clear_object (&priv->ndisc);
 }
 
@@ -9396,6 +9416,97 @@ impl_device_get_applied_connection (NMDevice *self,
 
 /*****************************************************************************/
 
+typedef struct {
+	gint64 timestamp_ms;
+	bool dirty;
+} IP6RoutesTemporaryNotAvailableData;
+
+static gboolean
+_rt6_temporary_not_available_timeout (gpointer user_data)
+{
+	NMDevice *self = NM_DEVICE (user_data);
+	NMDevicePrivate *priv = NM_DEVICE_GET_PRIVATE (self);
+
+	priv->rt6_temporary_not_available_id = 0;
+	nm_device_activate_schedule_ip6_config_result (self);
+
+	return G_SOURCE_REMOVE;
+}
+
+static gboolean
+_rt6_temporary_not_available_set (NMDevice *self,
+                                  GPtrArray *temporary_not_available)
+{
+	NMDevicePrivate *priv = NM_DEVICE_GET_PRIVATE (self);
+	IP6RoutesTemporaryNotAvailableData *data;
+	GHashTableIter iter;
+	gint64 now_ms, oldest_ms;
+	const gint64 MAX_AGE_MS = 20000;
+	guint i;
+	gboolean success = TRUE;
+
+	if (   !temporary_not_available
+	    || !temporary_not_available->len) {
+		/* nothing outstanding. Clear tracking the routes. */
+		g_clear_pointer (&priv->rt6_temporary_not_available, g_hash_table_unref);
+		nm_clear_g_source (&priv->rt6_temporary_not_available_id);
+		return success;
+	}
+
+	if (priv->rt6_temporary_not_available) {
+		g_hash_table_iter_init (&iter, priv->rt6_temporary_not_available);
+		while (g_hash_table_iter_next (&iter, NULL, (gpointer *) &data))
+			data->dirty = TRUE;
+	} else {
+		priv->rt6_temporary_not_available = g_hash_table_new_full ((GHashFunc) nmp_object_id_hash,
+		                                                           (GEqualFunc) nmp_object_id_equal,
+		                                                           (GDestroyNotify) nmp_object_unref,
+		                                                           nm_g_slice_free_fcn (IP6RoutesTemporaryNotAvailableData));
+	}
+
+	now_ms = nm_utils_get_monotonic_timestamp_ms ();
+	oldest_ms = now_ms;
+
+	for (i = 0; i < temporary_not_available->len; i++) {
+		const NMPObject *o = temporary_not_available->pdata[i];
+
+		data = g_hash_table_lookup (priv->rt6_temporary_not_available, o);
+		if (data) {
+			if (!data->dirty)
+				continue;
+			data->dirty = FALSE;
+			nm_assert (data->timestamp_ms > 0 && data->timestamp_ms <= now_ms);
+			if (now_ms > data->timestamp_ms + MAX_AGE_MS) {
+				/* timeout. Could not add this address. */
+				_LOGW (LOGD_DEVICE, "failure to add IPv6 route: %s",
+				       nmp_object_to_string (o, NMP_OBJECT_TO_STRING_PUBLIC, NULL, 0));
+				success = FALSE;
+			} else
+				oldest_ms = MIN (data->timestamp_ms, oldest_ms);
+			continue;
+		}
+
+		data = g_slice_new0 (IP6RoutesTemporaryNotAvailableData);
+		data->timestamp_ms = now_ms;
+		g_hash_table_insert (priv->rt6_temporary_not_available, (gpointer) nmp_object_ref (o), data);
+	}
+
+	g_hash_table_iter_init (&iter, priv->rt6_temporary_not_available);
+	while (g_hash_table_iter_next (&iter, NULL, (gpointer *) &data)) {
+		if (data->dirty)
+			g_hash_table_iter_remove (&iter);
+	}
+
+	nm_clear_g_source (&priv->rt6_temporary_not_available_id);
+	priv->rt6_temporary_not_available_id = g_timeout_add (oldest_ms + MAX_AGE_MS - now_ms,
+	                                                      _rt6_temporary_not_available_timeout,
+	                                                      self);
+
+	return success;
+}
+
+/*****************************************************************************/
+
 static void
 disconnect_cb (NMDevice *self,
                GDBusMethodInvocation *context,
@@ -9941,9 +10052,16 @@ nm_device_set_ip6_config (NMDevice *self,
 
 	/* Always commit to nm-platform to update lifetimes */
 	if (commit && new_config) {
+		gs_unref_ptrarray GPtrArray *temporary_not_available = NULL;
+
 		_commit_mtu (self, priv->ip4_config);
+
 		success = nm_ip6_config_commit (new_config,
-		                                nm_device_get_platform (self));
+		                                nm_device_get_platform (self),
+		                                &temporary_not_available);
+
+		if (!_rt6_temporary_not_available_set (self, temporary_not_available))
+			success = FALSE;
 	}
 
 	if (new_config) {
@@ -10876,6 +10994,8 @@ queued_ip6_config_change (gpointer user_data)
 			g_clear_object (&priv->dad6_ip6_config);
 			_set_ip_state (self, AF_INET6, IP_DONE);
 			check_ip_state (self, FALSE);
+			if (priv->rt6_temporary_not_available)
+				nm_device_activate_schedule_ip6_config_result (self);
 		}
 	}
 
@@ -12016,6 +12136,9 @@ _cleanup_generic_post (NMDevice *self, CleanupType cleanup_type)
 	g_clear_object (&priv->ip6_config);
 	g_clear_object (&priv->dad6_ip6_config);
 
+	g_clear_pointer (&priv->rt6_temporary_not_available, g_hash_table_unref);
+	nm_clear_g_source (&priv->rt6_temporary_not_available_id);
+
 	g_slist_free_full (priv->vpn4_configs, g_object_unref);
 	priv->vpn4_configs = NULL;
 	g_slist_free_full (priv->vpn6_configs, g_object_unref);
diff --git a/src/nm-iface-helper.c b/src/nm-iface-helper.c
index ae76ee94ef..43c2ae478b 100644
--- a/src/nm-iface-helper.c
+++ b/src/nm-iface-helper.c
@@ -226,7 +226,7 @@ ndisc_config_changed (NMNDisc *ndisc, const NMNDiscData *rdata, guint changed_in
 	}
 
 	nm_ip6_config_merge (existing, ndisc_config, NM_IP_CONFIG_MERGE_DEFAULT);
-	if (!nm_ip6_config_commit (existing, NM_PLATFORM_GET))
+	if (!nm_ip6_config_commit (existing, NM_PLATFORM_GET, NULL))
 		_LOGW (LOGD_IP6, "failed to apply IPv6 config");
 }
 
diff --git a/src/nm-ip4-config.c b/src/nm-ip4-config.c
index f3d87814ee..90407e34a1 100644
--- a/src/nm-ip4-config.c
+++ b/src/nm-ip4-config.c
@@ -837,6 +837,7 @@ nm_ip4_config_commit (const NMIP4Config *self,
 	                                ifindex,
 	                                routes,
 	                                nm_platform_lookup_predicate_routes_main,
+	                                NULL,
 	                                NULL))
 		success = FALSE;
 
diff --git a/src/nm-ip6-config.c b/src/nm-ip6-config.c
index ce77394256..acb9ccb646 100644
--- a/src/nm-ip6-config.c
+++ b/src/nm-ip6-config.c
@@ -529,7 +529,8 @@ nm_ip6_config_capture (NMDedupMultiIndex *multi_idx, NMPlatform *platform, int i
 
 gboolean
 nm_ip6_config_commit (const NMIP6Config *self,
-                      NMPlatform *platform)
+                      NMPlatform *platform,
+                      GPtrArray **out_temporary_not_available)
 {
 	gs_unref_ptrarray GPtrArray *addresses = NULL;
 	gs_unref_ptrarray GPtrArray *routes = NULL;
@@ -552,7 +553,8 @@ nm_ip6_config_commit (const NMIP6Config *self,
 	                                ifindex,
 	                                routes,
 	                                nm_platform_lookup_predicate_routes_main,
-	                                NULL))
+	                                NULL,
+	                                out_temporary_not_available))
 		success = FALSE;
 
 	return success;
diff --git a/src/nm-ip6-config.h b/src/nm-ip6-config.h
index ec8bee881d..71ae1e469c 100644
--- a/src/nm-ip6-config.h
+++ b/src/nm-ip6-config.h
@@ -108,7 +108,8 @@ struct _NMDedupMultiIndex *nm_ip6_config_get_multi_idx (const NMIP6Config *self)
 NMIP6Config *nm_ip6_config_capture (struct _NMDedupMultiIndex *multi_idx, NMPlatform *platform, int ifindex,
                                     gboolean capture_resolv_conf, NMSettingIP6ConfigPrivacy use_temporary);
 gboolean nm_ip6_config_commit (const NMIP6Config *self,
-                               NMPlatform *platform);
+                               NMPlatform *platform,
+                               GPtrArray **out_temporary_not_available);
 void nm_ip6_config_merge_setting (NMIP6Config *self, NMSettingIPConfig *setting, guint32 default_route_metric);
 NMSetting *nm_ip6_config_create_setting (const NMIP6Config *self);
 
diff --git a/src/platform/nm-platform.c b/src/platform/nm-platform.c
index 47ac8bfd52..c3b123b79f 100644
--- a/src/platform/nm-platform.c
+++ b/src/platform/nm-platform.c
@@ -3547,6 +3547,42 @@ nm_platform_ip_address_flush (NMPlatform *self,
 
 /*****************************************************************************/
 
+static gboolean
+_err_inval_due_to_ipv6_tentative_pref_src (NMPlatform *self, const NMPObject *obj)
+{
+	const NMPlatformIP6Route *r;
+	const NMPlatformIP6Address *a;
+
+	nm_assert (NM_IS_PLATFORM (self));
+	nm_assert (NMP_OBJECT_IS_VALID (obj));
+
+	/* trying to add an IPv6 route with pref-src fails, if the address is
+	 * still tentative (rh#1452684). We need to hack around that.
+	 *
+	 * Detect it, by guessing whether that's the case. */
+
+	if (NMP_OBJECT_GET_TYPE (obj) != NMP_OBJECT_TYPE_IP6_ROUTE)
+		return FALSE;
+
+	r = NMP_OBJECT_CAST_IP6_ROUTE (obj);
+
+	/* we only allow this workaround for routes added manually by the user. */
+	if (r->rt_source != NM_IP_CONFIG_SOURCE_USER)
+		return FALSE;
+
+	if (IN6_IS_ADDR_UNSPECIFIED (&r->pref_src))
+		return FALSE;
+
+	a = nm_platform_ip6_address_get (self, r->ifindex, r->pref_src);
+	if (!a)
+		return FALSE;
+	if (   !NM_FLAGS_HAS (a->n_ifa_flags, IFA_F_TENTATIVE)
+	    || NM_FLAGS_HAS (a->n_ifa_flags, IFA_F_DADFAILED))
+		return FALSE;
+
+	return TRUE;
+}
+
 /**
  * nm_platform_ip_route_sync:
  * @self: the #NMPlatform instance.
@@ -3560,6 +3596,8 @@ nm_platform_ip_address_flush (NMPlatform *self,
  *   routes. For example by passing @nm_platform_lookup_predicate_routes_main_skip_rtprot_kernel,
  *   routes with "proto kernel" will be left untouched.
  * @kernel_delete_userdata: user data for @kernel_delete_predicate.
+ * @out_temporary_not_available: (allow-none): (out): routes that could
+ *   currently not be synced. The caller shall keep them and try later again.
  *
  * Returns: %TRUE on success.
  */
@@ -3569,7 +3607,8 @@ nm_platform_ip_route_sync (NMPlatform *self,
                            int ifindex,
                            GPtrArray *routes,
                            NMPObjectPredicateFunc kernel_delete_predicate,
-                           gpointer kernel_delete_userdata)
+                           gpointer kernel_delete_userdata,
+                           GPtrArray **out_temporary_not_available)
 {
 	const NMPlatformVTableRoute *vt;
 	gs_unref_ptrarray GPtrArray *plat_routes = NULL;
@@ -3662,6 +3701,15 @@ nm_platform_ip_route_sync (NMPlatform *self,
 							       nmp_object_to_string (plat_entry->obj, NMP_OBJECT_TO_STRING_PUBLIC, sbuf2, sizeof (sbuf2)));
 						}
 					}
+				} else if (   -((int) plerr) == EINVAL
+				           && out_temporary_not_available
+				           && _err_inval_due_to_ipv6_tentative_pref_src (self, conf_o)) {
+					_LOGD ("route-sync: ignore failure to add IPv6 route with tentative IPv6 pref-src: %s: %s",
+					       nmp_object_to_string (conf_o, NMP_OBJECT_TO_STRING_PUBLIC, sbuf1, sizeof (sbuf1)),
+					       nm_platform_error_to_string (plerr, sbuf_err, sizeof (sbuf_err)));
+					if (!*out_temporary_not_available)
+						*out_temporary_not_available = g_ptr_array_new_full (0, (GDestroyNotify) nmp_object_unref);
+					g_ptr_array_add (*out_temporary_not_available, (gpointer) nmp_object_ref (conf_o));
 				} else if (NMP_OBJECT_CAST_IP_ROUTE (conf_o)->rt_source < NM_IP_CONFIG_SOURCE_USER) {
 					_LOGD ("route-sync: ignore failure to add IPv%c route: %s: %s",
 					       vt->is_ip4 ? '4' : '6',
@@ -3723,9 +3771,9 @@ nm_platform_ip_route_flush (NMPlatform *self,
 	                                   AF_INET6));
 
 	if (NM_IN_SET (addr_family, AF_UNSPEC, AF_INET))
-		success &= nm_platform_ip_route_sync (self, AF_INET,  ifindex, NULL, NULL, NULL);
+		success &= nm_platform_ip_route_sync (self, AF_INET,  ifindex, NULL, NULL, NULL, NULL);
 	if (NM_IN_SET (addr_family, AF_UNSPEC, AF_INET6))
-		success &= nm_platform_ip_route_sync (self, AF_INET6, ifindex, NULL, NULL, NULL);
+		success &= nm_platform_ip_route_sync (self, AF_INET6, ifindex, NULL, NULL, NULL, NULL);
 	return success;
 }
 
diff --git a/src/platform/nm-platform.h b/src/platform/nm-platform.h
index 917f623d01..e68457ca2b 100644
--- a/src/platform/nm-platform.h
+++ b/src/platform/nm-platform.h
@@ -1177,7 +1177,8 @@ gboolean nm_platform_ip_route_sync (NMPlatform *self,
                                     int ifindex,
                                     GPtrArray *routes,
                                     NMPObjectPredicateFunc kernel_delete_predicate,
-                                    gpointer kernel_delete_userdata);
+                                    gpointer kernel_delete_userdata,
+                                    GPtrArray **out_temporary_not_available);
 gboolean nm_platform_ip_route_flush (NMPlatform *self,
                                      int addr_family,
                                      int ifindex);
diff --git a/src/vpn/nm-vpn-connection.c b/src/vpn/nm-vpn-connection.c
index 517e7eb97a..0006cf3cc2 100644
--- a/src/vpn/nm-vpn-connection.c
+++ b/src/vpn/nm-vpn-connection.c
@@ -1159,7 +1159,8 @@ nm_vpn_connection_apply_config (NMVpnConnection *self)
 		if (priv->ip6_config) {
 			nm_assert (priv->ip_ifindex == nm_ip6_config_get_ifindex (priv->ip6_config));
 			if (!nm_ip6_config_commit (priv->ip6_config,
-			                           nm_netns_get_platform (priv->netns)))
+			                           nm_netns_get_platform (priv->netns),
+			                           NULL))
 				return FALSE;
 		}