tree-wide: fix return value handling of base64mem()

This returns an ssize_t, not an int. On populare archs that's the
difference between 64bit and 32bit. hence, let's be more careful here,
and not silently drop half the bits on the ground by assigning the
return value to "int".

As noticed by @malikabhi05:

https://github.com/systemd/systemd/pull/24754#discussion_r1062903159

3 files changed, 19 insertions, 15 deletions

diff --git a/src/cryptenroll/cryptenroll-fido2.c b/src/cryptenroll/cryptenroll-fido2.c
index e49b4a0cfe..2baeb92e07 100644
--- a/src/cryptenroll/cryptenroll-fido2.c
+++ b/src/cryptenroll/cryptenroll-fido2.c
@@ -19,6 +19,7 @@ int load_volume_key_fido2(
         _cleanup_(erase_and_freep) void *decrypted_key = NULL;
         _cleanup_(erase_and_freep) char *passphrase = NULL;
         size_t decrypted_key_size;
+        ssize_t passphrase_size;
         int r;
 
         assert_se(cd);
@@ -43,8 +44,8 @@ int load_volume_key_fido2(
 
         /* Because cryptenroll requires a LUKS header, we can assume that this device is not
          * a PLAIN device. In this case, we need to base64 encode the secret to use as the passphrase */
-        r = base64mem(decrypted_key, decrypted_key_size, &passphrase);
-        if (r < 0)
+        passphrase_size = base64mem(decrypted_key, decrypted_key_size, &passphrase);
+        if (passphrase_size < 0)
                 return log_oom();
 
         r = crypt_volume_key_get(
@@ -53,7 +54,7 @@ int load_volume_key_fido2(
                         ret_vk,
                         ret_vks,
                         passphrase,
-                        /* passphrase_size= */ r);
+                        passphrase_size);
         if (r < 0)
                 return log_error_errno(r, "Unlocking via FIDO2 device failed: %m");
 
@@ -74,6 +75,7 @@ int enroll_fido2(
         _cleanup_free_ char *keyslot_as_string = NULL;
         size_t cid_size, salt_size, secret_size;
         _cleanup_free_ void *cid = NULL;
+        ssize_t base64_encoded_size;
         const char *node, *un;
         int r, keyslot;
 
@@ -106,9 +108,9 @@ int enroll_fido2(
                 return r;
 
         /* Before we use the secret, we base64 encode it, for compat with homed, and to make it easier to type in manually */
-        r = base64mem(secret, secret_size, &base64_encoded);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode secret key: %m");
+        base64_encoded_size = base64mem(secret, secret_size, &base64_encoded);
+        if (base64_encoded_size < 0)
+                return log_error_errno(base64_encoded_size, "Failed to base64 encode secret key: %m");
 
         r = cryptsetup_set_minimal_pbkdf(cd);
         if (r < 0)
@@ -120,7 +122,7 @@ int enroll_fido2(
                         volume_key,
                         volume_key_size,
                         base64_encoded,
-                        strlen(base64_encoded));
+                        base64_encoded_size);
         if (keyslot < 0)
                 return log_error_errno(keyslot, "Failed to add new FIDO2 key to %s: %m", node);
 
diff --git a/src/cryptenroll/cryptenroll-pkcs11.c b/src/cryptenroll/cryptenroll-pkcs11.c
index 9f07a2e01d..54b6b86242 100644
--- a/src/cryptenroll/cryptenroll-pkcs11.c
+++ b/src/cryptenroll/cryptenroll-pkcs11.c
@@ -21,6 +21,7 @@ int enroll_pkcs11(
         size_t decrypted_key_size, encrypted_key_size;
         _cleanup_free_ void *encrypted_key = NULL;
         _cleanup_(X509_freep) X509 *cert = NULL;
+        ssize_t base64_encoded_size;
         const char *node;
         EVP_PKEY *pkey;
         int keyslot, r;
@@ -60,9 +61,9 @@ int enroll_pkcs11(
 
         /* Let's base64 encode the key to use, for compat with homed (and it's easier to type it in by
          * keyboard, if that might ever end up being necessary.) */
-        r = base64mem(decrypted_key, decrypted_key_size, &base64_encoded);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode secret key: %m");
+        base64_encoded_size = base64mem(decrypted_key, decrypted_key_size, &base64_encoded);
+        if (base64_encoded_size < 0)
+                return log_error_errno(base64_encoded_size, "Failed to base64 encode secret key: %m");
 
         r = cryptsetup_set_minimal_pbkdf(cd);
         if (r < 0)
@@ -74,7 +75,7 @@ int enroll_pkcs11(
                         volume_key,
                         volume_key_size,
                         base64_encoded,
-                        strlen(base64_encoded));
+                        base64_encoded_size);
         if (keyslot < 0)
                 return log_error_errno(keyslot, "Failed to add new PKCS#11 key to %s: %m", node);
 
diff --git a/src/cryptenroll/cryptenroll-tpm2.c b/src/cryptenroll/cryptenroll-tpm2.c
index 5c902908c4..96d5fc0695 100644
--- a/src/cryptenroll/cryptenroll-tpm2.c
+++ b/src/cryptenroll/cryptenroll-tpm2.c
@@ -145,6 +145,7 @@ int enroll_tpm2(struct crypt_device *cd,
         uint16_t pcr_bank, primary_alg;
         const char *node;
         _cleanup_(erase_and_freep) char *pin_str = NULL;
+        ssize_t base64_encoded_size;
         int r, keyslot;
         TPM2Flags flags = 0;
 
@@ -230,9 +231,9 @@ int enroll_tpm2(struct crypt_device *cd,
         }
 
         /* let's base64 encode the key to use, for compat with homed (and it's easier to every type it in by keyboard, if that might end up being necessary. */
-        r = base64mem(secret, secret_size, &base64_encoded);
-        if (r < 0)
-                return log_error_errno(r, "Failed to base64 encode secret key: %m");
+        base64_encoded_size = base64mem(secret, secret_size, &base64_encoded);
+        if (base64_encoded_size < 0)
+                return log_error_errno(base64_encoded_size, "Failed to base64 encode secret key: %m");
 
         r = cryptsetup_set_minimal_pbkdf(cd);
         if (r < 0)
@@ -244,7 +245,7 @@ int enroll_tpm2(struct crypt_device *cd,
                         volume_key,
                         volume_key_size,
                         base64_encoded,
-                        strlen(base64_encoded));
+                        base64_encoded_size);
         if (keyslot < 0)
                 return log_error_errno(keyslot, "Failed to add new TPM2 key to %s: %m", node);