Additional testcase for missing return check of BIO_set_md() calls

This tests the handling of PKCS7 signedAndEnveloped type.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>

2 files changed, 49 insertions, 2 deletions

diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index d414f552fe..c4cd53b8f3 100644
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -50,7 +50,7 @@ my ($no_des, $no_dh, $no_dsa, $no_ec, $no_ec2m, $no_rc2, $no_zlib)
 
 $no_rc2 = 1 if disabled("legacy");
 
-plan tests => 19;
+plan tests => 20;
 
 ok(run(test(["pkcs7_test"])), "test pkcs7");
 
@@ -1087,7 +1087,15 @@ with({ exit_checker => sub { return shift == 4; } },
                     '-inform', 'PEM',
                     '-in', data_file("pkcs7-md4.pem"),
                    ])),
-            "Check failure of EVP_DigestInit is handled correctly");
+            "Check failure of EVP_DigestInit in PKCS7 signed is handled");
+
+        ok(run(app(['openssl', 'smime', '-decrypt',
+                    '-inform', 'PEM',
+                    '-in', data_file("pkcs7-md4-encrypted.pem"),
+                    '-recip', srctop_file("test", "certs", "ee-cert.pem"),
+                    '-inkey', srctop_file("test", "certs", "ee-key.pem")
+                   ])),
+            "Check failure of EVP_DigestInit in PKCS7 signedAndEnveloped is handled");
     });
 
 sub check_availability {
diff --git a/test/recipes/80-test_cms_data/pkcs7-md4-encrypted.pem b/test/recipes/80-test_cms_data/pkcs7-md4-encrypted.pem
new file mode 100644
index 0000000000..18e316f21a
--- /dev/null
+++ b/test/recipes/80-test_cms_data/pkcs7-md4-encrypted.pem
@@ -0,0 +1,39 @@
+-----BEGIN PKCS7-----
+MIIG1QYJKoZIhvcNAQcEoIIGxjCCBsICAQExggEuMIIBKgIBADASMA0xCzAJBgNV
+BAMMAkNBAgECMA0GCSqGSIb3DQEBAQUABIIBADBf36mdqWUMiMHKg2tJyvYjj9I1
+w4Q/lKX0EEsBmoTJtVxzbLhPYRThkXuKBGxH8ln3pF37I09+yFVFhjdLixbVsW5D
+OIeZwctqLTsG5EBYsRMLfcaZCyMUtoJY7Xp7VXZDGHoacBTh/hlYCotX5jUsrBpR
+d89cKP9gr4x9RTvuf4y4eaCjGVSzJpIuvoetR6NxZN4sO5OEjI6AgFtF9BgA5eQH
+Au7TDcKdJeOgYekWJhT7nc9FgdjSD0edztbAivD/tjC1RnSQWPudp4Jnj912aVJr
+jbrNtW183Xx7pD1n7I3ggSDBNZ4aMM1RMmxnUs6cN7dUOrsFVbq9iethjZQxDjAM
+BggqhkiG9w0CBAUAMDwGCSqGSIb3DQEHATAdBglghkgBZQMEAQIEEOfu+O5Rqwyq
+/J0qyQvLBW6AEAVu2T3IVbV4MzxdIeDwS0CgggMkMIIDIDCCAgigAwIBAgIBAjAN
+BgkqhkiG9w0BAQsFADANMQswCQYDVQQDDAJDQTAgFw0xNjAxMTUwODE5NDlaGA8y
+MTE2MDExNjA4MTk0OVowGTEXMBUGA1UEAwwOc2VydmVyLmV4YW1wbGUwggEiMA0G
+CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCo/4lYYYWu3tssD9Vz++K3qBt6dWAr
+1H08c3a1rt6TL38kkG3JHPSKOM2fooAWVsu0LLuT5Rcf/w3GQ/4xNPgo2HXpo7uI
+gu+jcuJTYgVFTeAxl++qnRDSWA2eBp4yuxsIVl1lDz9mjsI2oBH/wFk1/Ukc3RxC
+MwZ4rgQ4I+XndWfTlK1aqUAfrFkQ9QzBZK1KxMY1U7OWaoIbFYvRmavknm+UqtKW
+5Vf7jJFkijwkFsbSGb6CYBM7YrDtPh2zyvlr3zG5ep5LR2inKcc/SuIiJ7TvkGPX
+79ByST5brbkb1Ctvhmjd1XMSuEPJ3EEPoqNGT4tniIQPYf55NB9KiR+3AgMBAAGj
+fTB7MB0GA1UdDgQWBBTnm+IqrYpsOst2UeWOB5gil+FzojAfBgNVHSMEGDAWgBS0
+ETPx1+Je91OeICIQT4YGvx/JXjAJBgNVHRMEAjAAMBMGA1UdJQQMMAoGCCsGAQUF
+BwMBMBkGA1UdEQQSMBCCDnNlcnZlci5leGFtcGxlMA0GCSqGSIb3DQEBCwUAA4IB
+AQBBtDxPYULl5b7VFC7/U0NgV8vTJk4zpPnUMMQ4QF2AWDFAek8oLKrz18KQ8M/D
+EhDxgkaoeXEMLT6BJUEVNYuFEYHEDGarl0nMDRXLxOgAExfz3Tf/pjsLaha5aWH7
+NyCSKWC+lYkIOJ/Kb/m/6QsDJoXsEC8AhrPfqJhzUzsCoxIlaDWqawH4+S8bdeX0
+tvs2VtJk/WOJHxMqXra6kgI4fAgyvr2kIZHinQ3ycgX40uAC38bwpE95kJ7FhSfQ
+lE1Rt7sOspUj098Dd0RNDn2uKyOTxEqIELHfw4AXO3XAzt8qDyho8nEd/xiQ6qgs
+QnvXa+hSRJw42g3/czVskxRxMYICEzCCAg8CAQEwEjANMQswCQYDVQQDDAJDQQIB
+AjAMBggqhkiG9w0CBAUAoIHUMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJ
+KoZIhvcNAQkFMQ8XDTIzMDExODE2NTU0NFowHwYJKoZIhvcNAQkEMRIEEEVzuEyq
+aUYAOFx2/Gw9aXMweQYJKoZIhvcNAQkPMWwwajALBglghkgBZQMEASowCwYJYIZI
+AWUDBAEWMAsGCWCGSAFlAwQBAjAKBggqhkiG9w0DBzAOBggqhkiG9w0DAgICAIAw
+DQYIKoZIhvcNAwICAUAwBwYFKw4DAgcwDQYIKoZIhvcNAwICASgwDQYJKoZIhvcN
+AQEBBQAEggEAm2NW3nbqjQ2m3M95dt7y9FAkUO/Q+6w0JAxfbNj8mhur36/bmOZH
+p5qci4e+txaf219cEG3rJoC4WCB7WG6+VRFsNbynZ8uc7nXqXmw/06R7IF4IGaFV
+umGuWfcLua4ert98sK5aTX7X+4qc5CThHJvX2hPwEq4csc5pL2H0axk27fBjH7J0
+muyklAGYwCFX6qQzLDA0uABMbuZ9DL9Ru9haXmCKTNURgqPHf95kW2a57ldGXoaP
+Oy7soXX0XdPw+L1TH0VAKcvuC7ZQxj01hvEeK7Bpt1f0Uqb0+rdWJLY8hYkOhDsH
+JVQefTghY72ykn8h0HiS3cXwdTti7qpnug==
+-----END PKCS7-----