Add support for PKCS7_set/get_detached (#2134)

We’re still missing two PKCS7 symbols for Ruby. This also applies to past
releases.

* `PKCS7_get_detached` is basically an alias to `PKCS7_is_detached`, so
I've done that accordingly.
* `PKCS7_set_detached` is used to "detach" data contents from the
internal `PKCS7` structure. The OpenSSL code uses a detached field in
the PKCS7 structure that’s missing in AWS-LC. This field isn't necessarily
needed though, we can just detect the functionality accordingly.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

crypto/pkcs7/pkcs7.c

@@ -843,6 +843,33 @@ int PKCS7_is_detached(PKCS7 *p7) {
   return 0;
 }
 
+int PKCS7_set_detached(PKCS7 *p7, int detach) {
+  GUARD_PTR(p7);
+  if (detach != 0 && detach != 1) {
+    // |detach| is meant to be used as a boolean int.
+    return 0;
+  }
+
+  if (PKCS7_type_is_signed(p7)) {
+    if (p7->d.sign == NULL) {
+      OPENSSL_PUT_ERROR(PKCS7, PKCS7_R_NO_CONTENT);
+      return 0;
+    }
+    if (detach && PKCS7_type_is_data(p7->d.sign->contents)) {
+      ASN1_OCTET_STRING_free(p7->d.sign->contents->d.data);
+      p7->d.sign->contents->d.data = NULL;
+    }
+    return detach;
+  } else {
+    OPENSSL_PUT_ERROR(PKCS7, PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE);
+    return 0;
+  }
+}
+
+int PKCS7_get_detached(PKCS7 *p7) {
+  return PKCS7_is_detached(p7);
+}
+
 
 static BIO *pkcs7_find_digest(EVP_MD_CTX **pmd, BIO *bio, int nid) {
   GUARD_PTR(pmd);

crypto/pkcs7/pkcs7_test.cc

@@ -2043,3 +2043,28 @@ TEST(PKCS7Test, PKCS7PrintNoop) {
   ASSERT_TRUE(BIO_mem_contents(bio.get(), &contents, &len));
   EXPECT_EQ(Bytes(contents, len), Bytes("PKCS7 printing is not supported"));
 }
+
+TEST(PKCS7Test, SetDetached) {
+  bssl::UniquePtr<PKCS7> p7(PKCS7_new());
+  // |PKCS7_set_detached| does not work on an uninitialized |PKCS7|.
+  EXPECT_FALSE(PKCS7_set_detached(p7.get(), 0));
+  EXPECT_FALSE(PKCS7_set_detached(p7.get(), 1));
+  EXPECT_TRUE(PKCS7_set_type(p7.get(), NID_pkcs7_signed));
+  EXPECT_TRUE(PKCS7_type_is_signed(p7.get()));
+
+  PKCS7 *p7_internal = PKCS7_new();
+  EXPECT_TRUE(PKCS7_set_type(p7_internal, NID_pkcs7_data));
+  EXPECT_TRUE(PKCS7_type_is_data(p7_internal));
+  EXPECT_TRUE(PKCS7_set_content(p7.get(), p7_internal));
+
+  // Access the |p7|'s internal contents to verify that |PKCS7_set_detached|
+  // has the right behavior.
+  EXPECT_TRUE(p7.get()->d.sign->contents->d.data);
+  EXPECT_FALSE(PKCS7_set_detached(p7.get(), 0));
+  EXPECT_TRUE(p7.get()->d.sign->contents->d.data);
+  EXPECT_FALSE(PKCS7_set_detached(p7.get(), 2));
+  EXPECT_TRUE(p7.get()->d.sign->contents->d.data);
+  // data is "detached" when |PKCS7_set_detached| is set with 1.
+  EXPECT_TRUE(PKCS7_set_detached(p7.get(), 1));
+  EXPECT_FALSE(p7.get()->d.sign->contents->d.data);
+}

include/openssl/pkcs7.h

@@ -467,6 +467,16 @@ OPENSSL_EXPORT OPENSSL_DEPRECATED int PKCS7_verify(PKCS7 *p7,
 // PKCS7_is_detached returns 0 if |p7| has attached content and 1 otherwise.
 OPENSSL_EXPORT OPENSSL_DEPRECATED int PKCS7_is_detached(PKCS7 *p7);
 
+// PKCS7_set_detached frees the attached content of |p7| if |detach| is set to
+// 1. It returns 0 if otherwise or if |p7| is not of type signed.
+//
+// Note: |detach| is intended to be a boolean and MUST be set with either 1 or
+//       0.
+OPENSSL_EXPORT OPENSSL_DEPRECATED int PKCS7_set_detached(PKCS7 *p7, int detach);
+
+// PKCS7_get_detached returns 0 if |p7| has attached content and 1 otherwise.
+OPENSSL_EXPORT OPENSSL_DEPRECATED int PKCS7_get_detached(PKCS7 *p7);
+
 // PKCS7_dataInit creates or initializes a BIO chain for reading data from or
 // writing data to |p7|. If |bio| is non-null, it is added to the chain.
 // Otherwise, a new BIO is allocated and returned to anchor the chain.
@@ -576,5 +586,6 @@ BSSL_NAMESPACE_END
 #define PKCS7_R_PKCS7_ADD_SIGNATURE_ERROR 132
 #define PKCS7_R_NO_DEFAULT_DIGEST 133
 #define PKCS7_R_CERT_MUST_BE_RSA 134
+#define PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE 135
 
 #endif  // OPENSSL_HEADER_PKCS7_H