Use pkeyutl command instead of rsautl starting openssl version 3.0.0

5 files changed, 89 insertions, 28 deletions

diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx
index 93c88f5..85cf5fa 100644
--- a/bpkg/auth.cxx
+++ b/bpkg/auth.cxx
@@ -12,6 +12,7 @@
 #include <libbutl/openssl.hxx>
 #include <libbutl/timestamp.hxx>
 #include <libbutl/filesystem.hxx>
+#include <libbutl/semantic-version.hxx>
 
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
@@ -23,10 +24,14 @@ using namespace butl;
 
 namespace bpkg
 {
-  static const string openssl_rsautl ("rsautl");
-  static const string openssl_x509   ("x509");
-
-  const char* openssl_commands[3] = {openssl_rsautl.c_str (),
+  static const string openssl_version ("version");
+  static const string openssl_pkeyutl ("pkeyutl");
+  static const string openssl_rsautl  ("rsautl");
+  static const string openssl_x509    ("x509");
+
+  const char* openssl_commands[5] = {openssl_version.c_str (),
+                                     openssl_pkeyutl.c_str (),
+                                     openssl_rsautl.c_str (),
                                      openssl_x509.c_str (),
                                      nullptr};
 
@@ -39,6 +44,49 @@ namespace bpkg
       print_process (args, n);
   }
 
+  // Return true if the openssl version is greater or equal to 3.0.0 and so
+  // pkeyutl needs to be used instead of rsautl. Cache the result on the first
+  // function call.
+  //
+  // Note that openssl 3.0.0 deprecates rsautl in favor of pkeyutl.
+  //
+  // Also note that pkeyutl is only implemented in openssl version 1.0.0 and
+  // its -verifyrecover mode is broken in the [1.1.1 1.1.1d] version range
+  // (see the 'pkeyutl -verifyrecover error "input data too long to be a
+  // hash"' issue report for details).
+  //
+  static optional<bool> use_pkeyutl;
+
+  static bool
+  use_openssl_pkeyutl (const common_options& co)
+  {
+    if (!use_pkeyutl)
+    {
+      const path& openssl_path (co.openssl ()[openssl_version]);
+
+      try
+      {
+        optional<openssl_info> oi (
+          openssl::info (print_command, 2, openssl_path));
+
+        use_pkeyutl = oi                    &&
+                      oi->name == "OpenSSL" &&
+                      oi->version >= semantic_version {3, 0, 0};
+      }
+      catch (const process_error& e)
+      {
+        fail << "unable to execute " << openssl_path << ": " << e << endf;
+      }
+      catch (const io_error& e)
+      {
+        fail << "unable to read '" << openssl_path << "' output: " << e
+             << endf;
+      }
+    }
+
+    return *use_pkeyutl;
+  }
+
   // Find the repository location prefix that ends with the version component.
   // We consider all repositories under this location to be related.
   //
@@ -829,15 +877,22 @@ namespace bpkg
         dr << ": " << *e;
     };
 
-    const path& openssl_path (co.openssl ()[openssl_rsautl]);
-    const strings& openssl_opts (co.openssl_option ()[openssl_rsautl]);
+    bool ku (use_openssl_pkeyutl (co));
+    const string& cmd (ku ? openssl_pkeyutl : openssl_rsautl);
+
+    const path& openssl_path (co.openssl ()[cmd]);
+    const strings& openssl_opts (co.openssl_option ()[cmd]);
 
     try
     {
       openssl os (print_command,
                   path ("-"), fdstream_mode::text, 2,
-                  openssl_path, openssl_rsautl,
-                  openssl_opts, "-verify", "-certin", "-inkey", f);
+                  openssl_path, cmd,
+                  openssl_opts,
+                  ku ? "-verifyrecover" : "-verify",
+                  "-certin",
+                  "-inkey",
+                  f);
 
       for (const auto& c: sm.signature)
         os.out.put (c); // Sets badbit on failure.
@@ -918,14 +973,18 @@ namespace bpkg
         dr << ": " << *e;
     };
 
-    const path& openssl_path (co.openssl ()[openssl_rsautl]);
-    const strings& openssl_opts (co.openssl_option ()[openssl_rsautl]);
+    const string& cmd (use_openssl_pkeyutl (co)
+                       ? openssl_pkeyutl
+                       : openssl_rsautl);
+
+    const path& openssl_path (co.openssl ()[cmd]);
+    const strings& openssl_opts (co.openssl_option ()[cmd]);
 
     try
     {
       openssl os (print_command,
                   fdstream_mode::text, path ("-"), 2,
-                  openssl_path, openssl_rsautl,
+                  openssl_path, cmd,
                   openssl_opts, "-sign", "-inkey", key_name);
 
       os.out << sha256sum;
diff --git a/bpkg/auth.hxx b/bpkg/auth.hxx
index 4cd2e56..54e6884 100644
--- a/bpkg/auth.hxx
+++ b/bpkg/auth.hxx
@@ -79,15 +79,11 @@ namespace bpkg
   // openssl x509 -noout -modulus -in cert.pem
   // openssl rsa -noout -modulus -in key.pem
   //
-  // But taking into account that we need to be able to use custom engines to
-  // access keys, it seems to be impossible to provide the same additional
-  // openssl options to fit both the rsa and pkeyutl commands. The first would
-  // require "-engine pkcs11 -inform engine", while the second -- "-engine
-  // pkcs11 -keyform engine". Also it would require to enter the key password
-  // again, which is a showstopper. Maybe the easiest would be to recover the
-  // sum back from the signature using the certificate, and compare it with
-  // the original sum (like we do in authenticate_repository()). But that
-  // would require to temporarily save the certificate to file.
+  // However, it would require to enter the key password again, which is a
+  // showstopper. Maybe the easiest would be to recover the sum back from the
+  // signature using the certificate, and compare it with the original sum
+  // (like we do in authenticate_repository()). But that would require to
+  // temporarily save the certificate to file.
   //
   std::vector<char>
   sign_repository (const common_options&,
diff --git a/bpkg/common.cli b/bpkg/common.cli
index 5f67357..dd0417d 100644
--- a/bpkg/common.cli
+++ b/bpkg/common.cli
@@ -287,13 +287,17 @@ namespace bpkg
        only applicable to the specific command, for example:
 
        \
-       bpkg rep-create                     \
-         --openssl rsautl:/path/to/openssl \
-         --openssl-option rsautl:-engine   \
-         --openssl-option rsautl:pkcs11    \
+       bpkg rep-create                      \
+         --openssl pkeyutl:/path/to/openssl \
+         --openssl-option pkeyutl:-engine   \
+         --openssl-option pkeyutl:pkcs11    \
          ...
        \
 
+       Note that for \cb{openssl} versions prior to \cb{3.0.0} \cb{bpkg} uses
+       the \cb{rsautl} command instead of \cb{pkeyutl} for the data signing
+       and recovery operations.
+
        An unqualified value that contains a colon can be specified as
        qualified with an empty command, for example, \cb{--openssl
        :C:\\bin\\openssl}. To see openssl commands executed by \cb{bpkg}, use
diff --git a/bpkg/options-types.hxx b/bpkg/options-types.hxx
index 8e8cbf1..741e93c 100644
--- a/bpkg/options-types.hxx
+++ b/bpkg/options-types.hxx
@@ -73,7 +73,7 @@ namespace bpkg
     }
   };
 
-  extern const char* openssl_commands[3]; // Clang bug requres explicit size.
+  extern const char* openssl_commands[5]; // Clang bug requres explicit size.
 }
 
 #endif // BPKG_OPTIONS_TYPES_HXX
diff --git a/bpkg/repository-signing.cli b/bpkg/repository-signing.cli
index 1796497..a85ecc0 100644
--- a/bpkg/repository-signing.cli
+++ b/bpkg/repository-signing.cli
@@ -193,11 +193,13 @@ just \cb{--key} as at step 4 (\c{\"SIGN key\"} is the label for the slot
 \c{9c} private key):
 
 \
-bpkg rep-create                                                   \
-  --openssl-option rsautl:-engine --openssl-option rsautl:pkcs11  \
-  --openssl-option rsautl:-keyform --openssl-option rsautl:engine \
+bpkg rep-create                                                     \
+  --openssl-option pkeyutl:-engine --openssl-option pkeyutl:pkcs11  \
+  --openssl-option pkeyutl:-keyform --openssl-option pkeyutl:engine \
   --key \"pkcs11:object=SIGN%20key\" /path/to/repository
 \
 
+Note that for \cb{openssl} versions prior to \cb{3.0.0} \cb{bpkg} uses the
+\cb{rsautl} command instead of \cb{pkeyutl} for the data signing operation.
 ||
 "