rubygems: Add PQC ML-DSA support for cryptographically signed gems workflow

lib/rubygems/commands/cert_command.rb

@@ -45,7 +45,10 @@ def initialize
     end
 
     add_option("-A", "--key-algorithm ALGORITHM",
-               "Select which key algorithm to use for --build") do |algorithm, options|
+               "Select key algorithm for --build from",
+               "RSA, DSA, EC, ML-DSA-44, ML-DSA-65,",
+               "or ML-DSA-87. Defaults to "\
+               "#{Gem::Security::DEFAULT_KEY_ALGORITHM}.") do |algorithm, options|
       options[:key_algorithm] = algorithm
     end
 
@@ -96,8 +99,19 @@ def open_private_key(key_file)
     check_openssl
     passphrase = ENV["GEM_PRIVATE_KEY_PASSPHRASE"]
     key = OpenSSL::PKey.read File.read(key_file), passphrase
-    raise Gem::OptionParser::InvalidArgument,
-      "#{key_file}: private key not found" unless key.private?
+    # ML-DSA keys (OpenSSL::PKey::PKey) do not have private? method.
+    # Try private_to_pem instead to check if the key has private data.
+    if key.respond_to?(:private?)
+      raise Gem::OptionParser::InvalidArgument,
+        "#{key_file}: private key not found" unless key.private?
+    else
+      begin
+        key.private_to_pem
+      rescue OpenSSL::PKey::PKeyError
+        raise Gem::OptionParser::InvalidArgument,
+          "#{key_file}: private key not found"
+      end
+    end
     key
   rescue Errno::ENOENT
     raise Gem::OptionParser::InvalidArgument, "#{key_file}: does not exist"

lib/rubygems/security.rb

@@ -153,7 +153,9 @@
 #                                      certificate for EMAIL_ADDR
 #     -C, --certificate CERT           Signing certificate for --sign
 #     -K, --private-key KEY            Key for --sign or --build
-#     -A, --key-algorithm ALGORITHM    Select key algorithm for --build from RSA, DSA, or EC. Defaults to RSA.
+#     -A, --key-algorithm ALGORITHM    Select key algorithm for --build from
+#                                      RSA, DSA, EC, ML-DSA-44, ML-DSA-65,
+#                                      or ML-DSA-87. Defaults to RSA.
 #     -s, --sign CERT                  Signs CERT with the key from -K
 #                                      and the certificate from -C
 #     -d, --days NUMBER_OF_DAYS        Days before the certificate expires
@@ -351,6 +353,20 @@ class Exception < Gem::Exception; end
 
   EC_NAME = "secp384r1"
 
+  ##
+  # ML-DSA algorithm names to use when building a key pair.
+  # ML-DSA-44: NIST security strength category 2, signature size 2420 bytes
+  # ML-DSA-65: NIST security strength category 3, signature size 3309 bytes
+  # ML-DSA-87: NIST security strength category 5, signature size 4627 bytes
+  # See NIST FIPS 204 Section 4 (Parameter Sets).
+  # https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.204.pdf
+  # See Security (Evaluation Criteria) - 4.A.5 Security Strength Categories.
+  # https://csrc.nist.gov/projects/post-quantum-cryptography/post-quantum-cryptography-standardization/evaluation-criteria/security-(evaluation-criteria)
+
+  ML_DSA_44_NAME = "ML-DSA-44"
+  ML_DSA_65_NAME = "ML-DSA-65"
+  ML_DSA_87_NAME = "ML-DSA-87"
+
   ##
   # Cipher used to encrypt the key pair used to sign gems.
   # Must be in the list returned by OpenSSL::Cipher.ciphers
@@ -464,25 +480,41 @@ def self.create_digest(algorithm = DIGEST_NAME)
   end
 
   ##
-  # Creates a new key pair of the specified +algorithm+. RSA, DSA, and EC
-  # are supported.
+  # Creates a new key pair of the specified +algorithm+. RSA, DSA, EC,
+  # ML-DSA-44, ML-DSA-65, and ML-DSA-87 are supported.
 
   def self.create_key(algorithm)
     if defined?(OpenSSL::PKey)
       case algorithm.downcase
-      when "dsa"
-        OpenSSL::PKey::DSA.new(RSA_DSA_KEY_LENGTH)
       when "rsa"
         OpenSSL::PKey::RSA.new(RSA_DSA_KEY_LENGTH)
+      when "dsa"
+        OpenSSL::PKey::DSA.new(RSA_DSA_KEY_LENGTH)
       when "ec"
         OpenSSL::PKey::EC.generate(EC_NAME)
+      when "ml-dsa-44"
+        OpenSSL::PKey.generate_key(ML_DSA_44_NAME)
+      when "ml-dsa-65"
+        OpenSSL::PKey.generate_key(ML_DSA_65_NAME)
+      when "ml-dsa-87"
+        OpenSSL::PKey.generate_key(ML_DSA_87_NAME)
       else
         raise Gem::Security::Exception,
-        "#{algorithm} algorithm not found. RSA, DSA, and EC algorithms are supported."
+          "#{algorithm} algorithm not found. RSA, DSA, EC, ML-DSA-44, "\
+          "ML-DSA-65, and ML-DSA-87 algorithms are supported."
       end
     end
   end
 
+  ##
+  # Returns whether the +key+ requires an explicit digest algorithm for signing
+  # and verification. ML-DSA has a built-in digest and does not accept one.
+
+  def self.digest_required?(key)
+    key.is_a?(OpenSSL::PKey::RSA) || key.is_a?(OpenSSL::PKey::DSA) ||
+      key.is_a?(OpenSSL::PKey::EC)
+  end
+
   ##
   # Turns +email_address+ into an OpenSSL::X509::Name
 
@@ -562,7 +594,8 @@ def self.sign(certificate, signing_key, signing_cert, age = ONE_YEAR, extensions
     signed = create_cert signee_subject, signee_key, age, extensions, serial
     signed.issuer = signing_cert.subject
 
-    signed.sign signing_key, Gem::Security::DIGEST_NAME
+    digest_name = Gem::Security::DIGEST_NAME if digest_required?(signing_key)
+    signed.sign signing_key, digest_name
   end
 
   ##
@@ -584,20 +617,56 @@ def self.trusted_certificates(&block)
     trust_dir.each_certificate(&block)
   end
 
+  ##
+  # Converts +pemmable+ to PEM format string. The +pemmable+ can be a key
+  # (OpenSSL::PKey::RSA, DSA, EC) or certificate (OpenSSL::X509::Certificate).
+  # RSA/DSA/EC keys and certificates support to_pem. ML-DSA keys
+  # (OpenSSL::PKey::PKey) do not support to_pem, so private_to_pem or
+  # public_to_pem is used instead.
+  # If +cipher+ and +passphrase+ are provided, the PEM output is encrypted.
+
+  def self.pemmable_to_pem(pemmable, passphrase = nil, cipher = KEY_CIPHER)
+    if pemmable.respond_to?(:to_pem)
+      # pemmable: OpenSSL::PKey::RSA, DSA, EC, OpenSSL::X509::Certificate
+      if cipher && passphrase
+        pemmable.to_pem cipher, passphrase
+      else
+        pemmable.to_pem
+      end
+    else
+      # pemmable: OpenSSL::PKey::PKey (ML-DSA)
+      if cipher && passphrase
+        # public_to_pem doesn't have cipher and passphrase arguments.
+        # So, given these arguments, only run private_to_pem.
+        pemmable.private_to_pem cipher, passphrase
+      else
+        # Check if the key is private or public by trying, and rescuing an
+        # error as OpenSSL::PKey::PKey doesn't have such a method.
+        begin
+          pemmable.private_to_pem
+        rescue OpenSSL::PKey::PKeyError
+          # public_to_pem doesn't have cipher and passphrase arguments
+          pemmable.public_to_pem
+        end
+      end
+    end
+  end
+
   ##
   # Writes +pemmable+, which must respond to +to_pem+ to +path+ with the given
   # +permissions+. If passed +cipher+ and +passphrase+ those arguments will be
-  # passed to +to_pem+.
+  # passed to +to_pem+. The +pemmable+ can be key or certificate.
 
   def self.write(pemmable, path, permissions = 0o600, passphrase = nil, cipher = KEY_CIPHER)
     path = File.expand_path path
 
     File.open path, "wb", permissions do |io|
-      if passphrase && cipher
-        io.write pemmable.to_pem cipher, passphrase
+      pem_str = if passphrase && cipher
+        pemmable_to_pem pemmable, passphrase, cipher
       else
-        io.write pemmable.to_pem
+        pemmable_to_pem pemmable
       end
+      io.write pem_str
     end
 
     path

lib/rubygems/security/policy.rb

@@ -73,8 +73,10 @@ def check_chain(chain, time)
   # the +digest+ algorithm.
 
   def check_data(public_key, digest, signature, data)
+    filtered_digest = digest if Gem::Security.digest_required?(public_key)
+
     raise Gem::Security::Exception, "invalid signature" unless
-      public_key.verify digest, signature, data.digest
+      public_key.verify filtered_digest, signature, data.digest
 
     true
   end
@@ -165,10 +167,13 @@ def check_trust(chain, digester, trust_dir)
     end
 
     save_cert = OpenSSL::X509::Certificate.new File.read path
-    save_dgst = digester.digest save_cert.public_key.to_pem
+    save_pkey = save_cert.public_key
+    save_pkey_str = Gem::Security.pemmable_to_pem(save_pkey)
+    save_dgst = digester.digest(save_pkey_str)
 
-    pkey_str = root.public_key.to_pem
-    cert_dgst = digester.digest pkey_str
+    root_pkey = root.public_key
+    root_pkey_str = Gem::Security.pemmable_to_pem(root_pkey)
+    cert_dgst = digester.digest(root_pkey_str)
 
     raise Gem::Security::Exception,
           "trusted root certificate #{root.subject} checksum " \

lib/rubygems/security/signer.rb

@@ -154,7 +154,8 @@ def sign(data)
 
     Gem::Security::SigningPolicy.verify @cert_chain, @key, {}, {}, full_name
 
-    @key.sign @digest_algorithm.new, data
+    digest = @digest_algorithm.new if Gem::Security.digest_required?(@key)
+    @key.sign digest, data
   end
 
   ##

test/rubygems/helper.rb

@@ -1721,6 +1721,39 @@ def self.key_path(key_name)
     PUBLIC_KEY  = nil
     PUBLIC_CERT = nil
   end if Gem::HAVE_OPENSSL
+
+  begin
+    ML_DSA_65_PRIVATE_KEY                 = load_key "mldsa65_private"
+    ML_DSA_65_PRIVATE_KEY_PATH            = key_path "mldsa65_private"
+
+    # ML_DSA_65_ENCRYPTED_PRIVATE_KEY is ML_DSA_65_PRIVATE_KEY encrypted with PRIVATE_KEY_PASSPHRASE
+    ML_DSA_65_ENCRYPTED_PRIVATE_KEY       = load_key "mldsa65_encrypted_private", PRIVATE_KEY_PASSPHRASE
+    ML_DSA_65_ENCRYPTED_PRIVATE_KEY_PATH  = key_path "mldsa65_encrypted_private"
+
+    ML_DSA_65_PUBLIC_KEY                  = OpenSSL::PKey.read(ML_DSA_65_PRIVATE_KEY.public_to_der)
+
+    ML_DSA_65_PUBLIC_CERT                 = load_cert "mldsa65_public"
+    ML_DSA_65_PUBLIC_CERT_PATH            = cert_path "mldsa65_public"
+  # OpenSSL::PKey::PKeyError is raised when ML-DSA is not supported by the
+  # OpenSSL version (requires OpenSSL >= 3.5).
+  rescue Errno::ENOENT, OpenSSL::PKey::PKeyError
+    ML_DSA_65_PRIVATE_KEY = nil
+    ML_DSA_65_PUBLIC_KEY  = nil
+    ML_DSA_65_PUBLIC_CERT = nil
+  end if Gem::HAVE_OPENSSL
+
+  require_relative "pqc_utilities"
+  include Gem::PqcUtilities
+
+  def omit_unless_support_pqc
+    without_pqc_support do |message|
+      omit message
+    end
+  end
+
+  def omit_if_support_pqc
+    omit "PQC is supported" if Gem::PqcUtilities.support_pqc?
+  end
 end
 
 # https://github.com/seattlerb/minitest/blob/13c48a03d84a2a87855a4de0c959f96800100357/lib/minitest/mock.rb#L192

test/rubygems/local_ssl_server_utilities.rb

@@ -5,8 +5,11 @@
 
 require "socket"
 require "openssl"
+require_relative "pqc_utilities"
 
 module Gem::LocalSSLServerUtilities
+  include Gem::PqcUtilities
+
   CERTS_DIR = __dir__
 
   def certs_dir
@@ -86,69 +89,4 @@ def cert(filename)
   def key(filename)
     OpenSSL::PKey.read(File.read(File.join(certs_dir, filename)))
   end
-
-  def without_pqc_support(&block)
-    # PQC algorithms ML-KEM and ML-DSA require OpenSSL >= 3.5.
-    # https://openssl-library.org/post/2025-04-08-openssl-35-final-release/
-    unless OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30500000
-      yield "PQC algorithms require OpenSSL >= 3.5"
-      return
-    end
-    # ctx.groups (OpenSSL::SSL::SSLContext#groups) used in start_ssl_server
-    # mode :pqc requires Ruby OpenSSL >= 4.0.
-    unless Gem::Version.new(OpenSSL::VERSION) >= Gem::Version.new("4.0")
-      yield "PQC test requires Ruby OpenSSL >= 4.0"
-      return
-    end
-    # Even with a new enough OpenSSL, the runtime may keep PQC groups and
-    # signature algorithms out of its default negotiation lists (for example
-    # RHEL's system-wide crypto policies). The PQC server forces both, while
-    # the gem fetcher connects with the default client configuration, so a
-    # real loopback handshake is the only reliable way to tell whether this
-    # environment can negotiate PQC at all.
-    unless Gem::LocalSSLServerUtilities.support_pqc_handshake?
-      yield "PQC handshake is not available in this OpenSSL configuration"
-    end
-  end
-
-  # Probe an actual PQC handshake between a forced-PQC server and a
-  # default-configured client, mirroring what the integration tests exercise.
-  # Memoized so the probe runs at most once per process.
-  def self.support_pqc_handshake?
-    return @support_pqc_handshake unless @support_pqc_handshake.nil?
-
-    @support_pqc_handshake = probe_pqc_handshake
-  end
-
-  def self.probe_pqc_handshake
-    server = TCPServer.new("127.0.0.1", 0)
-    ctx = OpenSSL::SSL::SSLContext.new
-    ctx.cert = OpenSSL::X509::Certificate.new(File.read(File.join(CERTS_DIR, "mldsa65_ssl_cert.pem")))
-    ctx.key = OpenSSL::PKey.read(File.read(File.join(CERTS_DIR, "mldsa65_ssl_key.pem")))
-    ctx.groups = "X25519MLKEM768"
-    ssl_server = OpenSSL::SSL::SSLServer.new(server, ctx)
-
-    port = server.addr[1]
-    server_thread = Thread.new do
-      client = ssl_server.accept
-      client.close
-    rescue OpenSSL::OpenSSLError
-      nil
-    end
-
-    client_ctx = OpenSSL::SSL::SSLContext.new
-    client_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
-    socket = TCPSocket.new("127.0.0.1", port)
-    ssl = OpenSSL::SSL::SSLSocket.new(socket, client_ctx)
-    ssl.connect
-    ssl.close
-    true
-  rescue OpenSSL::OpenSSLError, SystemCallError
-    false
-  ensure
-    server_thread&.join(5)
-    server_thread&.kill if server_thread&.alive?
-    ssl_server&.close
-    server&.close
-  end
 end