fix: use runtime OpenSSL algorithm detection to handle missing SM3 support

keylime-ima-emulator/src/main.rs

@@ -62,11 +62,11 @@ fn ml_extend(
 ) -> Result<usize> {
     let f = File::open(ml)?;
     let mut reader = BufReader::new(f);
-    let ima_digest: MessageDigest = ima_hash_alg.into();
-    let ima_start_hash = ima::Digest::start(ima_hash_alg);
-    let pcr_digest: MessageDigest = pcr_hash_alg.into();
-    let mut running_hash = ima::Digest::start(pcr_hash_alg);
-    let ff_hash = ima::Digest::ff(pcr_hash_alg);
+    let ima_digest: MessageDigest = MessageDigest::try_from(ima_hash_alg)?;
+    let ima_start_hash = ima::Digest::start(ima_hash_alg)?;
+    let pcr_digest: MessageDigest = MessageDigest::try_from(pcr_hash_alg)?;
+    let mut running_hash = ima::Digest::start(pcr_hash_alg)?;
+    let ff_hash = ima::Digest::ff(pcr_hash_alg)?;
     for line in reader.by_ref().lines().skip(position) {
         let line = line?;
         if line.is_empty() {
@@ -198,7 +198,8 @@ fn main() -> std::result::Result<(), ImaEmulatorError> {
                 )
             })?;
 
-        let pcr_digest: MessageDigest = (*pcr_hash_alg).into();
+        let pcr_digest: MessageDigest =
+            MessageDigest::try_from(*pcr_hash_alg)?;
         let pcr_start_hash = vec![0x00u8; pcr_digest.size()];
         if digest.value() != pcr_start_hash {
             log::warn!("IMA PCR is not empty, trying to find the last updated file in the measurement list...");

keylime/src/algorithms.rs

@@ -113,16 +113,33 @@ impl From<HashAlgorithm> for HashingAlgorithm {
     }
 }
 
-impl From<HashAlgorithm> for MessageDigest {
-    fn from(hash_algorithm: HashAlgorithm) -> Self {
-        match hash_algorithm {
-            HashAlgorithm::Sha1 => MessageDigest::sha1(),
-            HashAlgorithm::Sha256 => MessageDigest::sha256(),
-            HashAlgorithm::Sha384 => MessageDigest::sha384(),
-            HashAlgorithm::Sha512 => MessageDigest::sha512(),
-            HashAlgorithm::Sm3_256 => MessageDigest::sm3(),
+impl HashAlgorithm {
+    fn openssl_name(&self) -> &'static str {
+        match self {
+            HashAlgorithm::Sha1 => "sha1",
+            HashAlgorithm::Sha256 => "sha256",
+            HashAlgorithm::Sha384 => "sha384",
+            HashAlgorithm::Sha512 => "sha512",
+            HashAlgorithm::Sm3_256 => "sm3",
         }
     }
+
+    pub fn is_supported(&self) -> bool {
+        MessageDigest::from_name(self.openssl_name()).is_some()
+    }
+}
+
+impl TryFrom<HashAlgorithm> for MessageDigest {
+    type Error = AlgorithmError;
+
+    fn try_from(hash_algorithm: HashAlgorithm) -> Result<Self, Self::Error> {
+        let name = hash_algorithm.openssl_name();
+        MessageDigest::from_name(name).ok_or_else(|| {
+            AlgorithmError::UnsupportedHashingAlgorithm(format!(
+                "{name} (not supported by this OpenSSL build)"
+            ))
+        })
+    }
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -473,6 +490,17 @@ mod tests {
         assert!(result.is_err());
     }
 
+    #[test]
+    fn test_hash_to_message_digest() {
+        assert!(MessageDigest::try_from(HashAlgorithm::Sha1).is_ok());
+        assert!(MessageDigest::try_from(HashAlgorithm::Sha256).is_ok());
+        assert!(MessageDigest::try_from(HashAlgorithm::Sha384).is_ok());
+        assert!(MessageDigest::try_from(HashAlgorithm::Sha512).is_ok());
+        // SM3 may or may not be available depending on OpenSSL build;
+        // just verify it does not panic
+        let _ = MessageDigest::try_from(HashAlgorithm::Sm3_256);
+    }
+
     #[test]
     fn test_hash_to_hashing_algorithm() {
         let cases = [

keylime/src/context_info.rs

@@ -160,6 +160,16 @@ impl ContextInfo {
         let tpm_hash_alg = config.tpm_hash_alg;
         let tpm_signing_alg = config.tpm_signing_alg;
 
+        if !tpm_hash_alg.is_supported() {
+            return Err(ContextInfoError::InvalidAlgorithm(
+                algorithms::AlgorithmError::UnsupportedHashingAlgorithm(
+                    format!(
+                        "{tpm_hash_alg} (not supported by this OpenSSL build)"
+                    ),
+                ),
+            ));
+        }
+
         let ek_result = tpm_context.create_ek(tpm_encryption_alg, None)?;
         let ek_handle = ek_result.key_handle;
         let ek_hash = hash_ek::hash_ek_pubkey(ek_result.public.clone())
@@ -372,7 +382,8 @@ impl ContextInfo {
         let keylime_hash_alg: KeylimeInternalHashAlgorithm =
             name_h_alg_tss.try_into()?;
         let name_alg_id_value: u16 = name_h_alg_tss.into();
-        let openssl_message_digest: MessageDigest = keylime_hash_alg.into();
+        let openssl_message_digest: MessageDigest =
+            MessageDigest::try_from(keylime_hash_alg)?;
         let mut hasher = Hasher::new(openssl_message_digest)?;
         hasher.update(&marshalled_tpmt_public)?;
         let digest_bytes_vec = hasher.finish()?;

keylime/src/ima/entry.rs

@@ -32,7 +32,8 @@ pub struct Digest {
 impl Digest {
     /// Creates a new `Digest` with `algorithm` and `value`.
     pub fn new(algorithm: HashAlgorithm, value: &[u8]) -> Result<Self> {
-        let digest: MessageDigest = algorithm.into();
+        let digest: MessageDigest =
+            MessageDigest::try_from(algorithm).map_err(Error::other)?;
         if value.len() != digest.size() {
             return Err(Error::new(
                 ErrorKind::InvalidInput,
@@ -54,22 +55,24 @@ impl Digest {
 
     /// Returns a pre-defined digest value used to indicate the start
     /// of the IMA measurement list.
-    pub fn start(algorithm: HashAlgorithm) -> Self {
-        let digest: MessageDigest = algorithm.into();
-        Self {
+    pub fn start(algorithm: HashAlgorithm) -> Result<Self> {
+        let digest: MessageDigest =
+            MessageDigest::try_from(algorithm).map_err(Error::other)?;
+        Ok(Self {
             algorithm,
             value: vec![0x00u8; digest.size()],
-        }
+        })
     }
 
     /// Returns a pre-defined digest value used to indicate the ToMToU
     /// error in the IMA measurement list.
-    pub fn ff(algorithm: HashAlgorithm) -> Self {
-        let digest: MessageDigest = algorithm.into();
-        Self {
+    pub fn ff(algorithm: HashAlgorithm) -> Result<Self> {
+        let digest: MessageDigest =
+            MessageDigest::try_from(algorithm).map_err(Error::other)?;
+        Ok(Self {
             algorithm,
             value: vec![0xffu8; digest.size()],
-        }
+        })
     }
 }
 

keylime/src/tpm.rs

@@ -1818,6 +1818,16 @@ impl Context<'_> {
                 }
             }
         }
+        usable_algs.retain(|alg| {
+            let supported = alg.is_supported();
+            if !supported {
+                log::warn!(
+                    "Hash algorithm {alg} supported by TPM but not by \
+                     this OpenSSL build; excluding"
+                );
+            }
+            supported
+        });
         Ok(usable_algs)
     }
 
@@ -2329,14 +2339,14 @@ fn make_pcr_blob(
 fn hash_alg_to_message_digest(
     hash_alg: HashingAlgorithm,
 ) -> Result<MessageDigest> {
-    match hash_alg {
-        HashingAlgorithm::Sha256 => Ok(MessageDigest::sha256()),
-        HashingAlgorithm::Sha1 => Ok(MessageDigest::sha1()),
-        HashingAlgorithm::Sha384 => Ok(MessageDigest::sha384()),
-        HashingAlgorithm::Sha512 => Ok(MessageDigest::sha512()),
-        HashingAlgorithm::Sm3_256 => Ok(MessageDigest::sm3()),
-        other => Err(TpmError::UnsupportedHashingAlgorithm { alg: other }),
-    }
+    let keylime_alg = KeylimeInternalHashAlgorithm::try_from(hash_alg)
+        .map_err(|_| TpmError::UnsupportedHashingAlgorithm {
+            alg: hash_alg,
+        })?;
+    MessageDigest::try_from(keylime_alg).map_err(|e| {
+        log::debug!("OpenSSL digest lookup failed for {hash_alg:?}: {e}");
+        TpmError::UnsupportedHashingAlgorithm { alg: hash_alg }
+    })
 }
 
 /// Check if the data attested in the quote matches the data read from the TPM PCRs