Avoid overflow issues in X509_cmp.

The length is a long, so returning the difference does not quite work. Thanks to Torbjörn Granlund for noticing. Reviewed-by: Rich Salz <[email protected]> Reviewed-by: Richard Levitte <[email protected]>
enh · Apr 29, 2016 · 87a8405 · 87a8405
1 parent a1f4128
commit 87a8405
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
@@ -187,9 +187,10 @@ int X509_cmp(const X509 *a, const X509 *b)
         return rv;
     /* Check for match against stored encoding too */
     if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) {
-        rv = (int)(a->cert_info.enc.len - b->cert_info.enc.len);
-        if (rv)
-            return rv;
+        if (a->cert_info.enc.len < b->cert_info.enc.len)
+            return -1;
+        if (a->cert_info.enc.len > b->cert_info.enc.len)
+            return 1;
         return memcmp(a->cert_info.enc.enc, b->cert_info.enc.enc,
                       a->cert_info.enc.len);
     }