fix: signature verification due to leading zeros

According to FIPS 186-5, section 6.4.2 ECDSA Signature
Verification Algorithm, the hash of the message must be adjusted
based on the order n of the base point of the elliptic curve:

    If log2(n) ≥ hashlen, set E = H. Otherwise, set E equal to
    the leftmost log2(n) bits of H.

Unfortunately because elliptic converts messages to BN instances the
reported `byteLength()` for the message can be incorrect if the message
has 8 or more leading zero bits.

Here we fix it by:

1. Counting leading zeroes in hex strings provided as messages
2. Counting all array entries in Array-like (e.g. Buffer)
   messages
3. Providing an `msgBitLength` option to both `.sign`/`.verify` to let
   user override the behavior

Original PR: https://github.com/indutny/elliptic/pull/322
Credit: @Markus-MS

lib/elliptic/ec/index.js

@@ -78,8 +78,27 @@ EC.prototype.genKeyPair = function genKeyPair(options) {
   }
 };
 
-EC.prototype._truncateToN = function _truncateToN(msg, truncOnly) {
-  var delta = msg.byteLength() * 8 - this.n.bitLength();
+EC.prototype._truncateToN = function _truncateToN(msg, truncOnly, bitLength) {
+  var byteLength;
+  if (BN.isBN(msg) || typeof msg === 'number') {
+    msg = new BN(msg, 16);
+    byteLength = msg.byteLength();
+  } else if (typeof msg === 'object') {
+    // BN assumes an array-like input and asserts length
+    byteLength = msg.length;
+    msg = new BN(msg, 16);
+  } else {
+    // BN converts the value to string
+    var str = msg.toString();
+    // HEX encoding
+    byteLength = (str.length + 1) >>> 1;
+    msg = new BN(str, 16);
+  }
+  // Allow overriding
+  if (typeof bitLength !== 'number') {
+    bitLength = byteLength * 8;
+  }
+  var delta = bitLength - this.n.bitLength();
   if (delta > 0)
     msg = msg.ushrn(delta);
   if (!truncOnly && msg.cmp(this.n) >= 0)
@@ -97,7 +116,7 @@ EC.prototype.sign = function sign(msg, key, enc, options) {
     options = {};
 
   key = this.keyFromPrivate(key, enc);
-  msg = this._truncateToN(new BN(msg, 16));
+  msg = this._truncateToN(msg, false, options.msgBitLength);
 
   // Zero-extend key to provide enough entropy
   var bytes = this.n.byteLength();
@@ -153,8 +172,11 @@ EC.prototype.sign = function sign(msg, key, enc, options) {
   }
 };
 
-EC.prototype.verify = function verify(msg, signature, key, enc) {
-  msg = this._truncateToN(new BN(msg, 16));
+EC.prototype.verify = function verify(msg, signature, key, enc, options) {
+  if (!options)
+    options = {};
+
+  msg = this._truncateToN(msg, false, options.msgBitLength);
   key = this.keyFromPublic(key, enc);
   signature = new Signature(signature, 'hex');
 

lib/elliptic/ec/key.js

@@ -111,8 +111,8 @@ KeyPair.prototype.sign = function sign(msg, enc, options) {
   return this.ec.sign(msg, this, enc, options);
 };
 
-KeyPair.prototype.verify = function verify(msg, signature) {
-  return this.ec.verify(msg, signature, this);
+KeyPair.prototype.verify = function verify(msg, signature, options) {
+  return this.ec.verify(msg, signature, this, undefined, options);
 };
 
 KeyPair.prototype.inspect = function inspect() {

test/ecdsa-test.js

@@ -489,6 +489,50 @@ describe('ECDSA', function() {
       });
     });
 
+  it('Wycheproof special hash case with hex', function() {
+    var curve = new elliptic.ec('p192');
+    var msg =
+      '00000000690ed426ccf17803ebe2bd0884bcd58a1bb5e7477ead3645f356e7a9';
+    var sig = '303502186f20676c0d04fc40ea55d5702f798355787363a9' +
+              '1e97a7e50219009d1c8c171b2b02e7d791c204c17cea4cf5' +
+              '56a2034288885b';
+    var pub = '04cd35a0b18eeb8fcd87ff019780012828745f046e785deb' +
+              'a28150de1be6cb4376523006beff30ff09b4049125ced29723';
+    var pubKey = curve.keyFromPublic(pub, 'hex');
+    assert(pubKey.verify(msg, sig) === true);
+  });
+
+  it('Wycheproof special hash case with Array', function() {
+    var curve = new elliptic.ec('p192');
+    var msg = [
+      0x00, 0x00, 0x00, 0x00, 0x69, 0x0e, 0xd4, 0x26, 0xcc, 0xf1, 0x78,
+      0x03, 0xeb, 0xe2, 0xbd, 0x08, 0x84, 0xbc, 0xd5, 0x8a, 0x1b, 0xb5,
+      0xe7, 0x47, 0x7e, 0xad, 0x36, 0x45, 0xf3, 0x56, 0xe7, 0xa9,
+    ];
+    var sig = '303502186f20676c0d04fc40ea55d5702f798355787363a9' +
+              '1e97a7e50219009d1c8c171b2b02e7d791c204c17cea4cf5' +
+              '56a2034288885b';
+    var pub = '04cd35a0b18eeb8fcd87ff019780012828745f046e785deb' +
+              'a28150de1be6cb4376523006beff30ff09b4049125ced29723';
+    var pubKey = curve.keyFromPublic(pub, 'hex');
+    assert(pubKey.verify(msg, sig) === true);
+  });
+
+  it('Wycheproof special hash case with BN', function() {
+    var curve = new elliptic.ec('p192');
+    var msg = new BN(
+      '00000000690ed426ccf17803ebe2bd0884bcd58a1bb5e7477ead3645f356e7a9',
+      16,
+    );
+    var sig = '303502186f20676c0d04fc40ea55d5702f798355787363a9' +
+              '1e97a7e50219009d1c8c171b2b02e7d791c204c17cea4cf5' +
+              '56a2034288885b';
+    var pub = '04cd35a0b18eeb8fcd87ff019780012828745f046e785deb' +
+              'a28150de1be6cb4376523006beff30ff09b4049125ced29723';
+    var pubKey = curve.keyFromPublic(pub, 'hex');
+    assert(pubKey.verify(msg, sig, { msgBitLength: 32 * 8 }) === true);
+  });
+
   describe('Signature', function () {
     it('recoveryParam is 0', function () {
       var sig = new Signature({ r: '00', s: '00', recoveryParam: 0 });