fix: read and write masternode list diffs in Core's P2P format

lib/block/PartialMerkleTree.js

@@ -6,6 +6,10 @@ const isHexString = require('../util/js').isHexa;
 const constants = require('../constants');
 const MerkleTreeUtils = require('../util/merkletree');
 const BitArrayUtils = require('../util/bitarray');
+const readVarintCount = require('../util/readVarintCount');
+const isDenseArray = require('../util/isDenseArray');
+const $ = require('../util/preconditions');
+const { readExactReverse } = require('../transaction/payload/strictpayloadreader');
 
 const { convertBitArrayToUInt8Array } = BitArrayUtils;
 
@@ -63,15 +67,19 @@ PartialMerkleTree.fromBufferReader = function fromBufferReader(bufferReader) {
   const partialMerkleTree = new PartialMerkleTree();
   partialMerkleTree.totalTransactions = bufferReader.readUInt32LE();
 
-  const merkleHashesCount = bufferReader.readVarintNum();
+  const merkleHashesCount = readVarintCount(
+    bufferReader,
+    constants.SHA256_HASH_SIZE,
+    'Merkle hash'
+  );
   partialMerkleTree.merkleHashes = [];
   for (let i = 0; i < merkleHashesCount; i += 1) {
     partialMerkleTree.merkleHashes.push(
-      bufferReader.read(constants.SHA256_HASH_SIZE).reverse().toString('hex')
+      readExactReverse(bufferReader, constants.SHA256_HASH_SIZE).toString('hex')
     );
   }
 
-  const merkleFlagsCount = bufferReader.readVarintNum();
+  const merkleFlagsCount = readVarintCount(bufferReader, 1, 'Merkle flag byte');
   partialMerkleTree.merkleFlags = [];
   for (let i = 0; i < merkleFlagsCount; i += 1) {
     partialMerkleTree.merkleFlags.push(bufferReader.readUInt8());
@@ -106,6 +114,15 @@ PartialMerkleTree.prototype.toBuffer = function toBuffer() {
 
   bufferWriter.writeUInt32LE(this.totalTransactions);
 
+  // forEach skips empty slots, which would leave the counts wrong
+  $.checkArgument(
+    isDenseArray(this.merkleHashes),
+    'Expected merkleHashes to be an array without empty slots'
+  );
+  $.checkArgument(
+    isDenseArray(this.merkleFlags),
+    'Expected merkleFlags to be an array without empty slots'
+  );
   bufferWriter.writeVarintNum(this.merkleHashes.length);
   this.merkleHashes.forEach((hash) => {
     bufferWriter.write(Buffer.from(hash, 'hex').reverse());

lib/constants/index.js

@@ -65,7 +65,7 @@ const constants = {
     // 60 members, 45 (75%) threshold, one every 12 hours
     LLMQ_TYPE_60_75: 5,
 
-    // 25 members, 67 (67%) threshold, one per hour
+    // 25 members, 17 (67%) threshold, one per hour
     LLMQ_TYPE_25_67: 6,
 
     // 3 members, 2 (66%) threshold, one per hour (24 blocks)
@@ -80,7 +80,7 @@ const constants = {
     // Params might differ when -llmqtestparams is used
     LLMQ_TYPE_TEST_V17: 102,
 
-    // 4 members, 2 (66%) threshold, one per hour.
+    // 4 members, 3 (75%) threshold, one per hour.
     // Params might differ when -llmqtestparams is used
     LLMQ_TYPE_TEST_DIP0024: 103,
 
@@ -141,7 +141,7 @@ constants.LLMQ_TYPE_PARAMS= {
   },
   [constants.LLMQ_TYPES.LLMQ_TYPE_25_67]: {
     size: 25,
-      threshold: 67,
+      threshold: 17,
       maximumActiveQuorumsCount: 24,
   },
   [constants.LLMQ_TYPES.LLMQ_TYPE_LLMQ_TEST]: {
@@ -161,7 +161,7 @@ constants.LLMQ_TYPE_PARAMS= {
   },
   [constants.LLMQ_TYPES.LLMQ_TYPE_TEST_DIP0024]: {
     size: 4,
-      threshold: 2,
+      threshold: 3,
       maximumActiveQuorumsCount: 2,
   },
   [constants.LLMQ_TYPES.LLMQ_TYPE_TEST_INSTANTSEND]: {

lib/deterministicmnlist/QuorumEntry.js

@@ -9,6 +9,11 @@ const Hash = require('../crypto/hash');
 const bls = require('../crypto/bls');
 const utils = require('../util/js');
 const isHashQuorumIndexRequired = require('../util/isHashQuorumIndexRequired');
+const {
+  readExact,
+  readExactReverse,
+  readCompactSize,
+} = require('../transaction/payload/strictpayloadreader');
 
 const {
   isHexStringOfSize,
@@ -83,6 +88,36 @@ function QuorumEntry(arg, llmqParamsOverride = {}) {
   }
 }
 
+/**
+ * Reads the fields that precede a commitment's bit sets
+ * @param {BufferReader} bufferReader
+ * @return {{version: number, llmqType: number, quorumHash: string, quorumIndex: number}}
+ */
+function readCommitmentHead(bufferReader) {
+  const version = bufferReader.readUInt16LE();
+  return {
+    version,
+    llmqType: bufferReader.readUInt8(),
+    quorumHash: readExactReverse(bufferReader, SHA256_HASH_SIZE).toString('hex'),
+    quorumIndex: isHashQuorumIndexRequired(version) ? bufferReader.readInt16LE() : 0,
+  };
+}
+
+/**
+ * Reads the fields that follow a commitment's bit sets
+ * @param {BufferReader} bufferReader
+ * @return {{quorumPublicKey: string, quorumVvecHash: string, quorumSig: string,
+ * membersSig: string}}
+ */
+function readCommitmentTail(bufferReader) {
+  return {
+    quorumPublicKey: readExact(bufferReader, BLS_PUBLIC_KEY_SIZE).toString('hex'),
+    quorumVvecHash: readExactReverse(bufferReader, SHA256_HASH_SIZE).toString('hex'),
+    quorumSig: readExact(bufferReader, BLS_SIGNATURE_SIZE).toString('hex'),
+    membersSig: readExact(bufferReader, BLS_SIGNATURE_SIZE).toString('hex'),
+  };
+}
+
 /**
  * Parse buffer and returns QuorumEntry
  * @param {Buffer} buffer
@@ -99,8 +134,7 @@ QuorumEntry.fromBuffer = function fromBuffer(buffer, llmqParamsOverride = {}) {
     SMLQuorumEntry.version = bufferReader.readUInt16LE();
     SMLQuorumEntry.llmqType = bufferReader.readUInt8();
     SMLQuorumEntry.quorumHash = bufferReader
-      .read(constants.SHA256_HASH_SIZE)
-      .reverse()
+      .readReverse(constants.SHA256_HASH_SIZE)
       .toString('hex');
 
     if (isHashQuorumIndexRequired(this.version)) {
@@ -118,18 +152,7 @@ QuorumEntry.fromBuffer = function fromBuffer(buffer, llmqParamsOverride = {}) {
     return SMLQuorumEntry;
   }
   SMLQuorumEntry.isOutdatedRPC = false;
-  SMLQuorumEntry.version = bufferReader.readUInt16LE();
-  SMLQuorumEntry.llmqType = bufferReader.readUInt8();
-  SMLQuorumEntry.quorumHash = bufferReader
-    .read(constants.SHA256_HASH_SIZE)
-    .reverse()
-    .toString('hex');
-
-  if (isHashQuorumIndexRequired(SMLQuorumEntry.version)) {
-    SMLQuorumEntry.quorumIndex = bufferReader.readInt16LE();
-  } else {
-    SMLQuorumEntry.quorumIndex = 0;
-  }
+  Object.assign(SMLQuorumEntry, readCommitmentHead(bufferReader));
 
   SMLQuorumEntry.signersCount = bufferReader.readVarintNum();
   const signersBytesToRead =
@@ -143,23 +166,97 @@ QuorumEntry.fromBuffer = function fromBuffer(buffer, llmqParamsOverride = {}) {
   SMLQuorumEntry.validMembers = bufferReader
     .read(validMembersBytesToRead)
     .toString('hex');
-  SMLQuorumEntry.quorumPublicKey = bufferReader
-    .read(BLS_PUBLIC_KEY_SIZE)
-    .toString('hex');
-  SMLQuorumEntry.quorumVvecHash = bufferReader
-    .read(SHA256_HASH_SIZE)
-    .reverse()
-    .toString('hex');
-  SMLQuorumEntry.quorumSig = bufferReader
-    .read(BLS_SIGNATURE_SIZE)
-    .toString('hex');
-  SMLQuorumEntry.membersSig = bufferReader
-    .read(BLS_SIGNATURE_SIZE)
-    .toString('hex');
+  Object.assign(SMLQuorumEntry, readCommitmentTail(bufferReader));
 
   return SMLQuorumEntry;
 };
 
+/**
+ * Whether the unused high bits of a bit set's last byte are set, which Core's
+ * ReadFixedBitSet rejects
+ * @param {Buffer} bytes
+ * @param {number} size - number of bits
+ * @return {boolean}
+ */
+function hasOutOfRangeBits(bytes, size) {
+  const unused = bytes.length * 8 - size;
+  // eslint-disable-next-line no-bitwise
+  return unused > 0 && (bytes[bytes.length - 1] & ~(0xff >>> unused) & 0xff) !== 0;
+}
+
+/**
+ * Checks a bit set before it is written after the LLMQ size: a wrong length
+ * would shift every field that follows, and Core rejects bits past the size
+ * @param {string} bits - packed bits hex
+ * @param {number} size - LLMQ size in bits
+ * @param {string} name
+ */
+function checkBitSet(bits, size, name) {
+  $.checkArgument(
+    bits.length === Math.ceil(size / 8) * 2,
+    `Expected ${name} to have the LLMQ bit set length`
+  );
+  $.checkArgument(
+    !hasOutOfRangeBits(Buffer.from(bits, 'hex'), size),
+    'Expected no bits set past the LLMQ size'
+  );
+}
+
+/**
+ * Reads Core's dynamic bit set: its bit count, then the bits packed into bytes
+ * @param {BufferReader} bufferReader
+ * @param {Object} [params] - LLMQ type params, if known; Core requires the
+ * bit count to equal their size
+ * @return {{count: number, bits: string}} number of set bits, packed bits hex
+ */
+function readDynamicBitSet(bufferReader, params) {
+  // at most MAX_SIZE bits, and readExact bounds the bytes by the remaining data
+  const size = readCompactSize(bufferReader);
+  if (params && size !== params.size) {
+    throw new Error(`Expected a bit set of ${params.size} bits, got ${size}`);
+  }
+  const bytes = readExact(bufferReader, Math.ceil(size / 8));
+  if (hasOutOfRangeBits(bytes, size)) {
+    throw new Error('Out-of-range bits set');
+  }
+  let count = 0;
+  bytes.forEach((byte) => {
+    // clear the lowest set bit until none is left: at most 8 steps per byte
+    for (let bits = byte; bits !== 0; count += 1) {
+      // eslint-disable-next-line no-bitwise
+      bits &= bits - 1;
+    }
+  });
+  return { count, bits: bytes.toString('hex') };
+}
+
+/**
+ * Reads a final commitment as Core sends it in the P2P `mnlistdiff` message,
+ * the layout toBufferForHashing() writes. Like Core's RPC, signersCount and
+ * validMembersCount are the numbers of set bits.
+ * @param {BufferReader} bufferReader
+ * @param {Object} [llmqParamsOverride]
+ * @return {QuorumEntry}
+ */
+QuorumEntry.fromP2PBufferReader = function fromP2PBufferReader(
+  bufferReader,
+  llmqParamsOverride = {}
+) {
+  const obj = readCommitmentHead(bufferReader);
+
+  // Unknown types are read without the size check, but can't be serialized
+  const params = _.merge({}, constants.LLMQ_TYPE_PARAMS, llmqParamsOverride)[obj.llmqType];
+  const signers = readDynamicBitSet(bufferReader, params);
+  obj.signersCount = signers.count;
+  obj.signers = signers.bits;
+  const validMembers = readDynamicBitSet(bufferReader, params);
+  obj.validMembersCount = validMembers.count;
+  obj.validMembers = validMembers.bits;
+  Object.assign(obj, readCommitmentTail(bufferReader));
+
+  return QuorumEntry.fromObject(obj, llmqParamsOverride);
+};
+
 /**
  * @param {string} string
  * @param {Object} llmqParamsOverride
@@ -213,6 +310,8 @@ QuorumEntry.prototype.toBufferForHashing = function toBufferForHashing() {
   this.validate();
   const bufferWriter = new BufferWriter();
   const fixedCounterLength = this.getParams().size;
+  checkBitSet(this.signers, fixedCounterLength, 'signers');
+  checkBitSet(this.validMembers, fixedCounterLength, 'validMembers');
   bufferWriter.writeUInt16LE(this.version);
   bufferWriter.writeUInt8(this.llmqType);
   bufferWriter.write(Buffer.from(this.quorumHash, 'hex').reverse());
@@ -368,10 +467,12 @@ QuorumEntry.prototype.getParams = function getParams() {
  * @return {Uint8Array}
  */
 QuorumEntry.prototype.getCommitmentHash = function getCommitmentHash() {
+  const { size } = this.getParams();
+  checkBitSet(this.validMembers, size, 'validMembers');
   const bufferWriter = new BufferWriter();
   bufferWriter.writeUInt8(this.llmqType);
   bufferWriter.write(Buffer.from(this.quorumHash, 'hex').reverse());
-  bufferWriter.writeVarintNum(this.getParams().size);
+  bufferWriter.writeVarintNum(size);
   bufferWriter.write(Buffer.from(this.validMembers, 'hex'));
   bufferWriter.write(Buffer.from(this.quorumPublicKey, 'hex'));
   bufferWriter.write(Buffer.from(this.quorumVvecHash, 'hex').reverse());
@@ -543,7 +644,8 @@ QuorumEntry.prototype.calculateHash = function calculateHash() {
  * @return {QuorumEntry}
  */
 QuorumEntry.prototype.copy = function copy() {
-  return QuorumEntry.fromBuffer(this.toBuffer());
+  // fromBuffer needs the params of custom quorum types for the bit set sizes
+  return QuorumEntry.fromBuffer(this.toBuffer(), this.llmqParamsOverride);
 };
 
 module.exports = QuorumEntry;