Merge pull request #843 from EYBlockchain/ilyas/flexible-circuits

Flexible Input/Output circuits

common-files/classes/transaction.mjs

@@ -14,6 +14,13 @@ const { generalise } = gen;
 const TOKEN_TYPES = { ERC20: 0, ERC721: 1, ERC1155: 2 };
 const { TRANSACTION_TYPES } = constants;
 
+const arrayEquality = (as, bs) => {
+  if (as.length === bs.length) {
+    return as.every(a => bs.includes(a));
+  }
+  return false;
+};
+
 // function to compute the keccak hash of a transaction
 function keccak(preimage) {
   const web3 = Web3.connection();
@@ -30,7 +37,7 @@ function keccak(preimage) {
     compressedSecrets,
   } = preimage;
   let { proof } = preimage;
-  proof = compressProof(proof);
+  proof = arrayEquality(proof, [0, 0, 0, 0, 0, 0, 0, 0]) ? [0, 0, 0, 0] : compressProof(proof);
   const transaction = [
     value,
     historicRootBlockNumberL2,
@@ -57,7 +64,7 @@ class Transaction {
   // them undefined work?)
   constructor({
     fee,
-    historicRootBlockNumberL2,
+    historicRootBlockNumberL2: _historicRoot,
     transactionType,
     tokenType,
     tokenId,
@@ -69,11 +76,13 @@ class Transaction {
     compressedSecrets: _compressedSecrets, // this must be array of objects that are compressed from Secrets class
     proof, // this must be a proof object, as computed by zokrates worker
   }) {
-    if (proof === undefined) throw new Error('Proof cannot be undefined');
-    const flatProof = Object.values(proof).flat(Infinity);
     let commitments;
     let nullifiers;
     let compressedSecrets;
+    let flatProof;
+    let historicRootBlockNumberL2;
+    if (proof === undefined) flatProof = [0, 0, 0, 0, 0, 0, 0, 0];
+    else flatProof = Object.values(proof).flat(Infinity);
     if (_commitments === undefined) commitments = [{ hash: 0 }, { hash: 0 }];
     else if (_commitments.length === 1) commitments = [..._commitments, { hash: 0 }];
     else commitments = _commitments;
@@ -82,8 +91,11 @@ class Transaction {
     else nullifiers = _nullifiers;
     if (_compressedSecrets === undefined) compressedSecrets = [0, 0];
     else compressedSecrets = _compressedSecrets;
+    if (_historicRoot === undefined) historicRootBlockNumberL2 = [0, 0];
+    else if (_historicRoot.length === 1) historicRootBlockNumberL2 = [..._historicRoot, 0];
+    else historicRootBlockNumberL2 = _historicRoot;
 
-    if ((transactionType === 0 || transactionType === 3) && TOKEN_TYPES[tokenType] === undefined)
+    if ((transactionType === 0 || transactionType === 2) && TOKEN_TYPES[tokenType] === undefined)
       throw new Error('Unrecognized token type');
     // convert everything to hex(32) for interfacing with web3
     const preimage = generalise({
@@ -143,7 +155,7 @@ class Transaction {
       commitments,
       nullifiers,
       compressedSecrets,
-      proof: compressProof(proof),
+      proof: arrayEquality(proof, [0, 0, 0, 0, 0, 0, 0, 0]) ? [0, 0, 0, 0] : compressProof(proof),
     };
   }
 }

config/default.js

@@ -103,7 +103,7 @@ module.exports = {
   TRANSACTIONS_PER_BLOCK: Number(process.env.TRANSACTIONS_PER_BLOCK) || 2,
   RETRIES: Number(process.env.AUTOSTART_RETRIES) || 50,
   USE_STUBS: process.env.USE_STUBS === 'true',
-  VK_IDS: { deposit: 0, single_transfer: 1, double_transfer: 2, withdraw: 3 }, // used as an enum to mirror the Shield contracts enum for vk types. The keys of this object must correspond to a 'folderpath' (the .zok file without the '.zok' bit)
+  VK_IDS: { deposit: 0, transfer: 1, withdraw: 2 }, // withdraw: 3, withdraw_change: 4 }, // used as an enum to mirror the Shield contracts enum for vk types. The keys of this object must correspond to a 'folderpath' (the .zok file without the '.zok' bit)
   MAX_PUBLIC_VALUES: {
     ERCADDRESS: 2n ** 161n - 1n,
     COMMITMENT: 2n ** 249n - 1n,

nightfall-client/src/classes/commitment.mjs

@@ -30,7 +30,10 @@ class Commitment {
     // the compressedPkd is not part of the pre-image but it's used widely in the rest of
     // the code, so we hold it in the commitment object (but not as part of the preimage)
     this.preimage = generalise(items);
-    this.compressedZkpPublicKey = ZkpKeys.compressZkpPublicKey(this.preimage.zkpPublicKey);
+    this.compressedZkpPublicKey =
+      this.preimage.zkpPublicKey[0] === 0
+        ? [0, 0]
+        : ZkpKeys.compressZkpPublicKey(this.preimage.zkpPublicKey);
     // we encode the top four bytes of the tokenId into the empty bytes at the top of the erc address.
     // this is consistent to what we do in the ZKP circuits
     const [top4Bytes, remainder] = this.preimage.tokenId.limbs(224, 2).map(l => BigInt(l));

nightfall-client/src/event-handlers/block-proposed.mjs

@@ -76,12 +76,10 @@ async function blockProposedEventHandler(data, syncing) {
     } else if (transaction.transactionType === '0' && countOfNonZeroCommitments >= 1) {
       // case when deposit transaction created by user
       saveTxToDb = true;
-    } else if (transaction.transactionType === '3' && countOfNonZeroNullifiers >= 1) {
-      // case when withdraw transaction created by user
-      saveTxToDb = true;
     }
 
-    if (saveTxToDb)
+    if (saveTxToDb) {
+      logger.info('Saving Tx', transaction.transactionHash);
       await saveTransaction({
         transactionHashL1,
         blockNumber: data.blockNumber,
@@ -92,6 +90,7 @@ async function blockProposedEventHandler(data, syncing) {
         if (!syncing || !err.message.includes('replay existing transaction')) throw err;
         logger.warn('Attempted to replay existing transaction. This is expected while syncing');
       });
+    }
 
     return Promise.all([
       saveTxToDb,

nightfall-client/src/routes/transfer.mjs

@@ -10,10 +10,10 @@ const router = express.Router();
 router.post('/', async (req, res, next) => {
   logger.debug(`transfer endpoint received POST ${JSON.stringify(req.body, null, 2)}`);
   try {
-    const { rawTransaction: txDataToSign, transaction, salts } = await transfer(req.body);
+    const { rawTransaction: txDataToSign, transaction } = await transfer(req.body);
     logger.debug('returning raw transaction');
     logger.silly(` raw transaction is ${JSON.stringify(txDataToSign, null, 2)}`);
-    res.json({ txDataToSign, transaction, salts });
+    res.json({ txDataToSign, transaction });
   } catch (err) {
     logger.error(err);
     if (err.message.includes('No suitable commitments')) {

nightfall-client/src/services/commitment-storage.mjs

@@ -82,15 +82,18 @@ export async function countNullifiers(nullifiers) {
 // function to get count of transaction hashes of withdraw type. Used to decide if we should store sibling path of transaction hash to be used later for finalising or instant withdrawal
 export async function countWithdrawTransactionHashes(transactionHashes) {
   const connection = await mongo.connection(MONGO_URL);
-  const query = { transactionHash: { $in: transactionHashes }, nullifierTransactionType: '3' };
+  const query = {
+    transactionHash: { $in: transactionHashes },
+    nullifierTransactionType: '2',
+  };
   const db = connection.db(COMMITMENTS_DB);
   return db.collection(COMMITMENTS_COLLECTION).countDocuments(query);
 }
 
 // function to get if the transaction hash belongs to a withdraw transaction
 export async function isTransactionHashWithdraw(transactionHash) {
   const connection = await mongo.connection(MONGO_URL);
-  const query = { transactionHash, nullifierTransactionType: '3' };
+  const query = { transactionHash, nullifierTransactionType: '2' };
   const db = connection.db(COMMITMENTS_DB);
   return db.collection(COMMITMENTS_COLLECTION).countDocuments(query);
 }
@@ -511,7 +514,7 @@ export async function getWithdrawCommitments() {
   const db = connection.db(COMMITMENTS_DB);
   const query = {
     isNullified: true,
-    nullifierTransactionType: '3',
+    nullifierTransactionType: '2',
     isNullifiedOnChain: { $gte: 0 },
   };
   // Get associated nullifiers of commitments that have been spent on-chain and are used for withdrawals.
@@ -617,12 +620,30 @@ async function findUsableCommitments(compressedZkpPublicKey, ercAddress, tokenId
     await markPending(singleCommitment);
     return [singleCommitment];
   }
-  // If we get here it means that we have not been able to find a single commitment that matches the required value
-  if (onlyOne || commitments.length < 2) return null; // sometimes we require just one commitment
+  // If we only want one or there is only 1 commitment - then we should try a single transfer with change
+  if (onlyOne || commitments.length === 1) {
+    const valuesGreaterThanTarget = commitments.filter(c => c.preimage.value.bigInt > value.bigInt); // Do intermediary step since reduce has ugly exception
+    if (valuesGreaterThanTarget.length === 0) return null;
+    const singleCommitmentWithChange = valuesGreaterThanTarget.reduce((prev, curr) =>
+      prev.preimage.value.bigInt < curr.preimage.value.bigInt ? prev : curr,
+    );
+    return [singleCommitmentWithChange];
+  }
+  // If we get here it means that we have not been able to find a single commitment that satisfies our requirements (onlyOne)
+  if (commitments.length < 2) return null; // sometimes we require just one commitment
 
-  /* if not, maybe we can do a two-commitment transfer. The current strategy aims to prioritise smaller commitments while also
-     minimising the creation of low value commitments (dust)
+  /* if not, maybe we can do more flexible single or double commitment transfers. The current strategy aims to prioritise reducing the complexity of
+    the commitment set. I.e. Minimise the size of the commitment set by using smaller commitments while also minimising the creation of 
+    low value commitments (dust).
+
+    Transaction type in order of priority. (1) Double transfer without change, (2) Double Transfer with change, (3) Single Transfer with change.
+
+    Double Transfer Without Change:
+    1) Sort all commitments by value
+    2) Find candidate pairs of commitments that equal the transfer sum.
+    3) Select candidate that uses the smallest commitment as one of the input.
 
+    Double Transfer With Change:
     1) Sort all commitments by value
     2) Split commitments into two sets based of if their values are less than or greater than the target value. LT & GT respectively.
     3) If the sum of the two largest values in set LT is LESS than the target value:
@@ -636,13 +657,31 @@ async function findUsableCommitments(compressedZkpPublicKey, ercAddress, tokenId
       iii) If the sum of the commitments at the pointers is greater than the target value, we move pointer rhs to the left.
       iv) Otherwise, we move pointer lhs to the right.
       v) The selected commitments are the pair that minimise the change difference. The best case in this scenario is a change difference of -1.
+
+    Single Transfer With Change:
+    1) If this is the only commitment and it is greater than the transfer sum.
   */
 
   // sorting will help with making the search easier
   const sortedCommits = commitments.sort((a, b) =>
     Number(a.preimage.value.bigInt - b.preimage.value.bigInt),
   );
 
+  // Find two commitments that matches the transfer value exactly. Double Transfer With No Change.
+  let lhs = 0;
+  let rhs = sortedCommits.length - 1;
+  /** THIS WILL BE ENABLED LATED
+  while (lhs < rhs) {
+    const tempSum = sortedCommits[lhs].bigInt + sortedCommits[rhs].bigInt;
+    // The first valid solution will include the smallest usable commitment in the set.
+    if (tempSum === value.bigInt) break;
+    else if (tempSum > value.bigInt) rhs--;
+    else lhs++;
+  }
+  if (lhs < rhs) return [sortedCommits[lhs], sortedCommits[rhs]];
+   */
+
+  // Find two commitments are greater than the target. Double Transfer With Change
   // get all commitments less than the target value
   const commitsLessThanTargetValue = sortedCommits.filter(
     s => s.preimage.value.bigInt < value.bigInt,
@@ -660,8 +699,8 @@ async function findUsableCommitments(compressedZkpPublicKey, ercAddress, tokenId
   }
 
   // If we are here than we can use our commitments less than the target value to sum to greater than the target value
-  let lhs = 0;
-  let rhs = commitsLessThanTargetValue.length - 1;
+  lhs = 0;
+  rhs = commitsLessThanTargetValue.length - 1;
   let changeDiff = -Infinity;
   let commitmentsToUse = null;
   while (lhs < rhs) {
@@ -672,7 +711,7 @@ async function findUsableCommitments(compressedZkpPublicKey, ercAddress, tokenId
     // This value will always be negative,
     // this is equivalent to  tempSum - value.bigInt - commitsLessThanTargetValue[lhs].preimage.value.bigInt
     const tempChangeDiff = commitsLessThanTargetValue[rhs].preimage.value.bigInt - value.bigInt;
-    if (tempSum > value.bigInt) {
+    if (tempSum >= value.bigInt) {
       if (tempChangeDiff > changeDiff) {
         // We have a set of commitments that has a lower negative change in our outputs.
         changeDiff = tempChangeDiff;
@@ -685,11 +724,10 @@ async function findUsableCommitments(compressedZkpPublicKey, ercAddress, tokenId
     logger.info(
       `Found commitments suitable for two-token transfer: ${JSON.stringify(commitmentsToUse)}`,
     );
-  } else {
-    return null;
+    await Promise.all(commitmentsToUse.map(commitment => markPending(commitment)));
+    return commitmentsToUse;
   }
-  await Promise.all(commitmentsToUse.map(commitment => markPending(commitment)));
-  return commitmentsToUse;
+  return null;
 }
 
 // mutex for the above function to ensure it only runs with a concurrency of one

nightfall-client/src/services/deposit.mjs

@@ -16,6 +16,7 @@ import constants from 'common-files/constants/index.mjs';
 import { Commitment, Transaction } from '../classes/index.mjs';
 import { storeCommitment } from './commitment-storage.mjs';
 import { ZkpKeys } from './keys.mjs';
+import { computeWitness } from '../utils/compute-witness.mjs';
 
 const { ZOKRATES_WORKER_HOST, PROVING_SCHEME, BACKEND, PROTOCOL, USE_STUBS, BN128_GROUP_ORDER } =
   config;
@@ -33,14 +34,22 @@ async function deposit(items) {
   const commitment = new Commitment({ ercAddress, tokenId, value, zkpPublicKey, salt });
   logger.debug(`Hash of new commitment is ${commitment.hash.hex()}`);
   // now we can compute a Witness so that we can generate the proof
-  const witness = [
-    ercAddress.field(BN128_GROUP_ORDER),
-    tokenId.limbs(32, 8),
-    value.field(BN128_GROUP_ORDER),
-    ...zkpPublicKey.all.field(BN128_GROUP_ORDER),
-    salt.field(BN128_GROUP_ORDER),
-    commitment.hash.field(BN128_GROUP_ORDER),
-  ].flat(Infinity);
+  const publicData = Transaction.buildSolidityStruct(
+    new Transaction({
+      fee,
+      transactionType: 0,
+      tokenType: items.tokenType,
+      tokenId,
+      value,
+      ercAddress,
+      commitments: [commitment],
+    }),
+  );
+
+  const privateData = { salt, recipientPublicKeys: [zkpPublicKey] };
+  const roots = [];
+
+  const witness = computeWitness(publicData, roots, privateData);
   logger.debug(`witness input is ${witness.join(' ')}`);
   // call a zokrates worker to generate the proof
   let folderpath = 'deposit';