strong execution optimisation

tests/Dockerfile

@@ -1,59 +1,70 @@
 FROM ubuntu:jammy
+# See ./Dockerfile.notes for more details on different options
 
-## Base tools for compilation and download
-## Google chrome for puppeteer, optimised 
-## Node & Ethereum
+## Base tools installations
 RUN apt-get update && \
 	apt-get install -y software-properties-common apt-utils wget tree sudo && \
+	# ca-certificates is required for wget downloads, apt utils for ppa additions. Optimisations here don't end well
+	## Google direct
 	wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
 	apt-get install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb libxss1  && \
 	rm google-chrome-stable_current_amd64.deb && \
-	wget -qO- https://deb.nodesource.com/setup_18.x | sudo -E bash - && \
-	add-apt-repository ppa:ethereum/ethereum && \
+	## Node ppa
+	wget -qO- https://deb.nodesource.com/setup_18.x | bash - && \
 	apt-get update && \
-	apt-get install -y --no-install-recommends solc nodejs && \
-	apt-get clean
-	# gnupg ca-certificates procps are implicit dependencies required for chrome
-
+	apt-get install -y --no-install-recommends nodejs && \
+	## solc direct
+	wget -q https://github.com/ethereum/solidity/releases/download/v0.8.17/solc-static-linux && \
+	mv solc-static-linux /usr/bin/solc && \
+	chmod +x /usr/bin/solc && \
+	## Cleaning
+	apt-get clean && \ 
+	rm -rf /var/lib/apt/lists/*
+
 ## Bitcoin node
 RUN bitcoinCoreVersion="24.0.1"; \
-		wget -q "https://bitcoincore.org/bin/bitcoin-core-$bitcoinCoreVersion/bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"; \
-		tar xzf "bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"; \
-		mv bitcoin-$bitcoinCoreVersion/bin/* /usr/local/bin ; \
-		rm -rf bitcoin-$bitcoinCoreVersion/ ; \
-		rm "bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"
+	wget -q "https://bitcoincore.org/bin/bitcoin-core-$bitcoinCoreVersion/bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"; \
+	tar xzf "bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"; \
+	mv bitcoin-$bitcoinCoreVersion/bin/* /usr/local/bin ; \
+	rm -rf bitcoin-$bitcoinCoreVersion/ ; \
+	rm "bitcoin-$bitcoinCoreVersion-x86_64-linux-gnu.tar.gz"
 	# NB: default ports on regtest are 18443 RPC and 18444 P2P
 	# Details at https://github.com/cryptotuxorg/cryptotux/blob/master/install/bitcoin.sh
 
-# ## Preparing user & rights for runtime 
-RUN groupadd -r xa -g 1000 && useradd -rm -u 1000 -g xa -G audio,video xa
+## Preparing user & rights for runtime 
+RUN groupadd -r xa -g 1000 && useradd -rm -u 1000 -g xa -G audio,video xa 
 RUN mkdir -p /app; chown xa:xa /app
 	# The partition is read only at runtime and entrypoint is run as user 1000
-RUN npm install -g npm 
+RUN npm install -g bun
+	# After tests, we use bun for execution and npm for installation
+	# bun faster than npm for execution, does not work for install
+	# pnpm runs faster and results in smaller footprint for installation but requires tweaking
 
 USER xa 
 
 ## Preparing a Bitcoin node (as a user to facilitate runtime)
 WORKDIR /home/xa
 RUN mkdir -p /home/xa/.bitcoin; echo 'txindex=1\nregtest=1\nserver=1\nrest=1\nrpcallowip=127.0.0.1\nrpcuser=leeloo\nrpcpassword=multipass\nfallbackfee=0.00000003'> /home/xa/.bitcoin/bitcoin.conf
-RUN bitcoind -daemon -daemonwait; bitcoin-cli createwallet "testwallet"; address=`bitcoin-cli getnewaddress`; bitcoin-cli generatetoaddress 101 $address; bitcoin-cli getblockchaininfo; bitcoin-cli stop; sleep 2
-RUN bitcoind -daemon -daemonwait; bitcoin-cli loadwallet "testwallet"; address=`bitcoin-cli getnewaddress`; bitcoin-cli sendtoaddress "bcrt1qznrqryhtzr66tp8uzrxsuh58mn2vpfmjxpnxgz" 12 "Another tx" "anonymous";  bitcoin-cli generatetoaddress 1 $address; bitcoin-cli stop; sleep 2
+RUN bitcoind -daemon -daemonwait; bitcoin-cli createwallet "testwallet"; address=`bitcoin-cli getnewaddress`; bitcoin-cli generatetoaddress 101 $address; bitcoin-cli getblockchaininfo; bitcoin-cli stop; sleep 2; \
+		bitcoind -daemon -daemonwait; bitcoin-cli loadwallet "testwallet"; address=`bitcoin-cli getnewaddress`; bitcoin-cli sendtoaddress "bcrt1qznrqryhtzr66tp8uzrxsuh58mn2vpfmjxpnxgz" 12 "Another tx" "anonymous";  bitcoin-cli generatetoaddress 1 $address; bitcoin-cli stop; sleep 2
 
-## Installing npm modules
+## Copy the code
 WORKDIR /app
-COPY package.json .
-	# Duplicated but we need at only this file at this stage. Avoids rebuilds when only code changes
+# COPY --chown=xa:xa package.json .
+	# Duplicated copy when we want to avoid npm install on every change
+COPY --chown=xa:xa . .
+	# xa:xa ownership is not necessary locally but might prevent some issues in production
+
+## Installing npm modules
 RUN npm install --loglevel verbose --omit=dev	
-	# pnpm install --prod == npm --omit=dev
+	# pnpm install --prod == npm install --omit=dev
 	# pnpm has git as a dependency. Upon testing, gains are not significant and can have more unexpected side effects
 
-## Copy the code
-WORKDIR /app
-COPY . .
-
 ## Entrypoint
-USER root
-# RUN chown -R xa:xa /app
+# USER root 
+# RUN chown -R xa:xa /app 
+	# Both commands should not be required now that we have copied files with user ownership
 RUN chmod +x /app/entrypoint.sh
 USER 1000
+	# Necessary for production environment 
 ENTRYPOINT ["/app/entrypoint.sh"]

tests/entrypoint.sh

@@ -1,17 +1,20 @@
 #!/bin/bash
 
-# This script is expected to run in the docker environment provided in the Dockerfile. 
+# This script is expected to run in the docker environment provided in the Dockerfile.
 # It can be run in this environment with `DEBUG=false EXERCISE=retrieveBlockDate ./entrypoint.sh`
 
 ## Constants
 CONTENT_FOLDER=/app
 
-## Debugging info 
+shopt -s expand_aliases
+alias xx='bunx --bun' # alternatives : 'npx' 'bunx --bun'
+
+## Debugging info
 if [ $DEBUG ]; then
   echo ">>> Entrypoint script in debug mode <<<"
   /app/infos.sh
   # set -x # Can be used to identify commands launched
-else 
+else
   set -e
 fi
 
@@ -20,72 +23,73 @@ cp /app/package.json /jail
 ln -s /app/node_modules/ /jail/node_modules
 
 ## Distinct test environments are triggered depending in the test file folder
-if test -f "/app/btc/${EXERCISE}".test.js; 
-then
+if test -f "/app/btc/${EXERCISE}".test.js; then
   # Bitcoin related tests (Quest 1)
-  [ $DEBUG ] && ( echo ">> Bitcoin test <<" )
+  [ $DEBUG ] && (echo ">> Bitcoin test <<")
   cp -r /home/xa/.bitcoin .
-  bitcoind -daemon -daemonwait -datadir=/jail/.bitcoin 
+  bitcoind -daemon -daemonwait -datadir=/jail/.bitcoin
   if [ $DEBUG ]; then
     tree -C ~/.bitcoin
     bitcoin-cli loadwallet "testwallet"
     bitcoin-cli getbalance
     bitcoin-cli listwallets
     bitcoin-cli getblockchaininfo
   fi
-  npx mocha "/app/btc/${EXERCISE}.test.js"
+  xx mocha "/app/btc/${EXERCISE}.test.js"
 
 elif test -f "/app/js/$EXERCISE.test.js"; then
   # Pure js tests
-  [ $DEBUG ] && ( echo ">> Pure JS test <<" )
-  npx mocha "/app/js/${EXERCISE}.test.js"
+  [ $DEBUG ] && (echo ">> Pure JS test <<")
+  xx mocha "/app/js/${EXERCISE}.test.js"
 
 elif test -f "/app/mjs/${EXERCISE}.test.js"; then
   # Tests relying on the test.mjs (Quest 2)
-  [ $DEBUG ] && ( echo ">> JS test (test.mjs)<<" )
+  [ $DEBUG ] && (echo ">> JS test (test.mjs)<<")
   node /app/test.mjs ${EXERCISE}
 
 elif test -f "/app/sol/$EXERCISE.test.js"; then
   # Pure solidity tests (Quest 3)
-  [ $DEBUG ] && ( echo ">> Solidity test (HH) <<" )
+  [ $DEBUG ] && (echo ">> Solidity test (HH) <<")
   mkdir -p /jail/test
   cp /app/sol/${EXERCISE}.test.js /jail/test/
   cp /app/hardhat.config.js /jail/
   cd /jail
-  
+
   # TOCHECK Solution to ensure that only one contract is compiled at the time
-  # If /jail/student had only the submitted zip, they could be run directly with ro benefit, with propoer hardhat config
+  # If /jail/student had only the submitted zip, they could be run directly with ro benefit, with proper hardhat path config
+  # We could also adapt subjects to ensure that there is no duplicate contract name either loading prior exercise
+  # (but we need to be sure that they are available), or by renaming duplicated contract within a new file.
   mkdir -p /jail/contracts
   cp /jail/student/${EXERCISE}.sol /jail/contracts/
 
   if [ $DEBUG ]; then
     solc --version
-    npx hardhat --version
+    xx hardhat --version
     df -h
     tree /jail
-    npx hardhat --verbose test "/jail/test/${EXERCISE}.test.js"
+    xx hardhat --verbose test "/jail/test/${EXERCISE}.test.js"
     tree /jail
-  else 
-    npx hardhat test "/jail/test/${EXERCISE}.test.js"
-  fi 
+  else
+    xx hardhat test "/jail/test/${EXERCISE}.test.js"
+  fi
 
 elif test -f "/app/web3/${EXERCISE}".test.js; then
   # Interface related tests
-  [ $DEBUG ] && ( echo ">> Web3 interface test <<" )
+  [ $DEBUG ] && (echo ">> Web3 interface test <<")
   mkdir -p /jail/test
   cp /app/hardhat.config.js /jail/
   # When using the default chromium, this brings the configuration to the jail to redirect cache
   # cp /app/puppeteer.config.cjs /jail/
 
   cp "/app/web3/${EXERCISE}.test.js" /jail/test
   # Launch a local node
-  npx hardhat node >/dev/null& 
+  xx hardhat node >/dev/null &
   sleep 0.2 # Short wait for the node to be ready but students should check.
   # Launch the tests
-  npx hardhat test "/jail/test/${EXERCISE}.test.js"
-else 
+  xx hardhat test "/jail/test/${EXERCISE}.test.js"
+else
   # Failure
   echo "Entrypoint> No suitable test found for $EXERCISE"
-  tree -I node_modules /app 
+  tree -I node_modules /app
   exit 1
 fi

tests/hardhat.config.js

@@ -1,6 +1,12 @@
 // require("@nomiclabs/hardhat-waffle")
+// Relies on package.json:
+// "@nomiclabs/hardhat-waffle": "^2.0.3",
 require("@nomiclabs/hardhat-ethers")
+// Relies on package.json:
+// "@nomiclabs/hardhat-ethers": "^2.2.2",
 require("@nomicfoundation/hardhat-chai-matchers")
+// Relies on package.json(incompatible with waffle)
+// "@nomicfoundation/hardhat-chai-matchers": "^1.0.5",
 
 const { TASK_COMPILE_SOLIDITY_GET_SOLC_BUILD } = require("hardhat/builtin-tasks/task-names")
 const path = require("path")
@@ -21,22 +27,22 @@ subtask(TASK_COMPILE_SOLIDITY_GET_SOLC_BUILD, async (args, hre, runSuper) => {
       longVersion: "solc-linux-v" + expectedSolcVersion
     }
   }
-  console.log("Please use version", expectedSolcVersion)
+  console.log("Please use version", expectedSolcVersinecessaryon)
   // we just use the default subtask if the version is not this
   return runSuper()
 })
 
 
-// TOCHECK Contract compiling from the contract folder is a hack due to unexpected duplicates (see entrypoint)
 /**
  * @type import('hardhat/config').HardhatUserConfig
- */
+*/
 module.exports = {
   solidity: expectedSolcVersion,
 
   paths: {
     root: "/jail",
     sources: "./contracts",
+    // Compiling from the ./contract folder instead of ./student is to prevent duplicates (see entrypoint)
     tests: "./test",
     cache: "./cache",
     artifacts: "./artifacts"

tests/lib/helpers.js

@@ -10,15 +10,44 @@ if (process.env.DEBUG) {
   DEBUG = true
 }
 
-function displayBrowserLogs (page) {
+function displayBrowserLogs(page) {
   page
     .on('console', message =>
       console.log(`🖥️ ${message.type().substr(0, 3).toUpperCase()} ${message.text()}`))
-    .on('pageerror', ({ message }) => 
-      console.log("🖥️ ",message))
+    .on('pageerror', ({ message }) =>
+      console.log("🖥️ ", message))
     .on('response', response =>
       console.log(`🌍️${response.status()} ${response.url()}`))
     .on('requestfailed', request =>
       console.log(`🌍️${request.failure().errorText} ${request.url()}`))
 }
-module.exports = { sleep, DEBUG, displayBrowserLogs }
+
+const pp_options = {
+  executablePath: '/usr/bin/google-chrome-stable',
+  args: [
+    // This is needed to chrome with puppeteer
+    '--no-sandbox',
+    // Some of the following might be applied by default by puppeteer or ineffective
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage',
+    '--disable-accelerated-2d-canvas',
+    '--no-first-run',
+    '--no-zygote',
+    '--single-process',
+    '--disable-gpu',
+    '--mute-audio',
+    '--disable-web-security',
+    '--allow-file-access-from-files',
+    '--disable-client-side-phishing-detection',
+    '--disable-default-apps',
+    '--disable-features=Translate',
+    '--no-default-browser-check',
+    '--ignore-certificate-errors'
+    //--disable-extensions // Might be necessary for metamask tests
+  ]
+}
+// Interesting idea:
+// const opts = process.env.D ? { headless: false, slowMo: 250 } : {}; 
+
+
+module.exports = { sleep, DEBUG, displayBrowserLogs, pp_options }

tests/package.json

@@ -25,4 +25,4 @@
     "eslint": "^8.32.0",
     "typescript": "^4.9.4"
   }
-}
+}

tests/puppeteer.config.cjs

@@ -1,9 +1,11 @@
-const {join} = require('path');
+const { join } = require('path')
 
 /**
  * @type {import("puppeteer").Configuration}
  */
 module.exports = {
-  // Changes the cache location for Puppeteer. Current folder is :__dirname
+	// Changes the cache location for Puppeteer. Current folder is :__dirname
 	cacheDirectory: join('/home/xa/', '.cache', 'puppeteer'),
-};
+	// executablePath: '/usr/bin/chromium-browser',	
+	// Further optimisation ideas: https://stackoverflow.com/questions/62852481/how-to-speed-up-puppeteer
+}

tests/sol/register-with-events.test.js

@@ -6,37 +6,37 @@ const ANOTHERHASH = "0x27741ec14d337e77ec4fe2d33be71f2e8aa9d858b5982ab9faf5aebfc
 
 const REFDATE = 1609459200
 
-describe("Register smart contrat", function() {
+describe("Register smart contract", function () {
   let register = {}
-  beforeEach( async function () {
+  beforeEach(async function () {
     const RegisterFactory = await ethers.getContractFactory("RegisterWithEvents")
-    register = await RegisterFactory.deploy();const opts = {} 
+    register = await RegisterFactory.deploy()
 
     await register.deployed()
   })
-  it("Should return date of a document added", async function() {
+  it("Should return date of a document added", async function () {
     await register.addDocument(DOCHASH)
 
     expect(await register.getDate(DOCHASH)).to.be.at.least(REFDATE)
   })
-  it("Should return 0 for a document not added", async function() {
+  it("Should return 0 for a document not added", async function () {
     expect(await register.getDate(WRONGDOCHASH)).to.be.equal(0)
   })
-  it("Should have the correct event hash", async function() {
+  it("Should have the correct event hash", async function () {
     await register.addDocument(DOCHASH)
     let eventsFilter = await register.filters.DocumentAdded(null, null)
     const listEvents = await register.queryFilter(eventsFilter, 0)
 
     expect(listEvents[0].args[0]).to.equal(DOCHASH)
   })
-  it("Should have the correct event date", async function() {
+  it("Should have the correct event date", async function () {
     await register.addDocument(DOCHASH)
     let eventsFilter = await register.filters.DocumentAdded(null, null)
     const listEvents = await register.queryFilter(eventsFilter, 0)
 
     expect(listEvents[0].args[1]).to.be.at.least(REFDATE)
   })
-  it("Should produce the appropriate number of events", async function() {
+  it("Should produce the appropriate number of events", async function () {
     await register.addDocument(DOCHASH)
     await register.addDocument(WRONGDOCHASH)
     await register.addDocument(ANOTHERHASH)