From 9bf1fa169b682a1893d61f0a9db6547761894fef Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 09:16:45 +0100 Subject: [PATCH 01/13] Fix: Developers needed to manually create a PostgreSQL and Redis databases Solution: Provide a `shell.nix` file so developers can just run `nix-shell` to start temporary PostgreSQL and Redis servers. The exit signal of the shell is `trap`-ed in order to automatically stop the services on exit. --- shell.nix | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 shell.nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..ce4d1cc81 --- /dev/null +++ b/shell.nix @@ -0,0 +1,39 @@ +{ pkgs ? import {} }: + +pkgs.mkShell { + buildInputs = [ pkgs.postgresql pkgs.redis ]; + + shellHook = '' + echo "Setting up PostgreSQL environment..." + export PGDATA=$(mktemp -d) + PG_SOCKET_DIR=$(mktemp -d) + echo "Initializing database..." + initdb $PGDATA + + echo "Starting PostgreSQL with custom socket directory..." + pg_ctl -D $PGDATA -o "-k $PG_SOCKET_DIR" -l logfile start + + # Wait a bit for the server to start + sleep 1 + + # Create the 'aleph' role and a database + createuser -h $PG_SOCKET_DIR aleph + createdb -h $PG_SOCKET_DIR aleph -O aleph + + # Create a temporary directory for Redis + export REDIS_DATA_DIR=$(mktemp -d) + redis-server --daemonize yes --dir $REDIS_DATA_DIR --bind 127.0.0.1 --port 6379 + echo "Redis server started. Data directory is $REDIS_DATA_DIR" + + # Trap the EXIT signal to ensure PostgreSQL is stopped when exiting the shell + trap 'echo "Stopping PostgreSQL..."; pg_ctl -D "$PGDATA" stop; echo "Stopping Redis..."; redis-cli -p 6379 shutdown' EXIT + + echo + echo "PostgreSQL started. Data directory is $PGDATA, Socket directory is $PG_SOCKET_DIR" + echo "Redis started. Data directory is $REDIS_DATA_DIR" + echo "Use 'psql -h $PG_SOCKET_DIR' to connect to the database." + echo "Use 'redis-cli -p 6379' to connect to the Redis server." + echo "To stop PostgreSQL: 'pg_ctl -D $PGDATA stop'" + echo "To manually stop Redis: 'redis-cli -p 6379 shutdown'" + ''; +} From 475e0d348025945b8991514b70cc6a9eb87c94f2 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 17:41:26 +0100 Subject: [PATCH 02/13] Fix: nix-shell did not provide Python dependencies --- shell.nix | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/shell.nix b/shell.nix index ce4d1cc81..d3917b106 100644 --- a/shell.nix +++ b/shell.nix @@ -1,7 +1,19 @@ { pkgs ? import {} }: pkgs.mkShell { - buildInputs = [ pkgs.postgresql pkgs.redis ]; + buildInputs = [ + pkgs.postgresql + pkgs.redis + pkgs.kubo + + pkgs.python311 + pkgs.python311Packages.virtualenv + pkgs.python311Packages.pip + pkgs.python311Packages.setuptools + + pkgs.python311Packages.secp256k1 + pkgs.python311Packages.fastecdsa + ]; shellHook = '' echo "Setting up PostgreSQL environment..." @@ -25,8 +37,11 @@ pkgs.mkShell { redis-server --daemonize yes --dir $REDIS_DATA_DIR --bind 127.0.0.1 --port 6379 echo "Redis server started. Data directory is $REDIS_DATA_DIR" - # Trap the EXIT signal to ensure PostgreSQL is stopped when exiting the shell - trap 'echo "Stopping PostgreSQL..."; pg_ctl -D "$PGDATA" stop; echo "Stopping Redis..."; redis-cli -p 6379 shutdown' EXIT + echo "Starting IPFS Kubo..." + export IPFS_PATH=$(mktemp -d) + ipfs init + ipfs daemon & + echo "IPFS Kubo started. Data directory is $IPFS_PATH" echo echo "PostgreSQL started. Data directory is $PGDATA, Socket directory is $PG_SOCKET_DIR" @@ -35,5 +50,19 @@ pkgs.mkShell { echo "Use 'redis-cli -p 6379' to connect to the Redis server." echo "To stop PostgreSQL: 'pg_ctl -D $PGDATA stop'" echo "To manually stop Redis: 'redis-cli -p 6379 shutdown'" + + # Trap the EXIT signal to stop services when exiting the shell + trap 'echo "Stopping PostgreSQL..."; pg_ctl -D "$PGDATA" stop; echo "Stopping Redis..."; redis-cli -p 6379 shutdown; echo "Stopping IPFS Kubo..."; ipfs shutdown; deactivate' EXIT + + # Create a virtual environment in the current directory if it doesn't exist + if [ ! -d "venv" ]; then + python3 -m virtualenv venv + fi + + # Install the required Python packages + ./venv/bin/pip install -e .\[testing\] + + # Activate the virtual environment + source venv/bin/activate ''; } From 17509ee115eb873e471263c9278ad5c423499f40 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 17:41:36 +0100 Subject: [PATCH 03/13] Doc: Document usage of nix-shell --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 20cd46236..7a66159fa 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,18 @@ pip install -e .[testing,docs] You're ready to go! +### Developer setup using Nix + +We started to add Nix as an easy way to setup a development environment. +This is still a work in progress and not all dependencies are covered yet. + +To use it, you need to have Nix installed on your system. Then you can run: + +```bash +nix-shell +``` +This will provide you with a shell with PostgreSQL, Redis, and IPFS running. + ## Software used The Aleph CCN is written in Python and requires Python v3.8+. It will not work with older versions of Python. From 9317547a02f52adb0375124a4beacdd80ffc33fc Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 17:41:48 +0100 Subject: [PATCH 04/13] Fix: There was no CI on nix-shell --- .github/workflows/nix-ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/nix-ci.yml diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml new file mode 100644 index 000000000..039dba160 --- /dev/null +++ b/.github/workflows/nix-ci.yml @@ -0,0 +1,18 @@ + on: + push: + branches: + - '*' + + jobs: + nix-shell: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + + - name: Set up Nix + uses: cachix/install-nix-action@v13 + with: + nix_path: nixpkgs=channel:nixos-23.11 + + - name: Run tests + run: nix-shell --run "pytest" From fe5bb2b8197adfb858b98810d084328dc1cb515c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 17:54:34 +0100 Subject: [PATCH 05/13] fixup! git depth --- .github/workflows/nix-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 039dba160..57572987b 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -8,6 +8,9 @@ runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + with: + # Fetch the whole history for all tags and branches (required for aleph.__version__) + fetch-depth: 0 - name: Set up Nix uses: cachix/install-nix-action@v13 From b00683df78ef70bf1c4332a34fbb1b605a783cea Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 17:57:26 +0100 Subject: [PATCH 06/13] fixup! matrix run on macos too --- .github/workflows/nix-ci.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 57572987b..ba9e5d527 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -5,17 +5,21 @@ jobs: nix-shell: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - with: - # Fetch the whole history for all tags and branches (required for aleph.__version__) - fetch-depth: 0 + strategy: + matrix: + os: [ macos-latest, ubuntu-latest ] + runs-on: ${{ matrix.os }} - - name: Set up Nix - uses: cachix/install-nix-action@v13 - with: - nix_path: nixpkgs=channel:nixos-23.11 + steps: + - uses: actions/checkout@v2 + with: + # Fetch the whole history for all tags and branches (required for aleph.__version__) + fetch-depth: 0 - - name: Run tests - run: nix-shell --run "pytest" + - name: Set up Nix + uses: cachix/install-nix-action@v13 + with: + nix_path: nixpkgs=channel:nixos-23.11 + + - name: Run tests + run: nix-shell --run "pytest" From eb9a0282cf20f64d58ac280566223f37ced666ec Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 18:01:23 +0100 Subject: [PATCH 07/13] fixup! bump action versions --- .github/workflows/nix-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index ba9e5d527..9d58a4133 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -11,13 +11,13 @@ runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: # Fetch the whole history for all tags and branches (required for aleph.__version__) fetch-depth: 0 - name: Set up Nix - uses: cachix/install-nix-action@v13 + uses: cachix/install-nix-action@v25 with: nix_path: nixpkgs=channel:nixos-23.11 From 5edc7d8a632628057c0e0178d90cb0ccff319527 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 18:07:14 +0100 Subject: [PATCH 08/13] fixup! CI fixes --- .github/workflows/nix-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 9d58a4133..4e1852d56 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -8,6 +8,7 @@ strategy: matrix: os: [ macos-latest, ubuntu-latest ] + channel: [ nixos-23.11 nixpkgs-23.11-darwin ] runs-on: ${{ matrix.os }} steps: @@ -16,10 +17,13 @@ # Fetch the whole history for all tags and branches (required for aleph.__version__) fetch-depth: 0 + - name: Setup empty config file + run: touch config.yml + - name: Set up Nix uses: cachix/install-nix-action@v25 with: - nix_path: nixpkgs=channel:nixos-23.11 + nix_path: nixpkgs=channel:${{ matrix.channel }} - name: Run tests run: nix-shell --run "pytest" From 760b233bb2e4888869366b5a7bb73ef5b52f364c Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 18:09:32 +0100 Subject: [PATCH 09/13] fixup! CI fixes --- .github/workflows/nix-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 4e1852d56..d5be4567d 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -8,7 +8,7 @@ strategy: matrix: os: [ macos-latest, ubuntu-latest ] - channel: [ nixos-23.11 nixpkgs-23.11-darwin ] + channel: [ nixos-23.11, nixpkgs-23.11-darwin ] runs-on: ${{ matrix.os }} steps: From f196064e209bb4ec9a0b959b67f2618f8908ae86 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 18:16:21 +0100 Subject: [PATCH 10/13] fixup! CI fixes --- .github/workflows/nix-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index d5be4567d..9a21a2322 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -8,7 +8,6 @@ strategy: matrix: os: [ macos-latest, ubuntu-latest ] - channel: [ nixos-23.11, nixpkgs-23.11-darwin ] runs-on: ${{ matrix.os }} steps: @@ -23,7 +22,8 @@ - name: Set up Nix uses: cachix/install-nix-action@v25 with: - nix_path: nixpkgs=channel:${{ matrix.channel }} + # Use channel nixos-23.11 for Linux and nixpkgs-23.11-darwin for macOS + nix_path: nixpkgs=channel:${{ matrix.os == 'macos-latest' && 'nixpkgs-23.11-darwin' || 'nixos-23.11' }} - name: Run tests run: nix-shell --run "pytest" From 7528d7fe781bff6d93690878d6fd94009ba735f5 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 18:17:24 +0100 Subject: [PATCH 11/13] fixup! CI fixes --- .github/workflows/nix-ci.yml | 50 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 9a21a2322..f4b27a758 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -1,29 +1,31 @@ - on: - push: - branches: - - '*' +name: Test nix-shell - jobs: - nix-shell: - strategy: - matrix: - os: [ macos-latest, ubuntu-latest ] - runs-on: ${{ matrix.os }} +on: + push: + branches: + - '*' - steps: - - uses: actions/checkout@v4 - with: - # Fetch the whole history for all tags and branches (required for aleph.__version__) - fetch-depth: 0 +jobs: + nix-shell: + strategy: + matrix: + os: [ macos-latest, ubuntu-latest ] + runs-on: ${{ matrix.os }} - - name: Setup empty config file - run: touch config.yml + steps: + - uses: actions/checkout@v4 + with: + # Fetch the whole history for all tags and branches (required for aleph.__version__) + fetch-depth: 0 - - name: Set up Nix - uses: cachix/install-nix-action@v25 - with: - # Use channel nixos-23.11 for Linux and nixpkgs-23.11-darwin for macOS - nix_path: nixpkgs=channel:${{ matrix.os == 'macos-latest' && 'nixpkgs-23.11-darwin' || 'nixos-23.11' }} + - name: Setup empty config file + run: touch config.yml - - name: Run tests - run: nix-shell --run "pytest" + - name: Set up Nix + uses: cachix/install-nix-action@v25 + with: + # Use channel nixos-23.11 for Linux and nixpkgs-23.11-darwin for macOS + nix_path: nixpkgs=channel:${{ matrix.os == 'macos-latest' && 'nixpkgs-23.11-darwin' || 'nixos-23.11' }} + + - name: Run tests + run: nix-shell --run "pytest" From f9c10c8966a7dd2ee743529bc3e66fe2eaae7b7f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 14 Feb 2024 21:26:24 +0100 Subject: [PATCH 12/13] WIP: Try fixing greenlet on macos --- shell.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.nix b/shell.nix index d3917b106..ea01f7b35 100644 --- a/shell.nix +++ b/shell.nix @@ -13,6 +13,7 @@ pkgs.mkShell { pkgs.python311Packages.secp256k1 pkgs.python311Packages.fastecdsa + pkgs.python311Packages.greenlet ]; shellHook = '' From 79630c28f81207e826d591ca7eefd4b2d238eda0 Mon Sep 17 00:00:00 2001 From: Mike Hukiewitz <70762838+MHHukiewitz@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:16:20 +0100 Subject: [PATCH 13/13] Add nix installation link Co-authored-by: Hugo Herter --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a66159fa..4ecb03ce7 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ You're ready to go! We started to add Nix as an easy way to setup a development environment. This is still a work in progress and not all dependencies are covered yet. -To use it, you need to have Nix installed on your system. Then you can run: +To use it, you need to [have Nix installed on your system](https://nixos.org/download.html). Then you can run: ```bash nix-shell