refactor: remove use of cartesi/toolchain

Dockerfile

@@ -1,55 +1,86 @@
-FROM --platform=$TARGETPLATFORM cartesi/toolchain:0.17.0-rv64ima-lp64 as linux-env
-ARG GIT_COMMIT=""
-ARG DEBUG=no
-ARG COVERAGE=no
-ARG SANITIZE=no
+FROM --platform=$TARGETPLATFORM debian:bookworm-20241016 AS toolchain
 
 RUN apt-get update && \
-    DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
-    build-essential vim wget git lcov \
-    libboost1.81-dev libssl-dev libslirp-dev \
-    ca-certificates pkg-config lua5.4 liblua5.4-dev \
-    luarocks xxd procps && \
-    rm -rf /var/lib/apt/lists/* /var/cache/apt/*
-
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        build-essential vim wget git lcov \
+        libboost1.81-dev libssl-dev libslirp-dev \
+        ca-certificates pkg-config lua5.4 liblua5.4-dev \
+        luarocks xxd procps \
+        g++-12-riscv64-linux-gnu=12.2.0-13cross1 \
+        gcc-riscv64-unknown-elf=12.2.0-14+11+b1 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install clang 18
 RUN apt-get update && \
-    DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y wget software-properties-common gnupg && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        wget software-properties-common gnupg && \
     wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \
     add-apt-repository -y 'deb http://apt.llvm.org/bookworm/  llvm-toolchain-bookworm-18 main' && \
     add-apt-repository -y 'deb http://apt.llvm.org/bookworm/  llvm-toolchain-bookworm-18 main' && \
     apt-get update && \
-    apt-get install --no-install-recommends -y clang-tidy-18 clang-format-18 && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        clang-tidy-18 clang-format-18 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-18 120 && \
     update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-18 120 && \
-    rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+    rm -rf /var/lib/apt/lists/*
 
+# Install lua packages
 RUN luarocks install --lua-version=5.4 luasocket && \
     luarocks install --lua-version=5.4 luasec && \
     luarocks install --lua-version=5.4 luaposix && \
-    luarocks install --lua-version=5.4 luacheck && \
-    cargo install stylua@0.20.0 --features lua54
-
-# Environment has the riscv64-cartesi-linux-gnu-* toolchain
+    luarocks install --lua-version=5.4 luacheck
+
+# Install stylua
+RUN cd /tmp && \
+    wget https://github.com/JohnnyMorganz/StyLua/releases/download/v0.20.0/stylua-linux-`uname -m`.zip && \
+    case $(uname -m) in \
+      x86_64)  echo "28eddb9257bf85b20b1c337e536b7a3d16ba308863f067d447c1f4d24c6dec64  stylua-linux-x86_64.zip"  | sha256sum --check ;; \
+      aarch64) echo "376b675766bc0b9261b2b82c8d0f624c7e5f78e83bd8490330e0bf3d8f770ad7  stylua-linux-aarch64.zip" | sha256sum --check ;; \
+    esac && \
+    unzip stylua-linux-*.zip && \
+    mv stylua /usr/local/bin/ && \
+    rm -f stylua-linux-*.zip
+
+# Environment has the riscv64 toolchains
 ENV DEV_ENV_HAS_TOOLCHAIN=yes
 
-WORKDIR /usr/src/emulator
+# Install su-exec
+RUN cd /tmp && \
+    git clone --branch v0.2 --depth 1 https://github.com/ncopa/su-exec.git && \
+    cd su-exec && \
+    if [ `git rev-parse --verify HEAD` != 'f85e5bde1afef399021fbc2a99c837cf851ceafa' ]; then exit 1; fi && \
+    make && \
+    cp su-exec /usr/local/bin/ && \
+    rm -rf /tmp/su-exec
 
-FROM --platform=$TARGETPLATFORM linux-env as dep-builder
+# Install workaround to run as current user
+COPY tools/docker-entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
 
-COPY Makefile .
-COPY third-party third-party
+# Install necessary headers to make GNU libc work with lp64 ABI
+COPY tools/gnu/stubs-lp64.h /usr/riscv64-linux-gnu/include/gnu/stubs-lp64.h
 
-FROM --platform=$TARGETPLATFORM dep-builder as builder
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+WORKDIR /usr/src/emulator
+CMD ["/bin/bash", "-l"]
+
+####################################################################################################
+FROM --platform=$TARGETPLATFORM toolchain AS builder
+ARG GIT_COMMIT=""
+ARG DEBUG=no
+ARG COVERAGE=no
+ARG SANITIZE=no
 
 COPY . .
 RUN make -j$(nproc) git_commit=$GIT_COMMIT debug=$DEBUG coverage=$COVERAGE sanitize=$SANITIZE
 
+####################################################################################################
 FROM --platform=$TARGETPLATFORM builder as debian-packager
-ARG MACHINE_EMULATOR_VERSION=0.0.0
 
 RUN make install-uarch debian-package DESTDIR=$PWD/_install
 
-FROM --platform=$TARGETPLATFORM debian:bookworm-20230725-slim
+####################################################################################################
+FROM --platform=$TARGETPLATFORM debian:bookworm-20241016-slim
 ARG MACHINE_EMULATOR_VERSION=0.0.0
 ARG TARGETARCH
 
@@ -59,10 +90,9 @@ COPY --from=debian-packager \
 COPY --from=debian-packager /usr/local/lib/lua /usr/local/lib/lua
 COPY --from=debian-packager /usr/local/share/lua /usr/local/share/lua
 
-RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \
-    ./cartesi-machine.deb \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm cartesi-machine.deb
+RUN apt-get update && \
+    apt-get install -y ./cartesi-machine.deb && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/* cartesi-machine.deb
 
 RUN addgroup --system --gid 102 cartesi && \
     adduser --system --uid 102 --ingroup cartesi --disabled-login --no-create-home --home /nonexistent --gecos "cartesi user" --shell /bin/false cartesi

Makefile

@@ -117,6 +117,7 @@ relwithdebinfo?=no
 release?=no
 sanitize?=no
 coverage?=no
+git_commit?=
 
 # If not build type is chosen, set the default to release with debug information,
 # so the emulator is packaged correctly by default.
@@ -129,6 +130,7 @@ export debug
 export relwithdebinfo
 export release
 export coverage
+export git_commit
 
 COVERAGE_TOOLCHAIN?=gcc
 export COVERAGE_TOOLCHAIN
@@ -161,7 +163,7 @@ help:
 	@echo 'Main targets:'
 	@echo '* all                                 - Build the src/ code. To build from a clean clone, run: make submodules all'
 	@echo '  uarch                               - Build microarchitecture (requires riscv64-cartesi-linux-gnu-* toolchain)'
-	@echo '  uarch-with-linux-env                - Build microarchitecture using the linux-env docker image'
+	@echo '  uarch-with-toolchain                - Build microarchitecture using the toolchain docker image'
 	@echo '  build-tests-all                     - Build all tests (machine, uarch and misc)'
 	@echo '  build-tests-machine                 - Build machine emulator tests (requires rv64gc-lp64d riscv64-cartesi-linux-gnu-* toolchain)'
 	@echo '  build-tests-machine-with-toolchain  - Build machine emulator tests using the rv64gc-lp64d toolchain docker image'
@@ -177,7 +179,7 @@ help:
 	@echo 'Docker images targets:'
 	@echo '  build-emulator-image                - Build the machine-emulator debian based docker image'
 	@echo '  build-debian-package                - Build the cartesi-machine.deb package from image'
-	@echo '  build-linux-env                     - Build the linux environment docker image'
+	@echo '  build-toolchain                     - Build the emulator toolchain docker image'
 	@echo '  create-generated-files-patch        - Create patch that adds generated files to source tree'
 	@echo 'Cleaning targets:'
 	@echo '  clean                               - Clean the src/ artifacts'
@@ -251,8 +253,8 @@ $(SRCDIR)/machine-c-version.h:
 build-emulator-builder-image:
 	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --target builder -t cartesi/machine-emulator:builder -f Dockerfile .
 
-build-emulator-linux-env-image build-linux-env:
-	docker build $(DOCKER_PLATFORM) --target linux-env -t cartesi/machine-emulator:linux-env -f Dockerfile .
+build-emulator-toolchain-image build-toolchain:
+	docker build $(DOCKER_PLATFORM) --target toolchain -t cartesi/machine-emulator:toolchain -f Dockerfile .
 
 build-emulator-image:
 	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --build-arg MACHINE_EMULATOR_VERSION=$(MACHINE_EMULATOR_VERSION) -t cartesi/machine-emulator:$(TAG) -f Dockerfile .
@@ -285,36 +287,36 @@ copy:
 	docker cp uarch-ram-bin:/usr/src/emulator/uarch/uarch-pristine-hash.c .
 	docker rm uarch-ram-bin
 
-check-linux-env:
-	@if docker images $(DOCKER_PLATFORM) -q cartesi/machine-emulator:linux-env 2>/dev/null | grep -q .; then \
-		echo "Docker image cartesi/machine-emulator:linux-env exists"; \
+check-toolchain:
+	@if docker images $(DOCKER_PLATFORM) -q cartesi/machine-emulator:toolchain 2>/dev/null | grep -q .; then \
+		echo "Docker image cartesi/machine-emulator:toolchain exists"; \
 	else \
-		echo "Docker image cartesi/machine-emulator:linux-env does not exist. Creating:"; \
-		$(MAKE) build-linux-env; \
+		echo "Docker image cartesi/machine-emulator:toolchain does not exist. Creating:"; \
+		$(MAKE) build-toolchain; \
 	fi
 
-linux-env: check-linux-env
-	@docker run $(DOCKER_PLATFORM) --hostname linux-env -it --rm \
+toolchain-env: check-toolchain
+	@docker run $(DOCKER_PLATFORM) --hostname toolchain -it --rm \
 		-e USER=$$(id -u -n) \
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
 		-v `pwd`:/opt/cartesi/machine-emulator \
 		-w /opt/cartesi/machine-emulator \
-		cartesi/machine-emulator:linux-env /bin/bash
+		cartesi/machine-emulator:toolchain /bin/bash
 
-linux-env-exec: check-linux-env
-	@docker run --hostname linux-env --rm \
+toolchain-exec: check-toolchain
+	@docker run --hostname toolchain --rm \
 		-e USER=$$(id -u -n) \
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
 		-v `pwd`:/opt/cartesi/machine-emulator \
 		-w /opt/cartesi/machine-emulator \
-		cartesi/machine-emulator:linux-env /bin/bash -c "$(CONTAINER_COMMAND)"
+		cartesi/machine-emulator:toolchain /bin/bash -c "$(CONTAINER_COMMAND)"
 
-uarch-with-linux-env:
-	@$(MAKE) linux-env-exec CONTAINER_COMMAND="make uarch"
+uarch-with-toolchain:
+	@$(MAKE) toolchain-exec CONTAINER_COMMAND="make uarch"
 
 # Create install directories
 $(BIN_INSTALL_PATH) $(LIB_INSTALL_PATH) $(LUA_INSTALL_PATH) $(LUA_INSTALL_CPATH) $(LUA_INSTALL_CPATH)/cartesi $(LUA_INSTALL_PATH)/cartesi $(INC_INSTALL_PATH) $(IMAGES_INSTALL_PATH) $(UARCH_INSTALL_PATH) $(TESTS_DATA_INSTALL_PATH) $(TESTS_SCRIPTS_INSTALL_PATH) $(TESTS_LUA_INSTALL_PATH):

README.md

@@ -12,7 +12,7 @@ Run `make help` for a list of target options. Here are some of them:
 Main targets:
 * all                                 - Build the src/ code. To build from a clean clone, run: make submodules all
   uarch                               - Build microarchitecture (requires riscv64-cartesi-linux-gnu-* toolchain)
-  uarch-with-linux-env                - Build microarchitecture using the linux-env docker image
+  uarch-with-toolchain                - Build microarchitecture using the toolchain docker image
   build-tests-all                     - Build all tests (machine, uarch and misc)
   build-tests-machine                 - Build machine emulator tests (requires rv64gc-lp64d riscv64-cartesi-linux-gnu-* toolchain)
   build-tests-machine-with-toolchain  - Build machine emulator tests using the rv64gc-lp64d toolchain docker image
@@ -28,7 +28,7 @@ Main targets:
 Docker images targets:
   build-emulator-image                - Build the machine-emulator debian based docker image
   build-debian-package                - Build the cartesi-machine.deb package from image
-  build-linux-env                     - Build the linux environment docker image
+  build-toolchain                     - Build the emulator toolchain docker image
   create-generated-files-patch        - Create patch that adds generated files to source tree
 Cleaning targets:
   clean                               - Clean the src/ artifacts

src/Makefile

@@ -576,7 +576,7 @@ ifeq (,$(wildcard ../uarch/uarch-pristine-hash.c))
 	@if [ "$(DEV_ENV_HAS_TOOLCHAIN)" = "yes" ]; then \
 		$(MAKE) -C .. uarch; \
 	else \
-		$(MAKE) -C .. uarch-with-linux-env; \
+		$(MAKE) -C .. uarch-with-toolchain; \
 	fi
 endif
 

tests/Dockerfile

@@ -1,34 +1,20 @@
 ARG TAG=devel
-FROM --platform=$TARGETPLATFORM cartesi/toolchain:0.17.0 as machine-tests-builder
+FROM --platform=$TARGETPLATFORM cartesi/machine-emulator:builder AS tests-builder
 ARG DEBUG=no
 ARG COVERAGE=no
 ARG SANITIZE=no
 
-COPY . /usr/src/emulator
-
-WORKDIR /usr/src/emulator
-
 RUN make -j$(nproc) build-tests-machine debug=$DEBUG coverage=$COVERAGE sanitize=$SANITIZE
-
-FROM --platform=$TARGETPLATFORM cartesi/machine-emulator:builder as tests-builder
-ARG DEBUG=no
-ARG COVERAGE=no
-ARG SANITIZE=no
-
-COPY --from=machine-tests-builder /usr/src/emulator/tests/build/machine /usr/src/emulator/tests/build/machine
-
 RUN make -j$(nproc) build-tests-misc build-tests-uarch build-tests-images debug=$DEBUG coverage=$COVERAGE sanitize=$SANITIZE
 
-FROM tests-builder as tests-debian-packager
-ARG MACHINE_EMULATOR_VERSION=0.0.0
-ARG TARGETARCH
+####################################################################################################
+FROM tests-builder AS tests-debian-packager
 
 RUN make tests-debian-package DESTDIR=$PWD/dpkg/tests-install && \
     make tests-data-debian-package DESTDIR=$PWD/dpkg/tests-data-install
 
+####################################################################################################
 FROM --platform=$TARGETPLATFORM cartesi/machine-emulator:$TAG
-ARG MACHINE_EMULATOR_VERSION=0.0.0
-ARG TARGETARCH
 
 ENV CARTESI_IMAGES_PATH=/usr/share/cartesi-machine/tests/data/images
 ENV CARTESI_TESTS_PATH=/usr/share/cartesi-machine/tests/data/machine

tests/Makefile

@@ -19,11 +19,9 @@ INSTALLDIR = $(PREFIX)/tests
 
 LUA_BIN ?= $(shell which lua5.4)
 
-TOOLCHAIN_DOCKER_REPOSITORY ?= cartesi/toolchain
-TOOLCHAIN_TAG ?= 0.17.0
-
 BUILDDIR = $(abspath build)
-RISCV_PREFIX = riscv64-cartesi-linux-gnu-
+RISCV_PREFIX = riscv64-unknown-elf-
+RISCV_GCC_OPTS = -static -mcmodel=medany -fvisibility=hidden -ffreestanding -nostdlib -nostartfiles -I/usr/riscv64-linux-gnu/include
 
 MACHINE_EMULATOR_SRC_DIR = $(abspath ../src)
 
@@ -97,18 +95,19 @@ build-tests-misc: misc
 
 build-tests-images: images
 
-build-tests-uarch-with-toolchain uarch-with-toolchain riscv-arch-test-with-toolchain: TOOLCHAIN_TAG:=0.17.0-rv64ima-lp64
-
+build-tests-uarch-with-toolchain uarch-with-toolchain riscv-arch-test-with-toolchain:
 machine uarch misc:
 	@$(MAKE) $(BUILDDIR)/$@
 	@$(MAKE) -C $@ BUILDDIR=$(BUILDDIR)/$@
 
-riscv-tests: $(BUILDDIR)/riscv-tests $(BUILDDIR)/machine
+riscv-tests:  $(BUILDDIR)/riscv-tests/Makefile | $(BUILDDIR)/machine
 	@cd $(BUILDDIR)/$@ && ../../../third-party/riscv-tests/configure
-	@$(MAKE) -C $(BUILDDIR)/$@ RISCV_PREFIX=$(RISCV_PREFIX) isa
+	@$(MAKE) -C $(BUILDDIR)/$@ RISCV_PREFIX=$(RISCV_PREFIX) RISCV_GCC_OPTS="$(RISCV_GCC_OPTS)" isa
 	@cp -a $(BUILDDIR)/riscv-tests/isa/*.bin $(BUILDDIR)/riscv-tests/isa/*.dump $(BUILDDIR)/machine
 	@cd $(BUILDDIR)/riscv-tests/isa && find . -maxdepth 1 -type f ! -name "*.*" -exec cp -a {} $(BUILDDIR)/machine/{}.elf \;
-	@rm -rf $(BUILDDIR)/riscv-tests
+
+$(BUILDDIR)/riscv-tests/Makefile: | $(BUILDDIR)/riscv-tests
+	@cd $(BUILDDIR)/riscv-tests && ../../../third-party/riscv-tests/configure
 
 riscv-arch-test:
 	@$(MAKE) $(BUILDDIR)/uarch-$@
@@ -144,7 +143,7 @@ toolchain-env:
 		-e GID=$$(id -g) \
 		-v `pwd`/../:/usr/src/emulator \
 		-w /usr/src/emulator/tests \
-		$(TOOLCHAIN_DOCKER_REPOSITORY):$(TOOLCHAIN_TAG) /bin/bash
+		cartesi/machine-emulator:toolchain /bin/bash
 
 toolchain-exec:
 	@docker run --hostname toolchain-env --rm \
@@ -154,7 +153,7 @@ toolchain-exec:
 		-e GID=$$(id -g) \
 		-v `pwd`/../:/usr/src/emulator \
 		-w /usr/src/emulator/tests \
-		$(TOOLCHAIN_DOCKER_REPOSITORY):$(TOOLCHAIN_TAG) $(CONTAINER_COMMAND)
+		cartesi/machine-emulator:toolchain $(CONTAINER_COMMAND)
 
 emulator-builder-exec:
 	@docker run --hostname toolchain-env --rm \
@@ -167,7 +166,7 @@ emulator-builder-exec:
 		cartesi/machine-emulator:builder $(CONTAINER_COMMAND)
 
 $(TARGETS_WITH_TOOLCHAIN):
-	$(MAKE) toolchain-exec CONTAINER_COMMAND="make -j\$$(nproc) $(subst -with-toolchain,,$@)" TOOLCHAIN_DOCKER_REPOSITORY=$(TOOLCHAIN_DOCKER_REPOSITORY) TOOLCHAIN_TAG=$(TOOLCHAIN_TAG)
+	$(MAKE) toolchain-exec CONTAINER_COMMAND="make -j\$$(nproc) $(subst -with-toolchain,,$@)"
 
 build-tests-misc-with-builder-image:
 	$(MAKE) emulator-builder-exec CONTAINER_COMMAND="make -j\$$(nproc) build-tests-misc"