feature: Add project creation script and tests (#107)

* script to copy files

* Fix path to copy from and let copy example overwrite existing dir

Signed-off-by: Dennis Meister <dennis.meister@bosch.com>

* extract method

* Add integration tests for create command

* Distinguish between different matrix logs

* Add minimal mock config

* Disable local runtime

* Fix pre-commit in created projects

* Exclude examples from linting

* Fix execution order in workflow

* Update examples/seat-adjuster/src/vapp.py

Co-authored-by: Markus Petke <MP91@users.noreply.github.com>

* Fix review findings

* Exclude wrong tests

* WIP file check

Signed-off-by: Dennis Meister <dennis.meister@bosch.com>

* Get rid of vscode settings file check

Signed-off-by: Dennis Meister <dennis.meister@bosch.com>

* Add runtime stopping in tests again

Signed-off-by: Dennis Meister <dennis.meister@bosch.com>

* Remove code path for pre-existing git repository

* Add CLI tag for checkout

* Revert "Remove code path for pre-existing git repository"

This reverts commit 968dda7.

* Fix dependency conflicts

* Remove git initialization

* Fix pip-compile

* Fix pre-commit run

* Run pip-compile as module

* make pip-compile silent + remove tox

* Print to stdout/stderr for error finding in CI

* remove root requirements.txt since it is generated

* Revert "Print to stdout/stderr for error finding in CI"

This reverts commit d41921e.

* remove unnecessary pip compile

* bump sdk version

* fix creation logs

* fix

* remove appManifest from skeleton

* fix script

* another fix

* Upgrade grpcio(-tools) and fix header

* Debugging

* change CLI ref

* Fix workflow

* Fix CLI version for CI workflow

* remove requirements.txt as it is generated during creation

* fix license

---------

Signed-off-by: Dennis Meister <dennis.meister@bosch.com>
Co-authored-by: Markus Petke <MP91@users.noreply.github.com>
Co-authored-by: Dennis Meister <dennis.meister@bosch.com>

.devcontainer/Dockerfile

@@ -12,7 +12,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.1
+FROM ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.2
 
 # Force dapr to use localhost traffic
 ENV DAPR_HOST_IP="127.0.0.1"

.devcontainer/devcontainer.json

@@ -20,6 +20,7 @@
 				"python.formatting.provider": "black",
 				"python.linting.banditEnabled": true,
 				"python.linting.flake8Enabled": true,
+				"editor.defaultFormatter": "ms-python.black-formatter",
 				"terminal.integrated.defaultProfile.linux": "zsh",
 				"terminal.integrated.profiles.linux": {
 					"zsh": {
@@ -38,6 +39,7 @@
 				"ms-kubernetes-tools.vscode-kubernetes-tools",
 				"matangover.mypy",
 				"anweber.vscode-httpyac",
+				"ms-python.black-formatter",
 				"ms-python.isort",
 				"ms-python.flake8"
 			]

.github/workflows/project-creation.yaml

@@ -0,0 +1,120 @@
+# Copyright (c) 2023 Robert Bosch GmbH
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Project creation
+
+concurrency:
+  group: ${{ github.ref }}-project-creation
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    # Run only on branches/commits and not tags
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  check-project-creation:
+    name: Check project creation
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        example: ["no-example", "seat-adjuster"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: latest
+
+      - name: Checkout CLI
+        uses: actions/checkout@v4
+        with:
+          repository: eclipse-velocitas/cli
+          path: cli
+          ref: v0.6.3
+
+      - name: Checkout SDK repo
+        uses: actions/checkout@v4
+        with:
+          path: sdk
+
+      - name: Install CLI
+        shell: bash
+        run: |
+          cd cli
+          npm i && npm run build && npx oclif manifest && npm i -g .
+
+      - name: Create project
+        shell: bash
+        run: |
+          git config --global user.name "Git User"
+          git config --global user.email "gituser@email.com"
+
+          # overwrite SDK path in order to use scripts from this branch (rather than sdk/main)
+          export VELOCITAS_SDK_PATH_OVERRIDE=$(pwd)/sdk
+
+          mkdir app
+          cd app
+
+          # show available commands and version of CLI
+          velocitas
+
+          # copy "full" package-index.json from CLI repo
+          cp ../cli/testbench/test-create/vehicle-app-template/package-index.json ./package-index.json
+          CREATE_ARGS="-n MyApp"
+          if [ "${{ matrix.example }}" != "no-example" ]; then
+            CREATE_ARGS="-e ${{ matrix.example }}"
+          fi
+          velocitas create -l python $CREATE_ARGS
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          pip install -r ./sdk/.project-creation/test/requirements.txt
+
+      - name: Run verification on generated project
+        shell: bash
+        run: |
+          export VELOCITAS_SDK_ROOT=./sdk
+          export VELOCITAS_APP_ROOT=./app
+
+          # debug print contents
+          ls -al $VELOCITAS_APP_ROOT
+
+          pytest --ignore-glob=**/.devcontainer/* --ignore-glob=**/.skeleton/* ./sdk/.project-creation
+
+      - name: Check if devContainer starts up properly and app is usable
+        uses: devcontainers/ci@v0.3
+        with:
+          subFolder: ./app
+          runCmd: |
+            git init && \
+              pre-commit run --all-files && \
+              pip3 install -r .devcontainer/tests/automated_tests/requirements.txt && \
+              cat app/AppManifest.json && \
+              velocitas exec grpc-interface-support generate-sdk && \
+              pytest -sx .devcontainer/tests/automated_tests
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: logs_${{ matrix.example }}
+          path: ./app/logs
+          retention-days: 1

.pre-commit-config.yaml

@@ -53,22 +53,25 @@ repos:
           (?x)^(
               .*_pb2.py|
               .*_pb2.pyi|
-              .*_pb2_grpc.py
+              .*_pb2_grpc.py|
+              examples/.*|
+              .project-creation/.skeleton/.*
           )$
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.9.1
     hooks:
       - id: black
         exclude: >
           (?x)^(
               .*_pb2.py|
               .*_pb2.pyi|
-              .*_pb2_grpc.py
+              .*_pb2_grpc.py|
+              examples/.*
           )$
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         exclude: >
@@ -82,14 +85,14 @@ repos:
           - flake8-unused-arguments
 
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.4
+    rev: 1.7.5
     hooks:
       - id: bandit
         args: ["--skip=B101"]
         types_or: [python]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.991"
+    rev: v1.5.1
     hooks:
       - id: mypy
         exclude: >
@@ -98,7 +101,7 @@ repos:
               .*_pb2.pyi|
               .*_pb2_grpc.py
           )$
-        args: [velocitas_sdk]
+        args: [.]
         language: system
         pass_filenames: false
 

.project-creation/.skeleton/Dockerfile

@@ -0,0 +1,69 @@
+# Copyright (c) 2023 Contributors to the Eclipse Foundation
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# syntax=docker/dockerfile:1.2
+
+# Build stage, to create the executable
+FROM --platform=$TARGETPLATFORM ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.2 as builder
+ARG TARGETARCH
+
+RUN apt-get update && apt-get install -y python3-dev
+
+COPY ./.velocitas.json /workspace/.velocitas_org.json
+COPY ./app /workspace/app
+
+# FIXME: For build tooling we only need "devenv-devcontainer-setup", we should be able to
+# filter this without manual jq intervention...
+RUN cat ./workspace/.velocitas_org.json | jq 'del(.packages[] | select(.name|test(".*devenv-devcontainer-setup.*")|not))' > ./workspace/.velocitas.json
+
+# Remove this installation for Arm64 once staticx has a prebuilt wheel for Arm64
+RUN /bin/bash -c 'set -ex && \
+    ARCH=`uname -m` && \
+    if [ "$ARCH" == "aarch64" ]; then \
+    echo "ARM64" && \
+    apt-get install -y gcc && \
+    pip3 install --no-cache-dir scons; \
+    fi'
+
+RUN pip3 install --no-cache-dir pyinstaller==5.9.0 \
+    && pip3 install --no-cache-dir patchelf==0.17.0.0 \
+    && pip3 install --no-cache-dir staticx \
+    && pip3 install --no-cache-dir -r ./workspace/app/requirements.txt \
+    && pip3 install --no-cache-dir -r ./workspace/app/requirements-links.txt
+
+WORKDIR /workspace
+
+RUN velocitas init
+
+WORKDIR /workspace/app
+
+RUN pyinstaller --clean -F -s -p src src/main.py
+
+WORKDIR /workspace/app/dist
+
+RUN staticx main run-exe
+
+# Runner stage, to copy the executable
+FROM scratch
+
+COPY --from=builder ./workspace/app/dist/run-exe /dist/
+
+WORKDIR /tmp
+WORKDIR /dist
+
+ENV PATH="/dist:$PATH"
+
+LABEL org.opencontainers.image.source="https://github.com/eclipse-velocitas/vehicle-app-python-template"
+
+CMD ["./run-exe"]

.project-creation/.skeleton/README.md

@@ -0,0 +1,3 @@
+# AppName Readme
+
+A simple skeleton app for getting you up and running with Velocitas app development.

.project-creation/.skeleton/requirements-velocitas.txt

@@ -0,0 +1 @@
+velocitas-sdk==0.13.0

.project-creation/.skeleton/requirements.in

@@ -0,0 +1,4 @@
+grpcio==1.59.0
+protobuf==4.24.4
+dapr==1.11.0
+cloudevents==1.10.0

.project-creation/.skeleton/requirements.txt

@@ -0,0 +1,50 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+aiohttp==3.8.6
+    # via dapr
+aiosignal==1.3.1
+    # via aiohttp
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.1.0
+    # via aiohttp
+charset-normalizer==3.3.0
+    # via aiohttp
+cloudevents==1.10.0
+    # via -r requirements.in
+dapr==1.11.0
+    # via -r requirements.in
+deprecation==2.1.0
+    # via cloudevents
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+grpcio==1.59.0
+    # via
+    #   -r requirements.in
+    #   dapr
+idna==3.4
+    # via yarl
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+packaging==23.2
+    # via deprecation
+protobuf==4.24.4
+    # via
+    #   -r requirements.in
+    #   dapr
+python-dateutil==2.8.2
+    # via dapr
+six==1.16.0
+    # via python-dateutil
+typing-extensions==4.8.0
+    # via dapr
+yarl==1.9.2
+    # via aiohttp

.project-creation/.skeleton/src/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023 Contributors to the Eclipse Foundation
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0

.project-creation/.skeleton/src/main.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2022-2023 Contributors to the Eclipse Foundation
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""The launcher for starting a Vehicle App."""
+
+import asyncio
+import logging
+import signal
+
+from vapp import AppName  # type: ignore # noqa: E402
+from velocitas_sdk.util.log import (  # type: ignore
+    get_opentelemetry_log_factory,
+    get_opentelemetry_log_format,
+)
+
+logging.setLogRecordFactory(get_opentelemetry_log_factory())
+logging.basicConfig(format=get_opentelemetry_log_format())
+logging.getLogger().setLevel("DEBUG")
+logger = logging.getLogger(__name__)
+
+
+async def main():
+    """Main function"""
+    logger.info("Starting AppName app...")
+    app = AppName()
+    await app.run()
+
+
+LOOP = asyncio.get_event_loop()
+LOOP.add_signal_handler(signal.SIGTERM, LOOP.stop)
+LOOP.run_until_complete(main())
+LOOP.close()