Merge pull request #33 from DmitriySalnikov/volumetric

Volumetric and a bunch of other things

.gitattributes

@@ -9,9 +9,12 @@
 /.github			export-ignore
 /LICENSE			export-ignore
 /SConstruct			export-ignore
+/Doxyfile			export-ignore
 /*.*				export-ignore
 /src				export-ignore
 /godot-cpp			export-ignore
 /jni				export-ignore
 /patches			export-ignore
-/images				export-ignore
+/images				export-ignore
+/docs				export-ignore
+/dd3d_web_build		export-ignore

.github/actions/compile_gdextension/action.yml

@@ -6,37 +6,64 @@ inputs:
     required: true
   target:
     description: Compilation target (editor|template_release|template_debug)
+    required: true
     default: "release"
   arch:
     description: Arch (universal|x86_32|x86_64|arm32|arm64|rv64|ppc32|ppc64|wasm32)
+    required: true
     default: ""
+  artifact:
+    description: Artifact name. This name will be used for caching.
+    required: true
+    default: "bin"
   additional:
     description: Any additional arguments
     default: ""
-  artifact:
-    description: Artifact name
-    default: "bin"
+  additional_enabled_dd3d:
+    description: Use force_enabled_dd3d if possible
+    default: "true"
   output_libs_path:
     description: Path to compiled libraries
+    required: true
     default: "bin"
+  use_cache:
+    description: Use cache
+    default: "true"
+  token:
+    description: Secrets token
+  telemetry_version:
+    description: Telemetry version
+    default: 2fcd3ec9e528d1cd26978d0d27e9c6a3b2006270
 runs:
-  using: "composite"
+  using: composite
   steps:
     - name: Get godot-cpp SHA
       shell: bash
       id: get_godot_cpp_sha
       run: |
         echo "sha=$(git rev-parse @:./godot-cpp)" >> $GITHUB_OUTPUT
 
+    - name: Checkout telemetry repo
+      uses: actions/checkout@v4
+      id: checkout_tele_repo
+      if: ${{env.PRODUCTION_BUILD == 'true' && inputs.target == 'editor'}}
+      continue-on-error: true
+      with:
+        repository: ${{github.repository_owner}}/my_telemetry_modules
+        token: ${{inputs.token}}
+        ref: ${{inputs.telemetry_version}}
+        path: src/editor/my_telemetry_modules
+
     - name: Restore .scons_cache directory
+      if: inputs.use_cache != 'false'
+      id: restore_scons_cache
       uses: actions/cache/restore@v3
       with:
         path: ${{env.SCONS_CACHE}}
-        key: ${{github.job}}-${{inputs.arch}}_${{inputs.target}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}-${{github.sha}}
+        key: ${{github.job}}-${{inputs.artifact}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}-${{github.sha}}
         restore-keys: |
-          ${{github.job}}-${{inputs.arch}}_${{inputs.target}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}
-          ${{github.job}}-${{inputs.arch}}_${{inputs.target}}-${{steps.get_godot_cpp_sha.outputs.sha}}
-      continue-on-error: true
+          ${{github.job}}-${{inputs.artifact}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}
+          ${{github.job}}-${{inputs.artifact}}-${{steps.get_godot_cpp_sha.outputs.sha}}
 
     - name: Set up Python 3.x
       uses: actions/setup-python@v4
@@ -56,11 +83,11 @@ runs:
 
     - name: Linux dependencies
       shell: bash
-      if: runner.os == 'Linux'
+      if: (runner.os == 'Linux') && (inputs.platform == 'linux')
       run: |
-        sudo apt-get update
+        sudo apt update
         if [[ "${{inputs.arch}}" == *"32" ]];then
-          sudo apt-get install gcc-multilib g++-multilib
+          sudo apt install gcc-multilib g++-multilib
         fi
 
     - name: Compilation
@@ -72,8 +99,14 @@ runs:
         git apply --ignore-space-change --ignore-whitespace ../patches/godot_cpp_exclude_unused_classes.patch
         git apply --ignore-space-change --ignore-whitespace ../patches/unity_build.patch
         cd ..
-        scons platform=${{inputs.platform}} arch=${{inputs.arch}} target=${{inputs.target}} addon_output_dir=${{inputs.output_libs_path}} ${{inputs.additional}}
-        if [ "${{inputs.target}}" == "template_release" ] && [ "${{inputs.platform}}" != "android" ]; then
+
+        telemetry_args=""
+        if [ "${{env.PRODUCTION_BUILD}}" == "true" ] && [ "${{inputs.target}}" == "editor" ] && [ "${{steps.checkout_tele_repo.conclusion}}" == "success" ]; then
+          telemetry_args="telemetry_enabled=yes"
+        fi
+
+        scons platform=${{inputs.platform}} arch=${{inputs.arch}} target=${{inputs.target}} addon_output_dir=${{inputs.output_libs_path}} ${{inputs.additional}} $telemetry_args
+        if [ "${{inputs.target}}" == "template_release" ] && [ "${{inputs.additional_enabled_dd3d}}" == "true" ]; then
           scons platform=${{inputs.platform}} arch=${{inputs.arch}} target=${{inputs.target}} addon_output_dir=${{inputs.output_libs_path}} force_enabled_dd3d=yes ${{inputs.additional}}
         fi
         echo "::endgroup::"
@@ -95,15 +128,15 @@ runs:
         Remove-Item ${{inputs.output_libs_path}}/* -Recurse -Include *.exp,*.lib,*.pdb -Force
 
     - name: Upload Artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{inputs.artifact}}
         retention-days: 7
         path: ${{inputs.output_libs_path}}/*
 
     - name: Save .scons_cache directory
+      if: ${{!steps.restore_scons_cache.outputs.cache-hit && inputs.use_cache != 'false'}}
       uses: actions/cache/save@v3
       with:
         path: ${{env.SCONS_CACHE}}
-        key: ${{github.job}}-${{inputs.arch}}_${{inputs.target}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}-${{github.sha}}
-      continue-on-error: true
+        key: ${{github.job}}-${{inputs.artifact}}-${{steps.get_godot_cpp_sha.outputs.sha}}-${{github.ref}}-${{github.sha}}

.github/actions/delete_artifact/action.yml

@@ -0,0 +1,31 @@
+name: Delete the artifact if it exists
+description: Delete an artifact by its name if it exists. 'actions/upload-artifact@v4' usually removes artifacts itself when restarting jobs.
+inputs:
+  artifact:
+    description: Any name of artifact
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Delete `${{inputs.artifact}}`
+      shell: bash
+      continue-on-error: true
+      run: |
+        res=$(gh api \
+          -H "Accept: application/vnd.github+json" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          /repos/${{github.repository}}/actions/runs/${{github.run_id}}/artifacts?name="${{inputs.artifact}}")
+
+        artifact_id=$(echo "$res" | jq -r 'if .total_count > 0 then .artifacts[0].id else 0 end')
+
+        if [ "$artifact_id" == "0" ]; then
+            echo "No artifact to remove was found."
+        else
+          echo "Found the artifact ID $artifact_id for the ${{inputs.artifact}}."
+          gh api \
+            --method DELETE \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{github.repository}}/actions/artifacts/$artifact_id
+          echo "The artifact with the ID $artifact_id has been removed."
+        fi

.github/workflows/gdextension_build.yml

@@ -5,22 +5,36 @@ on:
   pull_request:
     paths: [src/**, .github/**, "!.github/**/util_*", "patches/**", lib_utils.py, SConstruct]
   workflow_dispatch:
-  repository_dispatch:
+    inputs:
+      production_build:
+        description: Production build
+        default: true
+        type: boolean
+      use_cache:
+        description: Use Cache
+        default: true
+        type: boolean
 
 # Stop the same workflow actions
 concurrency:
-  group: ${{github.workflow}}-${{github.event.pull_request.number || github.run_id}}
+  group: ${{github.workflow}}-${{github.event.pull_request.number || github.ref}}
   cancel-in-progress: true
 
+permissions:
+  actions: write
+
 env:
   SCONS_CACHE: ${{github.workspace}}/.scons-cache/
+  USE_CACHE: ${{!format('{0}', inputs.use_cache) && 'true' || format('{0}', inputs.use_cache)}} # Default true
+  PRODUCTION_BUILD: ${{!format('{0}', inputs.production_build) && 'false' || format('{0}', inputs.production_build)}} # Default false
   OUTPUT_LIBS_PATH: bin
   FORCE_DISABLE_UNITY: yes
+  GH_TOKEN: ${{ github.token }}
 
 jobs:
   windows-gdextension:
     name: 🏁 Windows
-    runs-on: "windows-latest"
+    runs-on: windows-latest
 
     strategy:
       fail-fast: false
@@ -30,7 +44,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
           submodules: recursive
@@ -41,15 +55,17 @@ jobs:
           platform: windows
           target: ${{matrix.target}}
           arch: ${{matrix.arch}}
-          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
-          artifact: windows
+          artifact: windows.${{matrix.target}}.${{matrix.arch}}
           additional: lto=yes
+          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
+          use_cache: ${{env.USE_CACHE}}
+          token: ${{secrets.TELEMETRY_TOKEN}}
 
   # ============================================
 
   linux-gdextension:
     name: 🐧 Linux
-    runs-on: "ubuntu-20.04"
+    runs-on: ubuntu-20.04
 
     strategy:
       fail-fast: false
@@ -59,7 +75,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
           submodules: recursive
@@ -70,15 +86,17 @@ jobs:
           platform: linux
           target: ${{matrix.target}}
           arch: ${{matrix.arch}}
-          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
-          artifact: linux
+          artifact: linux.${{matrix.target}}.${{matrix.arch}}
           additional: lto=yes
+          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
+          use_cache: ${{env.USE_CACHE}}
+          token: ${{secrets.TELEMETRY_TOKEN}}
 
   # ============================================
 
   macos-gdextension:
     name: 🍏 MacOS
-    runs-on: "macos-latest"
+    runs-on: macos-latest
 
     strategy:
       fail-fast: false
@@ -88,7 +106,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
           submodules: recursive
@@ -99,15 +117,17 @@ jobs:
           platform: macos
           target: ${{matrix.target}}
           arch: ${{matrix.arch}}
-          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
-          artifact: macos
+          artifact: macos.${{matrix.target}}.${{matrix.arch}}
           additional: lto=yes macos_deployment_target=10.14
+          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
+          use_cache: ${{env.USE_CACHE}}
+          token: ${{secrets.TELEMETRY_TOKEN}}
 
   # ============================================
 
   android-gdextension:
     name: 🤖 Android
-    runs-on: "ubuntu-latest"
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -117,7 +137,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
           submodules: recursive
@@ -148,15 +168,17 @@ jobs:
           platform: android
           target: ${{matrix.target}}
           arch: ${{matrix.arch}}
-          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
-          artifact: android
+          artifact: android.${{matrix.target}}.${{matrix.arch}}
           additional: lto=yes
+          additional_enabled_dd3d: false
+          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
+          use_cache: ${{env.USE_CACHE}}
 
   # ============================================
 
   web-gdextension:
     name: 🕸 Web
-    runs-on: "ubuntu-latest"
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -169,24 +191,30 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
           submodules: recursive
 
-      - uses: mymindstorm/setup-emsdk@v12
+      - name: Download Emscripten
+        uses: mymindstorm/setup-emsdk@v12
+        # Continue if failed to cache
+        # https://github.com/mymindstorm/setup-emsdk/issues/20
+        continue-on-error: true
         with:
           version: ${{env.EM_VERSION}}
+          actions-cache-folder: obj/emsdk_cache
 
       - name: Compile GDExtension
         uses: ./.github/actions/compile_gdextension
         with:
           platform: web
           target: ${{matrix.target}}
           arch: ${{matrix.arch}}
-          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
-          artifact: web
+          artifact: web.${{matrix.target}}.${{matrix.arch}}
           additional: lto=yes
+          output_libs_path: ${{env.OUTPUT_LIBS_PATH}}
+          use_cache: ${{env.USE_CACHE}}
 
   # ============================================
 
@@ -203,8 +231,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # https://github.com/actions/download-artifact/issues/249
       - name: Download Binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: extracted_files
 
@@ -215,9 +247,14 @@ jobs:
           rm -rf extracted_files/**/
           touch extracted_files/.gdignore
 
+      - name: Output file information
+        run: |
+          cd extracted_files
+          echo "Total size: $(du -ch -b | grep total | cut -f1 | awk '{printf "%.2f", $1/1048576}') MB, Total number of files: $(find . -type f | wc -l)" >> $GITHUB_STEP_SUMMARY
+
       - name: Upload GDExtension
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gdextension_libs
+          name: ${{env.PRODUCTION_BUILD == 'true' && '.gdextension_libs_production' || '.gdextension_libs'}}
           retention-days: 7
           path: extracted_files/*