This repository has been archived by the owner on Jan 10, 2025. It is now read-only.
🐧🧹 HealthChecks 🖳🗑️ #1763
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🐧🧹 HealthChecks 🖳🗑️ | |
| #MAX_RUNTIME: 02 Minutes */10 * * * * | |
| on: | |
| #push: | |
| workflow_dispatch: | |
| schedule: | |
| # - cron: "45 03 * * *" # 03:45 AM UTC --> 09:30 AM Morning NPT | |
| - cron: "0 */12 * * *" # Every 12 Hrs | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.TOOLPACKS }}" | |
| RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
| R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks" | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| check-post: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Debloat Runner | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/debloat.sh") | |
| continue-on-error: true | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| - name: Install Golang | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| pushd "$(mktemp -d)" >/dev/null | |
| echo "yes" | bash <(curl -qfsSL "https://git.io/go-installer") | |
| popd >/dev/null | |
| continue-on-error: true | |
| - name: Install NIX | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| "/nix/nix-installer" uninstall --no-confirm 2>/dev/null | |
| curl -qfsSL "https://install.determinate.systems/nix" | bash -s -- install linux --init none --no-confirm | |
| source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" | |
| sudo chown --recursive "runner" "/nix" | |
| echo "root ALL=(ALL:ALL) ALL" | sudo tee -a "/etc/sudoers" | |
| nix --version && nix-channel --list && nix-channel --update | |
| continue-on-error: true | |
| - name: Install Rust | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://sh.rustup.rs") -y | |
| continue-on-error: true | |
| - name: Install Zig | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| sudo rm "/usr/local/zig" -rf 2>/dev/null ; sudo rm "/usr/local/zig" -rf 2>/dev/null | |
| pushd "$(mktemp -d)" >/dev/null 2>&1 && curl -qfSLJO $(curl -qfsSL "https://ziglang.org/download/index.json" | jq -r '.master | ."x86_64-linux".tarball') | |
| find . -type f -name '*.tar*' -exec tar -xf {} \; | |
| sudo mkdir -p "/usr/local/zig" && sudo mv "$(find . -maxdepth 1 -type d | grep -v '^.$')"/* "/usr/local/zig" ; popd >/dev/null 2>&1 | |
| continue-on-error: true | |
| - name: Dos2Unix Everything | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find . -type f ! -path "./.git/*" -exec dos2unix {} \; 2>/dev/null | |
| continue-on-error: true | |
| - name: ActionLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find ".github/workflows" -type f -name "*ml" -exec actionlint {} \; | |
| continue-on-error: true | |
| - name: Shellcheck | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find ".github" -type f -name '*.sh' -exec shellcheck --exclude="SC2261" --severity=error "{}" \; 2>/dev/null | tee "${GITHUB_WORKSPACE}/main/.github/shellcheck.txt" | |
| continue-on-error: true | |
| - name: YAMLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| continue-on-error: true | |
| - name: Generate GO_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| go tool dist list --json | jq -r '["| [GOOS](https://pkg.go.dev/internal/platform) | [GOARCH](https://github.com/golang/go/blob/master/src/go/build/syslist.go) | [CGO_SUPPORT](https://web.archive.org/web/20240614051126/https://medium.com/@pengcheng1222/exploring-cgo-enabled-in-go-23cf5cf2fe88) | [NATIVE](https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63) |", "| --- | --- | --- | --- |"] + (map("| \(.GOOS) | \(.GOARCH) | \(.CgoSupported | if . then "✅" else "❌" end) | \(.FirstClass | if . then "✅" else "❌" end) |")) | join("\n")' > "$GITHUB_WORKSPACE/main/Docs/GO_TARGETS.md" | |
| continue-on-error: true | |
| - name: Generate NIX_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #https://github.com/NixOS/nixpkgs/blob/master/lib/systems/examples.nix | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| nix-instantiate --eval --strict --expr 'let pkgs = import <nixpkgs> {}; pkgsCross = pkgs.pkgsCross; in builtins.attrNames pkgsCross' | tr -d '[]"' | tr ' ' '\n' | sed '/^\s*$/d' | sed 's/^/pkgsCross./' > "$GITHUB_WORKSPACE/main/Docs/NIX_TARGETS.txt" | |
| continue-on-error: true | |
| - name: Generate RUST_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| rustup target list > "$GITHUB_WORKSPACE/main/Docs/RUST_TARGETS.txt" | |
| continue-on-error: true | |
| - name: Generate ZIG_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| zig targets | jq -r '.libc[]' > "$GITHUB_WORKSPACE/main/Docs/ZIG_TARGETS.txt" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo-I ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #copy & chdir to Repo | |
| cp -r "$GITHUB_WORKSPACE/main" "$SYSTMP/REPO" && cd "$SYSTMP/REPO" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| # rm -rf "$(pwd)/aarch64_arm64_Linux" | |
| # rm -rf "$(pwd)/x86_64_Linux" | |
| #Upload to Pub | |
| wait ; echo | |
| echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
| rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| ##Scripts_aarch64_Linux | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_aarch64_Linux" & | |
| ##Scripts_x86_64_Linux | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_x86_64_Linux" & | |
| ##Clean Pretty Formatted | |
| wait ; echo | |
| echo "[" $(cat metadata.json.tmp_aarch64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
| ##Update Metadata | |
| wait ; echo | |
| rclone copyto --checksum "./metadata.json_aarch64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/metadata.json" & | |
| rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/metadata.json" & | |
| ##Update Repo | |
| wait ; echo | |
| cat "./metadata.json_aarch64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/metadata.json" | |
| cat "./metadata.json_x86_64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/metadata.json" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/aarch64_arm64_Linux/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to ${GITHUB_WORKSPACE}/main/aarch64-Linux | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/FILE.txt" "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.json" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.min.json" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.min.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.toml" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.toml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.yaml" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.AIO.yaml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SIZE.txt" "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.md" 2>/dev/null | |
| cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.md" 2>/dev/null | |
| echo '```' >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" "${GITHUB_WORKSPACE}/main/aarch64-Linux/SHA256SUM.txt" 2>/dev/null | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| #-----------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/x86_64_Linux/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to ${GITHUB_WORKSPACE}/main/x86_64-Linux | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/FILE.txt" "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.json" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.min.json" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.min.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.toml" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.toml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.yaml" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.AIO.yaml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/SIZE.txt" "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.md" 2>/dev/null | |
| cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.md" 2>/dev/null | |
| echo '```' >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/SHA256SUM.txt" "${GITHUB_WORKSPACE}/main/x86_64-Linux/SHA256SUM.txt" 2>/dev/null | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Generate Repo Metadata (git-sizer) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Dust sizes | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee -a "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee "$GITHUB_WORKSPACE/main/.github/SIZE.txt" | |
| echo '```' >> "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| continue-on-error: true | |
| - name: Parse Build Logs (aarch64_arm64_Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #--------------------# | |
| ##aarch64_arm64_Linux## | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/BUILD.BIN.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| #Secrets & Tokens | |
| sed -i '/.*github_pat.*/Id' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*ghp_.*/Id' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*access_key_id.*/Id' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*secret_access_key.*/Id' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*cloudflarestorage.*/Id' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" | |
| echo -e '```mathematica' > "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Broken GH Repos | |
| echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -B5 -A3 -in ".* Authentication failed .*" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Eget Fails | |
| echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -B5 -in "asset .* not found" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Failed URLs | |
| echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Failed Installs | |
| echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -in "command not found" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Dynamically Linked | |
| echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -in "dynamically linked" "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Not correct binary | |
| echo -e "\n\n[-] Incorrect Arch --> # Not for aarch64\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -ivn "aarch64\|text" "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Build Failures | |
| echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| grep -in "No such file or directory" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #Empty Bins | |
| echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" >> "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| #END | |
| echo -e '```' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_ERROR.log.md" | |
| continue-on-error: true | |
| - name: Parse Build Logs (x86_64_Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #--------------------# | |
| ##x86_64_Linux## | |
| #Secrets & Tokens | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/BUILD.BIN.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*github_pat.*/Id' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*ghp_.*/Id' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*access_key_id.*/Id' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*secret_access_key.*/Id' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*cloudflarestorage.*/Id' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" | |
| echo -e '```mathematica' > "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Broken GH Repos | |
| echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -B5 -A3 -in ".* Authentication failed .*" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Eget Fails | |
| echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -B5 -in "asset .* not found" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Failed URLs | |
| echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Failed Installs | |
| echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -in "command not found" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Dynamically Linked | |
| echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -in "dynamically linked" "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Not correct binary | |
| echo -e "\n\n[-] Incorrect Arch --> # Not for x86_64\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -ivn "x86-64\|text" "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Broken Builds | |
| echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| grep -in "No such file or directory" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.BIN.log.txt" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #Empty Bins | |
| echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" >> "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| #END | |
| echo -e '```' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_ERROR.log.md" | |
| continue-on-error: true | |
| - name: Generate Markdown (Main README.md) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Generate $GITHUB_WORKSPACE/main/README.md | |
| ##bin.ajam.dev/aarch64_arm64_Linux | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s|\(https://img.shields.io/badge/Binaries-[(]*\)[0-9]\+|\1${TOTAL_PACKAGES}|" -i "$GITHUB_WORKSPACE/main/README.md" | |
| ##bin.ajam.dev/x86_64_Linux/ | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed -E "s|(https://img.shields.io/badge/Binaries-[^(]*\([0-9]+\)\+\()[0-9]+|\1${TOTAL_PACKAGES}|" -i "$GITHUB_WORKSPACE/main/README.md" | |
| sed -E "s~(https://bin\.pkgforge\.dev/list\.gif)[^\"]*~\1?$(basename "$(mktemp -u)")=$(basename "$(mktemp -u)")~g" -i "${GITHUB_WORKSPACE}/main/README.md" | |
| continue-on-error: true | |
| - name: Scrape Public Issues | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| bash <(curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks/refs/heads/main/.github/scripts/scrape_pub_issues.sh") | |
| continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| # Date Time | |
| NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
| echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| # - name: Git Pull | |
| # run: | | |
| # cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
| # continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
| commit_user_email: AjamX101@gmail.com # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
| commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
| #push_options: '--force' | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| - name: rClone Generate Metadata (MAIN) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| #--------------------# | |
| ##aarch64_arm64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bsum|capnp|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|xml|xz|zip|zstd)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/\(.Path)", extra_bins, pkg_family, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_aarch64_arm64_Linux" & | |
| ##x86_64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/x86_64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bsum|capnp|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|xml|xz|zip|zstd)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/\(.Path)", extra_bins, pkg_family, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_x86_64_Linux" & | |
| ##Clean Pretty Formatted | |
| wait ; echo | |
| echo "[" $(cat metadata.json.tmp_aarch64_arm64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_arm64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
| ##Update Metadata | |
| rclone copyto --checksum "./metadata.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/bin/x86_64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| wait ; echo | |
| continue-on-error: true | |
| - name: rClone Generate Metadata (BASEUTILS) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| #--------------------# | |
| ##aarch64_arm64_Linux## | |
| rclone copy "r2:/bin/aarch64_arm64_Linux/Baseutils/" "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##x86_64_Linux## | |
| rclone copy "r2:/bin/x86_64_Linux/Baseutils/" "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##Merge (INFO) | |
| wait ; echo | |
| ##aarch64_arm64_Linux## | |
| find "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" & | |
| find "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" & | |
| ##Enrich | |
| wait ; echo | |
| ##aarch64_arm64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bsum|capnp|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|xml|xz|zip|zstd)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/\(.Path)", extra_bins, pkg_family, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
| ##x86_64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/x86_64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bsum|capnp|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|xml|xz|zip|zstd)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/Baseutils/\(.Path)", extra_bins, pkg_family, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
| ##Clean | |
| wait ; echo | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
| ##Merge (ALL) | |
| wait ; echo | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_aarch64_arm64_Linux" & | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_x86_64_Linux" "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_x86_64_Linux" & | |
| ##Update | |
| rclone copyto --checksum "./metadata_Baseutils.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/Baseutils/METADATA.json" & | |
| rclone copyto --checksum "./metadata_Baseutils.json_x86_64_Linux" "r2:/bin/x86_64_Linux/Baseutils/METADATA.json" & | |
| wait ; echo | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo-II ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #chdir to Repo | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| #Upload to Pub | |
| echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
| rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-arm64-Linux: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_aarch64" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/aarch64_arm64_Linux/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/aarch64_Linux_FILE" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_FILE" "r2:/bin/aarch64_arm64_Linux/FILE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/aarch64_Linux_SIZE.txt" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_SIZE.txt" "r2:/bin/aarch64_arm64_Linux/SIZE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_BLAKE3SUM" "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_SHA256SUM" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_SHA256SUM" "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/" && yq ea '[.]' *.yaml > "${GITHUB_WORKSPACE}/main/aarch64-Linux/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-x86-64-Linux: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/x86_64_Linux) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_x86_64" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/x86_64_Linux/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/x86_64_Linux_FILE" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_FILE" "r2:/bin/x86_64_Linux/FILE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/x86_64_Linux_SIZE.txt" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_SIZE.txt" "r2:/bin/x86_64_Linux/SIZE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_BLAKE3SUM" "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_SHA256SUM" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_SHA256SUM" "r2:/bin/x86_64_Linux/SHA256SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/x86_64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/" && yq ea '[.]' *.yaml > "${GITHUB_WORKSPACE}/main/x86_64-Linux/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| merge-all: | |
| runs-on: ubuntu-latest | |
| needs: [enrich-arm64-Linux, enrich-x86-64-Linux] | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| - name: Merge & Update ("bin.ajam.dev/METADATA.json") | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| ##Fetch | |
| #aarch64_arm64_Linux | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| #cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
| #x86_64_Linux | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| #cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
| ##Merge | |
| if jq empty "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" 2>/dev/null; then | |
| jq -s '[{ "host": "aarch64-Linux" } + .[0][], { "host": "x86_64-Linux" } + .[1][]]' "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" | jq 'sort_by(.host)' > "$SYSTMP/METADATA.json" | |
| if jq empty "$SYSTMP/METADATA.json" 2>/dev/null; then | |
| ##rClone Update | |
| echo -e "\n Updating Metadata.json\n" | |
| rclone copyto --checksum "$SYSTMP/METADATA.json" "r2:/bin/METADATA.json" --check-first --checkers 2000 --transfers 1000 | |
| ##Copy to REPO | |
| rsync -av --checksum "$SYSTMP/METADATA.json" "$GITHUB_WORKSPACE/main/METADATA.json" | |
| #YAML | |
| cat "$GITHUB_WORKSPACE/main/METADATA.json" | yj -jy > "$GITHUB_WORKSPACE/main/METADATA.yaml" | |
| fi | |
| fi | |
| continue-on-error: true | |
| - name: Git Pull | |
| run: | | |
| cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
| continue-on-error: true | |
| - name: Update Repo | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| #aarch64_arm64_Linux | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.json" "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | |
| #x86_64_Linux | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.json" "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | |
| continue-on-error: true | |
| - name: Generate README (aarch64) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | yj -jy > "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.yaml" | |
| #Gen README | |
| cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "${GITHUB_WORKSPACE}/main/aarch64-Linux/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.md" | |
| sed 's/||//g' -i "${GITHUB_WORKSPACE}/main/aarch64-Linux/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Generate README (x86_64) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | yj -jy > "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.yaml" | |
| #Gen README | |
| cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "${GITHUB_WORKSPACE}/main/x86_64-Linux/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.md" | |
| sed 's/||//g' -i "${GITHUB_WORKSPACE}/main/x86_64-Linux/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Check Health (aarch64_arm64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/aarch64_arm64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "${GITHUB_WORKSPACE}/main/aarch64-Linux/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "${GITHUB_WORKSPACE}/main/aarch64-Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "${GITHUB_WORKSPACE}/main/aarch64-Linux/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "${GITHUB_WORKSPACE}/main/aarch64-Linux/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Check Health (x86_64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/x86_64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "${GITHUB_WORKSPACE}/main/x86_64-Linux/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "${GITHUB_WORKSPACE}/main/x86_64-Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "${GITHUB_WORKSPACE}/main/x86_64-Linux/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "${GITHUB_WORKSPACE}/main/x86_64-Linux/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| # Date Time | |
| NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
| echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
| commit_user_email: AjamX101@gmail.com # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
| commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
| #push_options: '--force' | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# |