Run more often, attempt to add to the jsonl automatically

.github/workflows/coinbase.yml

@@ -2,7 +2,7 @@ name: Coinbase
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '*/15 * * * *'
+    - cron:  '*/5 * * * *'
 concurrency: commit
 jobs:
   price_history:
@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.0.1'
+          ruby-version: '3.3.0'
       - name: Config git user
         run: |
           git config user.name "Russell Smith"
@@ -21,13 +21,13 @@ jobs:
         run: |
           cd coinbase
           ruby currencies.rb
-          ruby products.rb
           ruby 24h_stats.rb
-          git add products.json 
+          git add products.json
           git commit products.json -m "Updated Coinbase products in run ${GITHUB_RUN_NUMBER}" || true
           git add currencies.json
           git commit currencies.json -m "Updated Coinbase currencies in run ${GITHUB_RUN_NUMBER}" || true
           git add 24h_stats/*.json
           git commit 24h_stats/*.json -m "Updated Coinbase price history in run ${GITHUB_RUN_NUMBER}" || true
+          sh add_last_commit.sh
       - name: Push repo
         run: git push || true

add_last_commit.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+last_commit_hash=""
+last_commit_time=""
+
+# Generate a list of commit hashes and associated file names
+git log --full-history -n 1 --reverse --name-only --pretty=format:"Commit: %H" -- "coinbase/24h_stats/BTC*" | \
+awk '/^Commit:/ {commit=$2} !/^Commit:/ && NF {print commit, $0}' > commit_file_list.txt
+
+# Define a function to process each commit and file
+process_commit_file() {
+    read -r commit file <<<"$1"
+
+    # Fetch the commit time (no need to check for last commit as each job is independent)
+    commit_time=$(git show -s --format=%ci "$commit")
+
+    # Use git show to display the content of each file at the point of the commit
+    if git ls-tree -r --name-only "$commit" | grep -q "^$file$"; then
+        git show "${commit}:${file}" | jq -c --arg hash "$commit" --arg commit_time "$commit_time" '. + {commit_time: $commit_time, h: $hash}' >> "coinbase/jsonl/$(basename "$file")"
+    fi
+}
+
+export -f process_commit_file
+
+# Use GNU Parallel to process each commit and file in parallel
+cat commit_file_list.txt | parallel -j 30 process_commit_file