Update #418
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update | |
| on: | |
| schedule: | |
| - cron: '10 1,3,5,7,9,11,13,15,17,19,21,23 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| run-script: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| run: pip install --upgrade pip requests matplotlib | |
| - name: Download files | |
| run: | | |
| mkdir -p downloads | |
| curl -o downloads/nordomain_30day.txt ${{ secrets.NORDOMAIN_30DAY_URL }} | |
| curl -o downloads/nordomain_14day.txt ${{ secrets.NORDOMAIN_14DAY_URL }} | |
| curl -o downloads/phishing_30day.txt ${{ secrets.PHISHING_30DAY_URL }} | |
| curl -o downloads/phishing_14day.txt ${{ secrets.PHISHING_14DAY_URL }} | |
| curl -A "${{ secrets.USER_AGENT }}" -o downloads/additional.txt "${{ secrets.ADDITIONAL_SOURCE_URL }}" | |
| - name: Compute file hashes | |
| id: compute_hashes | |
| run: | | |
| mkdir -p hash | |
| for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do | |
| sha256sum downloads/$file.txt > hash/current_$file.txt | |
| done | |
| - name: Check if files have changed | |
| id: check_changes | |
| run: | | |
| CHANGED=false | |
| for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do | |
| if [ ! -f hash/previous_$file.txt ] || ! cmp -s hash/previous_$file.txt hash/current_$file.txt; then | |
| echo "$file has changed." | |
| CHANGED=true | |
| fi | |
| done | |
| echo "changed=$CHANGED" >> $GITHUB_ENV | |
| - name: Skip processing if unchanged | |
| if: ${{ env.changed == 'false' }} | |
| run: echo "Skipping further steps as no files have changed." | |
| - name: Run main script | |
| if: ${{ env.changed == 'true' }} | |
| env: | |
| NORDOMAIN_30DAY_URL: ${{ secrets.NORDOMAIN_30DAY_URL }} | |
| NORDOMAIN_14DAY_URL: ${{ secrets.NORDOMAIN_14DAY_URL }} | |
| PHISHING_30DAY_URL: ${{ secrets.PHISHING_30DAY_URL }} | |
| PHISHING_14DAY_URL: ${{ secrets.PHISHING_14DAY_URL }} | |
| ADDITIONAL_SOURCE_URL: ${{ secrets.ADDITIONAL_SOURCE_URL }} | |
| USER_AGENT: ${{ secrets.USER_AGENT }} | |
| run: python3 src/nrd-hunter.py | |
| - name: Filter domains using Tranco and Umbrella lists | |
| if: ${{ env.changed == 'true' }} | |
| run: | | |
| # Create a temporary directory for blocklist downloads | |
| mkdir -p blocklists | |
| # Download and extract Tranco and Umbrella lists | |
| curl -o blocklists/tranco.zip https://tranco-list.eu/top-1m.csv.zip | |
| curl -o blocklists/umbrella.zip https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip | |
| unzip blocklists/tranco.zip -d blocklists/ | |
| unzip blocklists/umbrella.zip -d blocklists/ | |
| # Combine the Tranco and Umbrella lists into a single sorted blocklist | |
| cat blocklists/top-1m.csv blocklists/top-1m.csv.1 | awk -F, '{print $2}' | sort -u > blocklists/combined_blocklist.txt | |
| # Define a filtering function to remove domains and subdomains | |
| filter_domains() { | |
| input_file=$1 | |
| output_file=$2 | |
| blocklist=$3 | |
| # Use grep to exclude matching domains and subdomains | |
| grep -vFf "$blocklist" "$input_file" > "$output_file" | |
| } | |
| # Filter each output file using the combined blocklist | |
| mkdir -p filtered_output | |
| for file in output/*.txt; do | |
| filtered_file="filtered_output/$(basename $file)" | |
| filter_domains "$file" "$filtered_file" blocklists/combined_blocklist.txt | |
| done | |
| # Replace the original files with the filtered ones | |
| mv filtered_output/* output/ | |
| - name: Move output files | |
| if: ${{ env.changed == 'true' }} | |
| run: | | |
| # Create necessary directories for organized output | |
| mkdir -p \ | |
| lists/14-day/domains-only \ | |
| lists/30-day/domains-only \ | |
| lists/14-day_phishing/domains-only \ | |
| lists/30-day_phishing/domains-only \ | |
| lists/14-day/adblock \ | |
| lists/30-day/adblock \ | |
| lists/14-day_phishing/adblock \ | |
| lists/30-day_phishing/adblock \ | |
| lists/14-day/wildcard \ | |
| lists/30-day/wildcard \ | |
| lists/14-day_phishing/wildcard \ | |
| lists/30-day_phishing/wildcard \ | |
| lists/14-day/base64 \ | |
| lists/30-day/base64 \ | |
| lists/14-day_phishing/base64 \ | |
| lists/30-day_phishing/base64 \ | |
| lists/14-day/unbound \ | |
| lists/30-day/unbound \ | |
| lists/14-day_phishing/unbound \ | |
| lists/30-day_phishing/unbound | |
| # Helper function to safely move files | |
| move_if_exists() { | |
| src=$1 | |
| dest=$2 | |
| if [ -f "$src" ]; then | |
| mv "$src" "$dest" | |
| else | |
| echo "Warning: File $src does not exist." | |
| fi | |
| } | |
| move_if_exists output/nrd-14day_wildcard.txt lists/14-day/wildcard | |
| move_if_exists output/nrd-14day.txt lists/14-day/domains-only | |
| move_if_exists output/nrd-30day_part1.txt lists/30-day/domains-only | |
| move_if_exists output/nrd-30day_part2.txt lists/30-day/domains-only | |
| move_if_exists output/nrd-14day_adblock.txt lists/14-day/adblock | |
| move_if_exists output/nrd-30day_adblock_part1.txt lists/30-day/adblock | |
| move_if_exists output/nrd-30day_adblock_part2.txt lists/30-day/adblock | |
| move_if_exists output/nrd-14day_base64.txt lists/14-day/base64 | |
| move_if_exists output/nrd-30day_base64_part1.txt lists/30-day/base64 | |
| move_if_exists output/nrd-30day_base64_part2.txt lists/30-day/base64 | |
| move_if_exists output/nrd-14day_unbound_part1.txt lists/14-day/unbound | |
| move_if_exists output/nrd-14day_unbound_part2.txt lists/14-day/unbound | |
| move_if_exists output/nrd-30day_unbound_part1.txt lists/30-day/unbound | |
| move_if_exists output/nrd-30day_unbound_part2.txt lists/30-day/unbound | |
| move_if_exists output/nrd-30day_unbound_part3.txt lists/30-day/unbound | |
| move_if_exists output/nrd-30day_wildcard_part1.txt lists/30-day/wildcard | |
| move_if_exists output/nrd-30day_wildcard_part2.txt lists/30-day/wildcard | |
| move_if_exists output/nrd-phishing-14day.txt lists/14-day_phishing/domains-only | |
| move_if_exists output/nrd-phishing-30day.txt lists/30-day_phishing/domains-only | |
| move_if_exists output/nrd-phishing-14day_adblock.txt lists/14-day_phishing/adblock | |
| move_if_exists output/nrd-phishing-30day_adblock.txt lists/30-day_phishing/adblock | |
| move_if_exists output/nrd-phishing-14day_base64.txt lists/14-day_phishing/base64 | |
| move_if_exists output/nrd-phishing-30day_base64.txt lists/30-day_phishing/base64 | |
| move_if_exists output/nrd-phishing-14day_unbound_part1.txt lists/14-day_phishing/unbound | |
| move_if_exists output/nrd-phishing-14day_unbound_part2.txt lists/14-day_phishing/unbound | |
| move_if_exists output/nrd-phishing-30day_unbound_part1.txt lists/30-day_phishing/unbound | |
| move_if_exists output/nrd-phishing-30day_unbound_part2.txt lists/30-day_phishing/unbound | |
| move_if_exists output/nrd-phishing-30day_unbound_part3.txt lists/30-day_phishing/unbound | |
| move_if_exists output/nrd-phishing-14day_wildcard.txt lists/14-day_phishing/wildcard | |
| move_if_exists output/nrd-phishing-30day_wildcard.txt lists/30-day_phishing/wildcard | |
| - name: Run statistics script | |
| if: ${{ env.changed == 'true' }} | |
| run: python3 src/generate-stats.py | |
| - name: Move stats image | |
| if: ${{ env.changed == 'true' }} | |
| run: mkdir -p img && mv stats.png img/ | |
| - name: Commit and push updates | |
| if: ${{ env.changed == 'true' }} | |
| run: | | |
| git config --global user.name "GitHub Actions" | |
| git config --global user.email "actions@github.com" | |
| git add lists img/stats.png hash/current_*.txt | |
| git commit -m "Update NRD Lists and Stats ($(date +'%Y-%m-%d'))" | |
| git push | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Save hashes for next run | |
| if: ${{ env.changed == 'true' }} | |
| run: | | |
| for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do | |
| mv hash/current_$file.txt hash/previous_$file.txt | |
| done | |
| git add hash/previous_*.txt | |
| git commit -m "Save hashes for next run" | |
| git push | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |