Skip to content

Update

Update #411

Workflow file for this run

name: Update
on:
schedule:
- cron: '10 1,3,5,7,9,11,13,15,17,19,21,23 * * *'
workflow_dispatch:
jobs:
run-script:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Install dependencies
run: pip install --upgrade pip requests matplotlib
- name: Download files
run: |
mkdir -p downloads
curl -o downloads/nordomain_30day.txt ${{ secrets.NORDOMAIN_30DAY_URL }}
curl -o downloads/nordomain_14day.txt ${{ secrets.NORDOMAIN_14DAY_URL }}
curl -o downloads/phishing_30day.txt ${{ secrets.PHISHING_30DAY_URL }}
curl -o downloads/phishing_14day.txt ${{ secrets.PHISHING_14DAY_URL }}
curl -A "${{ secrets.USER_AGENT }}" -o downloads/additional.txt "${{ secrets.ADDITIONAL_SOURCE_URL }}"
- name: Compute file hashes
id: compute_hashes
run: |
mkdir -p hash
for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do
sha256sum downloads/$file.txt > hash/current_$file.txt
done
- name: Check if files have changed
id: check_changes
run: |
CHANGED=false
for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do
if [ ! -f hash/previous_$file.txt ] || ! cmp -s hash/previous_$file.txt hash/current_$file.txt; then
echo "$file has changed."
CHANGED=true
fi
done
echo "changed=$CHANGED" >> $GITHUB_ENV
- name: Skip processing if unchanged
if: ${{ env.changed == 'false' }}
run: echo "Skipping further steps as no files have changed."
- name: Run main script
if: ${{ env.changed == 'true' }}
env:
NORDOMAIN_30DAY_URL: ${{ secrets.NORDOMAIN_30DAY_URL }}
NORDOMAIN_14DAY_URL: ${{ secrets.NORDOMAIN_14DAY_URL }}
PHISHING_30DAY_URL: ${{ secrets.PHISHING_30DAY_URL }}
PHISHING_14DAY_URL: ${{ secrets.PHISHING_14DAY_URL }}
ADDITIONAL_SOURCE_URL: ${{ secrets.ADDITIONAL_SOURCE_URL }}
USER_AGENT: ${{ secrets.USER_AGENT }}
run: python3 src/nrd-hunter.py
- name: Move output files
if: ${{ env.changed == 'true' }}
run: |
# Create necessary directories for organized output
mkdir -p \
lists/14-day/domains-only \
lists/30-day/domains-only \
lists/14-day_phishing/domains-only \
lists/30-day_phishing/domains-only \
lists/14-day/adblock \
lists/30-day/adblock \
lists/14-day_phishing/adblock \
lists/30-day_phishing/adblock \
lists/14-day/wildcard \
lists/30-day/wildcard \
lists/14-day_phishing/wildcard \
lists/30-day_phishing/wildcard \
lists/14-day/base64 \
lists/30-day/base64 \
lists/14-day_phishing/base64 \
lists/30-day_phishing/base64 \
lists/14-day/unbound \
lists/30-day/unbound \
lists/14-day_phishing/unbound \
lists/30-day_phishing/unbound
# Helper function to safely move files
move_if_exists() {
src=$1
dest=$2
if [ -f "$src" ]; then
mv "$src" "$dest"
else
echo "Warning: File $src does not exist."
fi
}
move_if_exists output/nrd-14day_wildcard.txt lists/14-day/wildcard
move_if_exists output/nrd-14day.txt lists/14-day/domains-only
move_if_exists output/nrd-30day_part1.txt lists/30-day/domains-only
move_if_exists output/nrd-30day_part2.txt lists/30-day/domains-only
move_if_exists output/nrd-14day_adblock.txt lists/14-day/adblock
move_if_exists output/nrd-30day_adblock_part1.txt lists/30-day/adblock
move_if_exists output/nrd-30day_adblock_part2.txt lists/30-day/adblock
move_if_exists output/nrd-14day_base64.txt lists/14-day/base64
move_if_exists output/nrd-30day_base64_part1.txt lists/30-day/base64
move_if_exists output/nrd-30day_base64_part2.txt lists/30-day/base64
move_if_exists output/nrd-14day_unbound_part1.txt lists/14-day/unbound
move_if_exists output/nrd-14day_unbound_part2.txt lists/14-day/unbound
move_if_exists output/nrd-30day_unbound_part1.txt lists/30-day/unbound
move_if_exists output/nrd-30day_unbound_part2.txt lists/30-day/unbound
move_if_exists output/nrd-30day_unbound_part3.txt lists/30-day/unbound
move_if_exists output/nrd-30day_wildcard_part1.txt lists/30-day/wildcard
move_if_exists output/nrd-30day_wildcard_part2.txt lists/30-day/wildcard
move_if_exists output/nrd-phishing-14day.txt lists/14-day_phishing/domains-only
move_if_exists output/nrd-phishing-30day.txt lists/30-day_phishing/domains-only
move_if_exists output/nrd-phishing-14day_adblock.txt lists/14-day_phishing/adblock
move_if_exists output/nrd-phishing-30day_adblock.txt lists/30-day_phishing/adblock
move_if_exists output/nrd-phishing-14day_base64.txt lists/14-day_phishing/base64
move_if_exists output/nrd-phishing-30day_base64.txt lists/30-day_phishing/base64
move_if_exists output/nrd-phishing-14day_unbound_part1.txt lists/14-day_phishing/unbound
move_if_exists output/nrd-phishing-14day_unbound_part2.txt lists/14-day_phishing/unbound
move_if_exists output/nrd-phishing-30day_unbound_part1.txt lists/30-day_phishing/unbound
move_if_exists output/nrd-phishing-30day_unbound_part2.txt lists/30-day_phishing/unbound
move_if_exists output/nrd-phishing-30day_unbound_part3.txt lists/30-day_phishing/unbound
move_if_exists output/nrd-phishing-14day_wildcard.txt lists/14-day_phishing/wildcard
move_if_exists output/nrd-phishing-30day_wildcard.txt lists/30-day_phishing/wildcard
- name: Run statistics script
if: ${{ env.changed == 'true' }}
run: python3 src/generate-stats.py
- name: Move stats image
if: ${{ env.changed == 'true' }}
run: mkdir -p img && mv stats.png img/
- name: Commit and push updates
if: ${{ env.changed == 'true' }}
run: |
git config --global user.name "GitHub Actions"
git config --global user.email "actions@github.com"
git add lists img/stats.png hash/current_*.txt
git commit -m "Update NRD Lists and Stats ($(date +'%Y-%m-%d'))"
git push
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Save hashes for next run
if: ${{ env.changed == 'true' }}
run: |
for file in nordomain_30day nordomain_14day phishing_30day phishing_14day additional; do
mv hash/current_$file.txt hash/previous_$file.txt
done
git add hash/previous_*.txt
git commit -m "Save hashes for next run"
git push
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}