Build V2Ray rules dat files #1169
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build V2Ray rules dat files | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 22 * * *" | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "**/README.md" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set variables | |
| run: | | |
| echo "RELEASE_NAME=Released on $(date +%Y%m%d%H%M)" >> $GITHUB_ENV | |
| echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV | |
| echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf" >> $GITHUB_ENV | |
| echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV | |
| echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV | |
| echo "EASYLISTCHINA_EASYLIST_REJECT_URL=https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt" >> $GITHUB_ENV | |
| echo "PETERLOWE_REJECT_URL=https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext" >> $GITHUB_ENV | |
| echo "ADGUARD_DNS_REJECT_URL=https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" >> $GITHUB_ENV | |
| echo "DANPOLLOCK_REJECT_URL=https://someonewhocares.org/hosts/hosts" >> $GITHUB_ENV | |
| echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/cn.txt" >> $GITHUB_ENV | |
| echo "CUSTOM_PROXY=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV | |
| echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV | |
| echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV | |
| echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Checkout the "hidden" branch of this repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: hidden | |
| - name: Checkout Loyalsoldier/domain-list-custom | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Loyalsoldier/domain-list-custom | |
| path: custom | |
| - name: Checkout v2fly/domain-list-community | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: v2fly/domain-list-community | |
| path: community | |
| - name: Checkout cokebar/gfwlist2dnsmasq | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: cokebar/gfwlist2dnsmasq | |
| path: gfwlist2dnsmasq | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./custom/go.mod | |
| cache-dependency-path: ./custom/go.sum | |
| - name: Get geoip.dat relative files | |
| run: | | |
| wget https://github.com/leemars/geoip/raw/release/geoip.dat | |
| wget https://github.com/leemars/geoip/raw/release/geoip.dat.sha256sum | |
| - name: Generate GFWList domains | |
| run: | | |
| cd gfwlist2dnsmasq || exit 1 | |
| chmod +x ./gfwlist2dnsmasq.sh | |
| ./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt | |
| - name: Get and add direct domains into temp-direct.txt file | |
| run: | | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt | |
| curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-direct.txt | |
| - name: Get and add proxy domains into temp-proxy.txt file | |
| run: | | |
| cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-proxy.txt | |
| - name: Get and add reject domains into temp-reject.txt file | |
| run: | | |
| curl -sSL $EASYLISTCHINA_EASYLIST_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' > temp-reject.txt | |
| curl -sSL $ADGUARD_DNS_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' >> temp-reject.txt | |
| curl -sSL $PETERLOWE_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})$/ && print "$1\n"' >> temp-reject.txt | |
| curl -sSL $DANPOLLOCK_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})/ && print "$1\n"' | sed '1d' >> temp-reject.txt | |
| - name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files | |
| run: | | |
| curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt | |
| curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt | |
| - name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files | |
| run: | | |
| cat proxy.txt >> temp-proxy.txt | |
| cat direct.txt >> temp-direct.txt | |
| cat reject.txt >> temp-reject.txt | |
| - name: Sort and generate redundant lists | |
| run: | | |
| cat temp-proxy.txt | sort --ignore-case -u > proxy-list-with-redundant | |
| cat temp-direct.txt | sort --ignore-case -u > direct-list-with-redundant | |
| cat temp-reject.txt | sort --ignore-case -u > reject-list-with-redundant | |
| - name: Remove redundant domains | |
| run: | | |
| chmod +x findRedundantDomain.py | |
| ./findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort | |
| ./findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort | |
| ./findRedundantDomain.py ./reject-list-with-redundant ./reject-list-deleted-unsort | |
| [ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort | |
| [ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort | |
| [ ! -f "reject-list-deleted-unsort" ] && touch reject-list-deleted-unsort | |
| sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort | |
| sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort | |
| sort ./reject-list-deleted-unsort > ./reject-list-deleted-sort | |
| diff ./direct-list-deleted-sort ./direct-list-with-redundant | awk '/^>/{print $2}' > ./direct-list-without-redundant | |
| diff ./proxy-list-deleted-sort ./proxy-list-with-redundant | awk '/^>/{print $2}' > ./proxy-list-without-redundant | |
| diff ./reject-list-deleted-sort ./reject-list-with-redundant | awk '/^>/{print $2}' > ./reject-list-without-redundant | |
| - name: Remove domains from "need-to-remove" lists in "hidden" branch | |
| run: | | |
| diff ./direct-need-to-remove.txt ./direct-list-without-redundant | awk '/^>/{print $2}' > temp-cn.txt | |
| diff ./proxy-need-to-remove.txt ./proxy-list-without-redundant | awk '/^>/{print $2}' > temp-geolocation-\!cn.txt | |
| diff ./reject-need-to-remove.txt ./reject-list-without-redundant | awk '/^>/{print $2}' > temp-category-ads-all.txt | |
| - name: Remove domains end with ".cn" in "temp-geolocation-!cn.txt" and write lists to data directory | |
| run: | | |
| cat temp-cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn | |
| cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt | |
| cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' | perl -ne 'print if not /\.cn$/' > ./community/data/geolocation-\!cn | |
| cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt | |
| cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/category-ads-all | |
| cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > reject-tld-list.txt | |
| - name: Add `full`, `regexp` and `keyword` type of rules back into "cn", "geolocation-!cn" and "category-ads-all" list | |
| run: | | |
| [ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn | |
| [ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn | |
| [ -f "reject-reserve.txt" ] && cat reject-reserve.txt >> ./community/data/category-ads-all | |
| cp ./community/data/cn direct-list.txt | |
| cp ./community/data/geolocation-\!cn proxy-list.txt | |
| cp ./community/data/category-ads-all reject-list.txt | |
| - name: Create `china-list`、`google-cn`、`apple-cn`、`gfw`、`greatfire` lists | |
| run: | | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > ./community/data/china-list | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > china-list.txt | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > google-cn.txt | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > apple-cn.txt | |
| cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw | |
| cat ./community/data/gfw | sort --ignore-case -u > gfw.txt | |
| cat ./community/data/greatfire | sort --ignore-case -u > greatfire.txt | |
| curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-spy | |
| curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > win-spy.txt | |
| curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-update | |
| curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > win-update.txt | |
| curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-extra | |
| curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > win-extra.txt | |
| - name: Build geosite.dat file | |
| run: | | |
| cd custom || exit 1 | |
| go run ./ --datapath=../community/data | |
| - name: Move and zip files and generate sha256 hash | |
| run: | | |
| install -Dp ./geoip.dat ./publish/geoip.dat | |
| install -Dp ./geoip.dat.sha256sum ./publish/geoip.dat.sha256sum | |
| install -Dp ./custom/publish/geosite.dat ./publish/geosite.dat | |
| install -p {proxy,direct,reject}-tld-list.txt ./publish/ | |
| install -p {proxy,direct,reject}-list.txt ./publish/ | |
| install -p {china-list,apple-cn,google-cn,gfw,greatfire,win-spy,win-update,win-extra}.txt ./publish/ | |
| cd ./publish || exit 1 | |
| zip rules.zip {proxy,direct,reject}-list.txt geoip.dat geosite.dat | |
| sha256sum geosite.dat > geosite.dat.sha256sum | |
| sha256sum rules.zip > rules.zip.sha256sum | |
| - name: Release and upload assets | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: ${{ env.RELEASE_NAME }} | |
| tag: ${{ env.TAG_NAME }} | |
| file_glob: true | |
| file: ./publish/* | |
| - name: Git push assets to "release" branch | |
| run: | | |
| cd publish || exit 1 | |
| git init | |
| git config --local user.name "github-actions[bot]" | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git checkout -b release | |
| git add . | |
| git commit -m "${{ env.RELEASE_NAME }}" | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f -u origin release | |
| - name: Purge jsdelivr CDN | |
| run: | | |
| cd publish || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}" | |
| done |