Update #526
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Update" | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '30 19 * * *' | |
| jobs: | |
| updater: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Settings DoT DNS with systemd-resolved | |
| run: | | |
| sudo apt update | |
| sudo apt install -y bind9-dnsutils sudo --no-install-suggests | |
| sudo mkdir -pv /etc/systemd/resolved.conf.d/ | |
| echo -e "[Resolve]\nDNS=8.8.4.4#dns.google\nFallbackDNS=\nDNSOverTLS=yes\nDNSSEC=yes\nCache=yes\nCacheFromLocalhost=yes\nReadEtcHosts=no" | sudo tee /etc/systemd/resolved.conf.d/google.conf &>/dev/null; | |
| echo -e "[Resolve]\nDNS=1.0.0.1#1dot1dot1dot1.cloudflare-dns.com\nFallbackDNS=\nDNSOverTLS=yes\nDNSSEC=yes\nCache=yes\nCacheFromLocalhost=yes\nReadEtcHosts=no" | sudo tee /etc/systemd/resolved.conf.d/cloudflare.conf &>/dev/null; | |
| cat /etc/systemd/resolved.conf.d/google.conf | |
| echo "" | |
| cat /etc/systemd/resolved.conf.d/cloudflare.conf | |
| echo "" | |
| echo "nameserver 127.0.0.53" | sudo tee /etc/resolv.conf.DNSoverride &>/dev/null; | |
| sudo ln -svf /etc/resolv.conf.DNSoverride /etc/resolv.conf | |
| cat -n /etc/resolv.conf | |
| echo "" | |
| cat -n /etc/hosts | |
| echo "" | |
| sudo systemctl restart systemd-resolved.service | |
| sleep 3s | |
| nc -z -v 127.0.0.53 53 && echo "" | |
| - name: Prepare packages | |
| run: sudo apt-get install -y zstd zip grepcidr libarchive-tools whois dos2unix moreutils jq ipcalc ripgrep gawk parallel bind9-dnsutils language-pack-ru language-pack-en locales && sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && sudo locale-gen --no-purge en_US.UTF-8 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('utils/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| ${{ runner.os }}- | |
| - name: Install dependencies | |
| run: | | |
| pip install -r utils/requirements.txt | |
| - name: Download IPs | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| find . -name downloader.sh| sort -h | awk '{print "Executing "$1"...";system("bash "$1)}' | |
| # - name: Merge ipv4 Ranges | |
| # run: | | |
| # set -euo pipefail | |
| # set -x | |
| # find . -name ipv4.txt| sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -h '>' {.}_merged.txt | |
| # - name: Merge ipv6 Ranges | |
| # run: | | |
| # set -euo pipefail | |
| # set -x | |
| # find . -name ipv6.txt| sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -h '>' {.}_merged.txt | |
| - name: Merge ipv4 Ranges Mat1RX script | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| find . -type f -name ipv4.txt| sort -h | parallel --will-cite -j $(nproc) echo "Merging '{}'"';'python utils/merge_Mat1RX.py -c 1000000 --source={} '|' sort -h '>' {.}_merged.txt | |
| - name: Merge ipv6 Ranges Mat1RX script | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| find . -type f -name ipv6.txt| sort -h | parallel --will-cite -j $(nproc) echo "Merging '{}'"';'python utils/merge_Mat1RX.py -c 1000000 --source={} '|' sort -h '>' {.}_merged.txt | |
| rm -fv youtube/fullnets.txt youtube/nets.txt vpnhosts/vpnhosts-knownips.txt | |
| - name: Fix large file copy ipv4_merged.txt as ipv4.txt for Github fix | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| find . -name copy_as.sh| sort -h | parallel --will-cite -j $(nproc) echo "script execution '{}'"';' bash {.}.sh | |
| - name: Protection against empty files. If one such file is found, no changes will be committed to the repository. Finishing with code 1. Error code for the repository author. | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| for filename in $(find . -type f -name "ipv4*.txt" -print | sort -t . | uniq); do if [[ -s ${filename} ]]; then echo "No empty $filename";else echo -e "\e[1;33mEmpty\033[0m $filename" && export EMPTY=1; fi; done | |
| if [[ $EMPTY == 1 ]]; then exit 1; else echo "OK!"; fi | |
| - name: Commit files | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "Auto-update ip ranges" | |
| file_pattern: '*/*.txt */*/*.txt' | |
| commit_options: "--no-verify" | |
| repository: . | |
| commit_user_name: github-actions[bot] | |
| commit_user_email: github-actions[bot]@users.noreply.github.com | |
| commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com> | |
| - name: Delete temp files in tmp directory and delete all archives in repository | |
| run: | | |
| set -euo pipefail | |
| find /tmp/ -maxdepth 1 -type f -name "*.txt" -delete && find . -type f -name "*.txt.zst" -delete | |
| - name: Downloading iprange and unmerge ipranges | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| find . -name unmerging.sh| sort -h | parallel --will-cite -j $(nproc) echo "Script running '{}'"';'bash {} | |
| - name: Commit files | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "Auto-update IPs unmerged" | |
| file_pattern: '*/*.txt.zst */*/*.txt.zst' | |
| commit_options: "--no-verify" | |
| repository: . | |
| commit_user_name: github-actions[bot] | |
| commit_user_email: github-actions[bot]@users.noreply.github.com | |
| commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com> | |
| - name: Merge ipv4 Ranges Smart utils ip2net | |
| env: | |
| THRESHOLDv4: 1/4 | |
| IP2NET_PREFIX_LENGTH: 16-26 | |
| run: | | |
| find . -type f -name ipv4.txt.zst| sort -h | parallel --will-cite -j $(nproc) echo "Merging Smart list '{}'"';'zstdcat {} '|' utils/ip2net --v4-threshold=${THRESHOLDv4} --prefix-length=${IP2NET_PREFIX_LENGTH} '|' sort -h '>' {.}_smart.txt | |
| - name: Fix naming files | |
| run: | | |
| find . -type f -name "*.txt_smart.txt" -exec bash -c 'mv -v "$0" "${0%.txt_smart.txt}_smart.txt"' {} \; | |
| - name: Protection against empty files. If one such file is found, no changes will be committed to the repository. Finishing with code 1. Error code for the repository author. | |
| env: | |
| LANG: en_US.UTF-8 | |
| CHARSET: UTF-8 | |
| run: | | |
| for filename in $(find . -type f -name "ipv4_smart.txt" -print | sort -t . | uniq); do if [[ -s ${filename} ]]; then echo "No empty $filename";else echo -e "\e[1;33mEmpty\033[0m $filename" && export EMPTY=1; fi; done | |
| if [[ $EMPTY == 1 ]]; then exit 1; else echo "OK!"; fi | |
| - name: Commit files smart list | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "Auto-update IPs smart list" | |
| file_pattern: '*/*smart.txt */*/*smart.txt' | |
| commit_options: "--no-verify" | |
| repository: . | |
| commit_user_name: github-actions[bot] | |
| commit_user_email: github-actions[bot]@users.noreply.github.com | |
| commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com> |