forked from antonme/ipranges
-
Notifications
You must be signed in to change notification settings - Fork 1
180 lines (153 loc) · 7.52 KB
/
update.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
name: "Update"
on:
workflow_dispatch:
schedule:
- cron: '30 19 * * *'
jobs:
updater:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Settings DoT DNS with systemd-resolved
run: |
sudo apt update
sudo apt install -y bind9-dnsutils sudo --no-install-suggests
sudo mkdir -pv /etc/systemd/resolved.conf.d/
echo -e "[Resolve]\nDNS=8.8.4.4#dns.google\nFallbackDNS=\nDNSOverTLS=yes\nDNSSEC=yes\nCache=yes\nCacheFromLocalhost=yes\nReadEtcHosts=no" | sudo tee /etc/systemd/resolved.conf.d/google.conf &>/dev/null;
echo -e "[Resolve]\nDNS=1.0.0.1#1dot1dot1dot1.cloudflare-dns.com\nFallbackDNS=\nDNSOverTLS=yes\nDNSSEC=yes\nCache=yes\nCacheFromLocalhost=yes\nReadEtcHosts=no" | sudo tee /etc/systemd/resolved.conf.d/cloudflare.conf &>/dev/null;
cat /etc/systemd/resolved.conf.d/google.conf
echo ""
cat /etc/systemd/resolved.conf.d/cloudflare.conf
echo ""
echo "nameserver 127.0.0.53" | sudo tee /etc/resolv.conf.DNSoverride &>/dev/null;
sudo ln -svf /etc/resolv.conf.DNSoverride /etc/resolv.conf
cat -n /etc/resolv.conf
echo ""
cat -n /etc/hosts
echo ""
sudo systemctl restart systemd-resolved.service
sleep 3s
nc -z -v 127.0.0.53 53 && echo ""
- name: Prepare packages
run: sudo apt-get install -y zstd zip grepcidr libarchive-tools whois dos2unix moreutils jq ipcalc ripgrep gawk parallel bind9-dnsutils language-pack-ru language-pack-en locales && sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && sudo locale-gen --no-purge en_US.UTF-8
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Cache pip
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('utils/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Install dependencies
run: |
pip install -r utils/requirements.txt
- name: Download IPs
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
set -euo pipefail
set -x
find . -name downloader.sh| sort -h | awk '{print "Executing "$1"...";system("bash "$1)}'
# - name: Merge ipv4 Ranges
# run: |
# set -euo pipefail
# set -x
# find . -name ipv4.txt| sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -h '>' {.}_merged.txt
# - name: Merge ipv6 Ranges
# run: |
# set -euo pipefail
# set -x
# find . -name ipv6.txt| sort -h | parallel --will-cite -j 1 echo "Merging '{}'"';'python utils/merge.py --source={} '|' sort -h '>' {.}_merged.txt
- name: Merge ipv4 Ranges Mat1RX script
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
set -euo pipefail
set -x
find . -type f -name ipv4.txt| sort -h | parallel --will-cite -j $(nproc) echo "Merging '{}'"';'python utils/merge_Mat1RX.py -c 1000000 --source={} '|' sort -h '>' {.}_merged.txt
- name: Merge ipv6 Ranges Mat1RX script
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
set -euo pipefail
set -x
find . -type f -name ipv6.txt| sort -h | parallel --will-cite -j $(nproc) echo "Merging '{}'"';'python utils/merge_Mat1RX.py -c 1000000 --source={} '|' sort -h '>' {.}_merged.txt
rm -fv youtube/fullnets.txt youtube/nets.txt vpnhosts/vpnhosts-knownips.txt
- name: Fix large file copy ipv4_merged.txt as ipv4.txt for Github fix
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
set -euo pipefail
set -x
find . -name copy_as.sh| sort -h | parallel --will-cite -j $(nproc) echo "script execution '{}'"';' bash {.}.sh
- name: Protection against empty files. If one such file is found, no changes will be committed to the repository. Finishing with code 1. Error code for the repository author.
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
for filename in $(find . -type f -name "ipv4*.txt" -print | sort -t . | uniq); do if [[ -s ${filename} ]]; then echo "No empty $filename";else echo -e "\e[1;33mEmpty\033[0m $filename" && export EMPTY=1; fi; done
if [[ $EMPTY == 1 ]]; then exit 1; else echo "OK!"; fi
- name: Commit files
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Auto-update ip ranges"
file_pattern: '*/*.txt */*/*.txt'
commit_options: "--no-verify"
repository: .
commit_user_name: github-actions[bot]
commit_user_email: github-actions[bot]@users.noreply.github.com
commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
- name: Delete temp files in tmp directory and delete all archives in repository
run: |
set -euo pipefail
find /tmp/ -maxdepth 1 -type f -name "*.txt" -delete && find . -type f -name "*.txt.zst" -delete
- name: Downloading iprange and unmerge ipranges
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
find . -name unmerging.sh| sort -h | parallel --will-cite -j $(nproc) echo "Script running '{}'"';'bash {}
- name: Commit files
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Auto-update IPs unmerged"
file_pattern: '*/*.txt.zst */*/*.txt.zst'
commit_options: "--no-verify"
repository: .
commit_user_name: github-actions[bot]
commit_user_email: github-actions[bot]@users.noreply.github.com
commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
- name: Merge ipv4 Ranges smart with utils ip2net
env:
THRESHOLDv4: 1/4
IP2NET_PREFIX_LENGTH: 16-26
run: |
find . -type f -name ipv4.txt.zst| sort -h | parallel --will-cite -j $(nproc) echo "Merging Smart list '{}'"';'zstdcat {} '|' utils/ip2net --v4-threshold=${THRESHOLDv4} --prefix-length=${IP2NET_PREFIX_LENGTH} '|' sort -h '>' {.}_smart.txt
- name: Fix naming files smart list
run: |
find . -type f -name "*.txt_smart.txt" -exec bash -c 'mv -v "$0" "${0%.txt_smart.txt}_smart.txt"' {} \;
- name: Protection against empty files (smart list). If one such file is found, no changes will be committed to the repository. Finishing with code 1. Error code for the repository author.
env:
LANG: en_US.UTF-8
CHARSET: UTF-8
run: |
for filename in $(find . -type f -name "ipv4_smart.txt" -print | sort -t . | uniq); do if [[ -s ${filename} ]]; then echo "No empty $filename";else echo -e "\e[1;33mEmpty\033[0m $filename" && export EMPTY=1; fi; done
if [[ $EMPTY == 1 ]]; then exit 1; else echo "OK!"; fi
- name: Commit files smart list
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Auto-update IPs smart list"
file_pattern: '*/*smart.txt */*/*smart.txt'
commit_options: "--no-verify"
repository: .
commit_user_name: github-actions[bot]
commit_user_email: github-actions[bot]@users.noreply.github.com
commit_author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>