Extract IPs from URL #1155
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extract IPs from URL | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "*/30 * * * *" | |
| jobs: | |
| extract-ips: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Download source file | |
| run: curl -s -o trackers_all_ip.txt https://raw.githubusercontent.com/ngosang/trackerslist/refs/heads/master/trackers_all_ip.txt | |
| - name: Download source file with trackers 1 | |
| run: curl -s -o trackers_all_from_src1.txt https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt | |
| - name: Download source file with trackers 2 | |
| run: curl -s -o trackers_all_from_src2.txt https://cf.trackerslist.com/all.txt | |
| - name: Concatenate custom hosts | |
| run: | | |
| cat custom_hosts.txt trackers_all_from_src1.txt trackers_all_from_src2.txt > trackers_all.txt | |
| echo "Combined trackers_all.txt with custom_hosts.txt" | |
| cat trackers_all.txt | |
| - name: Extract IP addresses from the IP file | |
| run: | | |
| grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' trackers_all_ip.txt >> extracted_ips_raw.txt | |
| echo "Extracted IPs from file:" | |
| cat extracted_ips_raw.txt | |
| - name: Extract IP addresses from hosts in trackers_all.txt | |
| run: | | |
| echo "Extracting IPs from the host list..." | |
| grep -oP '(?<=://)[^:/]+' trackers_all.txt > hostnames.txt | |
| while IFS= read -r host; do | |
| ips=$(dig +short $host) | |
| if [ -n "$ips" ]; then | |
| echo "Resolved $host to the following IPs:" | |
| for ip in $ips; do | |
| if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then | |
| echo "$ip" | |
| echo "$ip" >> host_to_ips.txt | |
| else | |
| echo "Skipping non-IP value: $ip" | |
| fi | |
| done | |
| else | |
| echo "Failed to resolve $host" | |
| fi | |
| done < hostnames.txt | |
| - name: Combine IPs | |
| run: | | |
| cat extracted_ips_raw.txt host_to_ips.txt > extracted_ips_unfiltered.txt | |
| echo "Combined unfiltered unique IPs:" | |
| cat extracted_ips_unfiltered.txt | |
| - name: Install grepcidr | |
| run: sudo apt-get update && sudo apt-get install -y grepcidr | |
| - name: Filter Private and Cloudflare IPs | |
| run: | | |
| # Download Cloudflare IP ranges directly | |
| curl -s https://www.cloudflare.com/ips-v4/ > cloudflare_ips.txt | |
| # First filter out private IPs | |
| grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|169\.254\.|224\.|239\.|240\.|255\.)' extracted_ips_unfiltered.txt > temp_filtered_ips.txt | |
| # Create a temporary file for IPs to keep | |
| touch keep_ips.txt | |
| # Process each IP against Cloudflare ranges | |
| while IFS= read -r ip; do | |
| is_cloudflare=false | |
| while IFS= read -r cf_range; do | |
| if grepcidr "$cf_range" <(echo "$ip") >/dev/null 2>&1; then | |
| is_cloudflare=true | |
| echo "Filtered out Cloudflare IP: $ip (matched $cf_range)" | |
| break | |
| fi | |
| done < cloudflare_ips.txt | |
| if [ "$is_cloudflare" = false ]; then | |
| echo "$ip" >> keep_ips.txt | |
| fi | |
| done < temp_filtered_ips.txt | |
| # Move kept IPs to final file | |
| mv keep_ips.txt extracted_ips.txt | |
| rm temp_filtered_ips.txt | |
| echo "Final filtered IPs:" | |
| cat extracted_ips.txt | |
| - name: Ensure uniqueness of extracted IPs | |
| run: | | |
| sort -u extracted_ips.txt -o extracted_ips.txt | |
| echo "Unique extracted IPs:" | |
| cat extracted_ips.txt | |
| - name: Commit and push changes | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git add extracted_ips.txt | |
| git commit -m "Update extracted IPs" || (echo "No changes to commit" && exit 0) | |
| git push |