auto-updater.yml

name: Extract IPs from URL

on:
  push:
    branches:
      - main
  workflow_dispatch:
  schedule:
    - cron: "*/30 * * * *"

jobs:
  extract-ips:
    runs-on: ubuntu-latest

    steps:
      # Previous steps remain the same until the Filter Private IPs step
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Download source file
        run: curl -s -o trackers_all_ip.txt https://raw.githubusercontent.com/ngosang/trackerslist/refs/heads/master/trackers_all_ip.txt

      - name: Download source file with trackers 1
        run: curl -s -o trackers_all_from_src1.txt https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt

      - name: Download source file with trackers 2
        run: curl -s -o trackers_all_from_src2.txt https://cf.trackerslist.com/all.txt

      - name: Concatenate custom hosts
        run: |
          cat custom_hosts.txt trackers_all_from_src1.txt trackers_all_from_src2.txt > trackers_all.txt
          echo "Combined trackers_all.txt with custom_hosts.txt"
          cat trackers_all.txt

      - name: Extract IP addresses from the IP file
        run: |
          grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' trackers_all_ip.txt >> extracted_ips_raw.txt
          echo "Extracted IPs from file:"
          cat extracted_ips_raw.txt

      - name: Extract IP addresses from hosts in trackers_all.txt
        run: |
          echo "Extracting IPs from the host list..."
          grep -oP '(?<=://)[^:/]+' trackers_all.txt > hostnames.txt

          while IFS= read -r host; do
            ips=$(dig +short $host)
            if [ -n "$ips" ]; then
              echo "Resolved $host to the following IPs:"
              for ip in $ips; do
                if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
                  echo "$ip"
                  echo "$ip" >> host_to_ips.txt
                else
                  echo "Skipping non-IP value: $ip"
                fi
              done
            else
              echo "Failed to resolve $host"
            fi
          done < hostnames.txt

      - name: Combine IPs
        run: |
          cat extracted_ips_raw.txt host_to_ips.txt > extracted_ips_unfiltered.txt
          echo "Combined unfiltered unique IPs:"
          cat extracted_ips_unfiltered.txt

      - name: Download and prepare Cloudflare IP ranges
        run: |
          # Download Cloudflare IPv4 ranges
          curl -s https://www.cloudflare.com/ips-v4/ > cloudflare_ips.txt
          # Clean up the file to ensure one CIDR per line
          sed -i 's/<[^>]*>//g' cloudflare_ips.txt
          sed -i 's/^[[:space:]]*//g' cloudflare_ips.txt
          sed -i 's/[[:space:]]*$//g' cloudflare_ips.txt
          sed -i '/^$/d' cloudflare_ips.txt
          echo "Downloaded Cloudflare IP ranges:"
          cat cloudflare_ips.txt

      - name: Install ipcalc
        run: sudo apt-get update && sudo apt-get install -y ipcalc

      - name: Filter Private and Cloudflare IPs
        run: |
          # First filter out private IPs
          grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|169\.254\.|224\.|239\.|240\.|255\.)' extracted_ips_unfiltered.txt > temp_ips.txt
          
          # Then filter out Cloudflare IPs
          while IFS= read -r cf_range; do
            # Skip empty lines or invalid CIDR notation
            [[ -z "$cf_range" ]] && continue
            [[ ! "$cf_range" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$ ]] && continue
            
            # Convert CIDR to network and broadcast IPs for comparison
            network=$(ipcalc "$cf_range" | grep "Network:" | awk '{print $2}')
            broadcast=$(ipcalc "$cf_range" | grep "Broadcast:" | awk '{print $2}')
            
            # Convert IPs to numbers for comparison
            network_num=$(echo "$network" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}')
            broadcast_num=$(echo "$broadcast" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}')
            
            # Filter out IPs in the Cloudflare range
            while IFS= read -r ip; do
              ip_num=$(echo "$ip" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}')
              if [ "$ip_num" -lt "$network_num" ] || [ "$ip_num" -gt "$broadcast_num" ]; then
                echo "$ip" >> extracted_ips.txt
              else
                echo "Filtered out Cloudflare IP: $ip"
              fi
            done < temp_ips.txt
            > temp_ips.txt  # Clear the temporary file for the next iteration
          done < cloudflare_ips.txt
          
          rm temp_ips.txt
          echo "Filtered IPs:"
          cat extracted_ips.txt

      - name: Ensure uniqueness of extracted IPs
        run: |
          sort -u extracted_ips.txt -o extracted_ips.txt
          echo "Unique extracted IPs:"
          cat extracted_ips.txt

      - name: Commit and push changes
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git add extracted_ips.txt
          git commit -m "Update extracted IPs" || echo "No changes to commit" && exit 0
          git push