up

dvershinin · Sep 9, 2020 · 7249ff9 · 7249ff9
1 parent 437cf98
commit 7249ff9
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 171 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -14,6 +14,7 @@ Complete walkthrough:
 * Create a virtualenv with system's Python version and access to system-wide packages:   
 `sudo virtualenv  --system-site-packages /opt/fds/venv`
 
+
 sudo mkdir -p /var/cache/fds /var/lib/fds
 
 

diff --git a/README.md b/README.md
@@ -4,39 +4,38 @@ The go-to **F**irewallD CLI app that **d**oesn't **s**uck.
 
 ### What is `fds`?
 
- Firewall management is often a task that you do once at the time of setting up a server.
- FirewallD is so great with the concepts of zones, source, support for IP sets.
-
- But if you're maintaining a server like a PRO, you are monitoring logs, and blocking malicious users as they come, on a *regular basis*.
-
- Blocking and managing blocked IP addresses, is where, unfortunately, `firewall-cmd` is very awkward to use.
- And if you're using Cloudflare firewall, you also want to propagate your blocked IP addresses there for even better protection.
+Firewall management is often a task that you do once at the time of setting up a server.
+But if you're maintaining a server like a PRO, you are monitoring logs, and blocking malicious users as they come, on a *regular basis*.
+
+FirewallD is a great firewall software. It has the concepts of zones, sources, and supports IP sets. 
+However, its client app, `firewall-cmd` is far from user-friendly when it comes to blocking and managing blocked IP addresses.
+Furthermore, if you also use Cloudflare firewall, you also want to propagate your blocked IP addresses to it for best protection.
 
 `fds` is the CLI client for FirewallD/Cloudflare, that you'll love to use any day.
+It is an alternative, client for FirewallD.
+
+Use it for simple or complex banning tasks, instead of `firewall-cmd`.
 
-It is an alternative (to `firewall-cmd`) client for FirewallD.
+Look how simple things are with `fds`:
 
-It makes the task of managing your FirewallD easy and human-friendly :-)
+```bash
+fds block <country name>
+fds block 1.2.3.4
+```
 
-## BETA !!! HIGHLY NON-FUNCTIONAL
+It makes the task of managing your FirewallD easy and human-friendly.
 
-## Goals 
+## What `fds` can do 
 
 The `fds` is utility program for users of FirewallD. It is a helper to easily perform day-to-day 
 firewall tasks:
 
 * block countries
-* block ranges in the Cloudflare firewall (kinda helper firewall for cloudflare) 
-and works with both firewalld and cloudflare, e.g. http://jodies.de/ipcalc?host=114.119.128.0&mask1=18&mask2=24
-* declare a CDN of servers and push blocking commands across those server from one place (ansible-like), useful for dynamic blocking
-from the central server (honeypot)
-* drop outbound connections (shortcut to https://cogitantium.blogspot.com/2017/06/how-to-drop-outbound-connections-with.html) 
+* block arbitrary IP addresses
 
 # Synopsys
 
-### Works
-
-### Ban a single IP
+## Ban a single IP
 
 ```bash
 fds block 1.2.3.4
@@ -46,12 +45,27 @@ This blocks IP address in a proper(©) fashion by ensuring that the IP is in a s
 that the set is a source to FirewallD's `drop` zone. Using IP sets is the corner stone of consistent
 firewall management!
 
-### Planned
+## Ban a country
 
 ```bash
 fds block <Country Name>
+fds block China
 ```
 
+## Reset bans
+
+```bash
+fds reset
+```
+
+### Planned
+
+* block ranges in the Cloudflare firewall (kinda helper firewall for cloudflare) 
+and works with both firewalld and cloudflare, e.g. http://jodies.de/ipcalc?host=114.119.128.0&mask1=18&mask2=24
+* declare a CDN of servers and push blocking commands across those server from one place (ansible-like), useful for dynamic blocking
+from the central server (honeypot)
+* drop outbound connections (shortcut to https://cogitantium.blogspot.com/2017/06/how-to-drop-outbound-connections-with.html) 
+
 ## Installation on CentOS/RHEL 7, 8
 
 ```bash
@@ -63,6 +77,6 @@ See contributing guide for development setup (if not using packages).
 
 ## Files
 
-* /etc/fds.conf (info on currently blocked countries or otherwise small data sets suitable for a single config file)
-* /var/lib/fds: zone files, (state data) + (info on what is currently blocked) (???)
-* /var/cache/fds: cachecontrol cache
+* `/etc/fds.conf` (info on currently blocked countries or otherwise small data sets suitable for a single config file)
+* `/var/lib/fds`: zone files, (state data) + (info on what is currently blocked) (???)
+* `/var/cache/fds`: cachecontrol cache
diff --git a/cds/README.md b/cds/README.md
@@ -1,2 +1 @@
-`cds` is planned to be shipped with `fds` as a helper package for workign with Cloudflare.
-For now it is non-function and a copy of another project with similiar code base.
+`cds` is planned to be shipped with `fds` as a helper package for working with Cloudflare.
diff --git a/cds/cds.py b/cds/cds.py
@@ -7,81 +7,3 @@
 from CloudFlare.exceptions import CloudFlareAPIError
 
 cf = CloudFlare()
-
-zoneDomain = 'pokerchiplounge.com'
-
-def ensure_unique_record_for_name(hostname, )
-
-try:
-    params = {'name': zoneDomain}
-    zones = cf.zones.get(params=params)[0]
-    print(zones)
-except CloudFlareAPIError as e:
-    log.error('Bad auth - %s' % e)
-    print('badauth')
-
-    if len(zones) == 0:
-        log.error('No host')
-        return 'nohost'
-
-    if len(zones) != 1:
-        log.error('/zones.get - %s - api call returned %d items' % (zoneDomain, len(zones)))
-        return 'notfqdn'
-
-    zone_id = zones[0]['id']
-    log.debug("Zone ID is {}".format(zone_id))
-
-    try:
-        params = {'name': hostname, 'match': 'all', 'type': ipAddressType}
-        dns_records = cf.zones.dns_records.get(zone_id, params=params)
-    except CloudFlareAPIError as e:
-        log.error('/zones/dns_records %s - %d %s - api call failed' % (hostname, e, e))
-        return '911'
-
-    desiredRecordData = {
-        'name': hostname,
-        'type': ipAddressType,
-        'content': ip
-    }
-    if ttl:
-        desiredRecordData['ttl'] = ttl
-
-    # update the record - unless it's already correct
-    for dnsRecord in dns_records:
-        oldIp = dnsRecord['content']
-        oldIpType = dnsRecord['type']
-
-        if ipAddressType not in ['A', 'AAAA']:
-            # we only deal with A / AAAA records
-            continue
-
-        if ipAddressType != oldIpType:
-            # only update the correct address type (A or AAAA)
-            # we don't see this becuase of the search params above
-            log.debug('IGNORED: %s %s ; wrong address family' % (hostname, oldIp))
-            continue
-
-        if ip == oldIp:
-            log.info('UNCHANGED: %s == %s' % (hostname, ip))
-            # nothing to do, record already matches to desired IP
-            return 'nochg'
-
-        # Yes, we need to update this record - we know it's the same address type
-        dnsRecordId = dnsRecord['id']
-
-        try:
-            cf.zones.dns_records.put(zone_id, dnsRecordId, data=desiredRecordData)
-        except CloudFlare.exceptions.CloudFlareAPIError as e:
-            log.error('/zones.dns_records.put %s - %d %s - api call failed' % (hostname, e, e))
-            return '911'
-        log.info('UPDATED: %s %s -> %s' % (hostname, oldIp, ip))
-        return 'good'
-
-    # no exsiting dns record to update - so create dns record
-    try:
-        cf.zones.dns_records.post(zone_id, data=desiredRecordData)
-        log.info('CREATED: %s %s' % (hostname, ip))
-        return 'good'
-    except CloudFlare.exceptions.CloudFlareAPIError as e:
-        log.error('/zones.dns_records.post %s - %d %s - api call failed' % (hostname, e, e))
-        return '911'
diff --git a/fds.spec b/fds.spec
@@ -33,6 +33,7 @@ BuildRequires:  python2-devel
 BuildRequires:  python2-six
 BuildRequires:  python-netaddr
 BuildRequires:  python2-tqdm
+BuildRequires:  python2-cloudflare
 # will bring in msgpack and lockfile dependencies:
 Requires:       python2-CacheControl
 # For tests
@@ -44,6 +45,7 @@ BuildRequires:  python%{python3_pkgversion}-devel
 BuildRequires:  python%{python3_pkgversion}-six
 BuildRequires:  python%{python3_pkgversion}-netaddr
 BuildRequires:  python%{python3_pkgversion}-tqdm
+BuildRequires:  python%{python3_pkgversion}-cloudflare
 # will bring in msgpack and lockfile dependencies:
 Requires:       python%{python3_pkgversion}-CacheControl
 # For tests
@@ -71,6 +73,7 @@ BuildArch:      noarch
 Requires:       python2-six
 Requires:       python-netaddr
 Requires:       python2-tqdm
+Requires:       python2-cloudflare
 # will bring in msgpack and lockfile dependencies:
 Requires:       python2-CacheControl
 %{?python_provide:%python_provide python2-%{name}}
@@ -87,6 +90,7 @@ BuildArch:      noarch
 Requires:       python%{python3_pkgversion}-six
 Requires:       python%{python3_pkgversion}-netaddr
 Requires:       python%{python3_pkgversion}-tqdm
+Requires:       python%{python3_pkgversion}-cloudflare
 # will bring in msgpack and lockfile dependencies:
 Requires:       python%{python3_pkgversion}-CacheControl
 %{?python_provide:%python_provide python%{python3_pkgversion}-%{name}}

diff --git a/fds/FirewallWrapper.py b/fds/FirewallWrapper.py
@@ -1,8 +1,11 @@
+# from netaddr import IPAddress
+import logging as log  # for verbose output
+
 import dbus
 from firewall.client import FirewallClient
 from firewall.client import FirewallClientIPSetSettings
-# from netaddr import IPAddress
-import logging as log  # for verbose output
+
+from fds.WebClient import WebClient
 
 
 def do_maybe_already_enabled(func):
@@ -106,6 +109,10 @@ def get_block_ipset_for_ip(self, ip):
             return self.get_block_ipset6()
         return None
 
+    @do_maybe_already_enabled
+    def ensure_ipset_entries(self, ipset, entries):
+        return ipset.setEntries(entries)
+
     @do_maybe_already_enabled
     def ensure_entry_in_ipset(self, ipset, entry):
         return ipset.addEntry(str(entry))
@@ -134,7 +141,7 @@ def block(self, ip):
 
 
     @do_maybe_already_enabled
-    def block(self, ip):
+    def block_ip(self, ip):
         block_ipset = self.get_block_ipset_for_ip(ip)
         if not block_ipset:
             # TODO err: unsupported protocol
@@ -153,28 +160,83 @@ def remove_ipset_from_zone(self, zone, ipset_name):
             ipset_name
         ))
 
-
-    def reset(self):
-        # firewalld up to this commit
-        # https://github.com/firewalld/firewalld/commit/f5ed30ce71755155493e78c13fd9036be8f70fc4
-        # does not delete runtime ipsets, so we can only clear them? :(
+    def clear_ipset_by_name(self, ipset_name):
         try:
-            self.fw.setEntries(self.NETWORKBLOCK_IPSET4, [])
-        except dbus.exceptions.DBusException:
-            pass
-        try:
-            self.fw.setEntries(self.NETWORKBLOCK_IPSET6, [])
+            # does not work: ipset.setEntries([])
+            self.fw.setEntries(ipset_name, [])
         except dbus.exceptions.DBusException:
             pass
 
-        block_ipset4 = self.get_block_ipset4()
-        block_ipset4.setEntries([])
-        block_ipset4.remove()
-        block_ipset6 = self.get_block_ipset6()
-        block_ipset6.remove()
+    def destroy_ipset_by_name(self, name):
+        log.info('Destroying IPSet {}'.format(name))
+        # firewalld up to this commit
+        # https://github.com/firewalld/firewalld/commit/f5ed30ce71755155493e78c13fd9036be8f70fc4
+        # does not delete runtime ipsets, so we have to clear them :(
+        # they are not removed from runtime as still reported by ipset -L
+        # although they *are* removed from FirewallD
+        if name not in self.fw.getIPSets():
+            return
+
+        ipset = self.config.getIPSetByName(name)
+        if ipset:
+            self.clear_ipset_by_name(name)
+            ipset.remove()
 
+    def reset(self):
         drop_zone = self.config.getZoneByName('drop')
+
         self.remove_ipset_from_zone(drop_zone, self.NETWORKBLOCK_IPSET4)
+        self.destroy_ipset_by_name(self.NETWORKBLOCK_IPSET4)
+
         self.remove_ipset_from_zone(drop_zone, self.NETWORKBLOCK_IPSET6)
+        self.destroy_ipset_by_name(self.NETWORKBLOCK_IPSET6)
+
+        all_ipsets = self.fw.getIPSets()
+        # get any ipsets prefixed with "fds-"
+        for ipset_name in all_ipsets:
+            self.remove_ipset_from_zone(drop_zone, ipset_name)
+            self.destroy_ipset_by_name(ipset_name)
+
+        self.fw.reload()
+
 
+    def block_country(self, ip_or_country_name):
+        # print('address/netmask is invalid: %s' % sys.argv[1])
+        # parse out as a country
+        from .Countries import Countries
+        countries = Countries()
+        c = countries.getByName(ip_or_country_name)
+
+        if not c:
+            log.error('{} does not look like a correct IP or a country name'.format(ip_or_country_name))
+            return False
+
+        log.info('Blocking {} {}'.format(c.name, c.getFlag()))
+        # print("\N{grinning face}")
+
+        # TODO get aggregated zone file, save as cache,
+        # do diff to know which stuff was changed and add/remove blocks
+        # https://docs.python.org/2/library/difflib.html
+        # TODO persist info on which countries were blocked (in the config file)
+        # then sync zones via "fds cron"
+        # TODO conditional get test on getpagespeed.com
+        w = WebClient()
+        country_networks = w.get_country_networks(country=c)
+
+        ipset = self.get_create_set(c.get_set_name())
+        self.ensure_ipset_entries(ipset, country_networks)
+
+        # this is slow. setEntries is a lot faster
+        # for network in tqdm(country_networks, unit='network',
+        #                     desc='Adding {} networks to IPSet {}'.format(c.getNation(), c.get_set_name())):
+        #     log.debug(network)
+        #     fw.ensure_entry_in_ipset(ipset=ipset, entry=network)
+
+        # TODO retry, timeout
+        # this action re-adds all entries entirely
+        # there should be "fds-<country.code>-<family>" ip set
+        self.ensure_block_ipset_in_drop_zone(ipset)
+        log.info('Reloading FirewallD...')
         self.fw.reload()
+        log.info('Done!')
+        # while cron will do "sync" behavior"
diff --git a/fds/WebClient.py b/fds/WebClient.py
@@ -73,7 +73,7 @@ def get_country_networks(self, country):
         log.debug('Downloading {}'.format(url))
         content = self.download_file(
             url,
-            display_name='{} {} block list'.format(country.getNation(), country.getFlag()),
+            display_name='{} networks list'.format(country.getNation()),
             local_filename=get_country_zone_filename(country),
             return_type='contents'
         )

diff --git a/fds/__about__.py b/fds/__about__.py
@@ -1 +1 @@
-__version__ = "0.0.1"
+__version__ = "0.0.2"