Implement policy routing for VPC using systemd-networkd

Includes support for * Per-interface policy routing rules to accommodate VPC source/dest restrictions * Configuration of secondary IPv4 addresses * Configuration of ENIs upon hotplug * Routing configuration for delegated prefixes
amazonlinux · Oct 28, 2021 · 15acc20 · 15acc20
commit 15acc20
Show file tree

Hide file tree

Showing 21 changed files with 936 additions and 0 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+debian/ export-ignore
diff --git a/GNUmakefile b/GNUmakefile
@@ -0,0 +1,77 @@
+VERSION=2.0.0
+
+# Used by 'install'
+PREFIX?=/usr/local
+BINDIR=${DESTDIR}${PREFIX}/bin
+UDEVDIR=${DESTDIR}/etc/udev/rules.d
+SYSTEMDDIR=${DESTDIR}/etc/systemd
+SYSTEMD_SYSTEM_DIR=${SYSTEMDDIR}/system
+SYSTEMD_NETWORK_DIR=${SYSTEMDDIR}/network
+SYSCTL_DIR=${DESTDIR}/etc/sysctl.d
+
+SHELLSCRIPTS=$(wildcard bin/*.sh)
+UDEVRULES=$(wildcard udev/*.rules)
+SYSCTL_FILES=$(wildcard sysctl/*.conf)
+
+DIRS:=${BINDIR} ${UDEVDIR} ${SYSTEMDDIR} ${SYSTEMD_SYSTEM_DIR} ${SYSTEMD_NETWORK_DIR} ${SYSCTL_DIR}
+
+DIST_TARGETS=dist-xz dist-gz
+
+help: ## show help
+	@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+${DIRS}:
+	install -d $@
+
+install: ${SHELLSCRIPTS} ${UDEVRULES} ${SYSCTL_FILES} | ${DIRS} ## Install the software. Respects DESTDIR
+	$(foreach f,${SHELLSCRIPTS},install -m755 $f ${BINDIR}/$$(basename --suffix=.sh $f);)
+	$(foreach f,${UDEVRULES},install -m644 $f ${UDEVDIR};)
+	$(foreach f,$(wildcard systemd/network/*.network),install -m644 $f ${SYSTEMD_NETWORK_DIR};)
+	$(foreach f,$(wildcard systemd/system/*.service systemd/system/*.timer),install -m644 $f ${SYSTEMD_SYSTEM_DIR};)
+	$(foreach f,${SYSCTL_FILES},install -m644 $f ${SYSCTL_DIR})
+
+check: ## Run tests
+	@set -x; for script in ${SHELLSCRIPTS}; do \
+		shellcheck --severity warning $${script};\
+	done
+
+dist-tar:
+	git archive --format tar --prefix amazon-ec2-net-utils-$(VERSION)/ v$(VERSION) > ../amazon-ec2-net-utils-$(VERSION).tar
+
+dist-xz: dist-tar
+	xz --keep ../amazon-ec2-net-utils-$(VERSION).tar
+
+dist-gz: dist-tar
+	gzip -c ../amazon-ec2-net-utils-$(VERSION).tar > ../amazon-ec2-net-utils-$(VERSION).tar.gz
+
+dist: dist-hook
+	$(MAKE) $(DIST_TARGETS)
+	rm ../amazon-ec2-net-utils-$(VERSION).tar
+
+tmp-dist: uncommitted-check
+	$(MAKE) $(AM_MAKEFLAGS) VERSION=$(patsubst v%,%,$(shell git describe --tags)) DISTHOOK=0 dist
+
+uncommitted-check:
+	@if ! git update-index --refresh --unmerged || \
+		! git diff-index --name-only --exit-code HEAD; then \
+	echo "*** ERROR: Uncommitted changes in above files"; exit 1; fi
+
+version-check:
+	@if [ -z "$(VERSION)" ]; then \
+		echo "*** ERROR: VERSION not set"; exit 1; fi
+
+ DISTHOOK=1
+dist-hook: uncommitted-check
+	if [ $(DISTHOOK) = 1 ]; then \
+         if ! git rev-parse --verify v$(VERSION) > /dev/null 2>&1; then \
+         echo "*** ERROR: Version v$(VERSION) is not tagged"; exit 1; fi ; \
+         if ! git diff --name-only --exit-code v$(VERSION) HEAD > /dev/null; then \
+         echo "*** ERROR: Git checkout not at version v$(VERSION)"; exit 1; fi ; \
+     fi
+
+tag: uncommitted-check version-check
+	@if git rev-parse --verify v$(VERSION) > /dev/null 2>&1; then \
+		echo "*** ERROR: Version v$(VERSION) is already tagged"; exit 1; fi
+	@git tag v$(VERSION)
+
+.PHONY: dirs check install all dist dist-hook dist-tar dist-xz version-check tag
diff --git a/README.md b/README.md
@@ -0,0 +1,155 @@
+# amazon-ec2-net-utils #
+
+## Rationale ##
+
+The existing amazon-ec2-net-utils package provides functionality
+needed to configure a Linux instance for optimal performance in a VPC
+environment. It handles:
+
+* Per-interface policy routing rules to accommodate VPC source/dest
+  restrictions
+* Configuration of secondary IPv4 addresses
+* Configuration of ENIs upon hotplug
+* Routing configuration for delegated prefixes
+
+The existing amazon-ec2-net-utils package has a long history and is
+tightly coupled to dhclient and initscripts network
+configuration. Both of these components are deprecated upstream and
+will not make up the primary network configuration framework in future
+releases of Amazon Linux. Thus a new implementation is required.
+
+## Implementation ##
+
+amazon-ec2-net-utils leverages systemd-networkd for most of the actual
+interface configuration, and is primarily responsible for mapping
+configuration information available via IMDS to systemd-networkd input
+configuration. It provides event-based configuration via udev rules,
+with timer based actions in order to detect non event based changes
+(e.g. secondary IP address assignment). Generated configuration is
+stored in the /run/ ephemeral filesystem and is not persisted across
+instance reboots. The generated configuration is expected to be
+regenerated from scratch upon reboot. Customers can override the
+behavior of the package by creating configuration files in the local
+administration network directory /etc/systemd/network as described in
+systemd-networkd's documentation.
+
+By utilizing a common framework in the form of systemd, the
+amazon-ec2-net-utils package should be able to integrate with any
+systemd-based distribution. This allows us to provide customers with a
+common baseline behavior regardless of whether they choose Amazon
+Linux or a third-party distribution. Testing has been performed on
+Debian, Fedora, and AL2022.
+
+## Usage ##
+
+amazon-ec2-net-utils is expected to be pre-installed on Amazon Linux
+2022 and future releases. In the common case, customers should not
+need to be aware of its operation. Configuration of network interfaces
+should occur following the principle of least astonishment. That is,
+traffic should be routed via the ENI associated with the source
+address.  Custom configuration should be respected. New ENI
+attachments should be used automatically, and associated resources
+should be cleaned up on detachment. Manipulation of an ENI attachment
+should not impact the functionality of any other ENIs.
+
+## Build and install ##
+
+The recommended way to install amazon-ec2-net-utils is by building a
+package for your distribution. A spec file and debian subdirectory are
+provided and should be reasonably suitable for modern rpm or dpkg
+based distributions. Build dependencies are declared in debian/control
+and in amazon-ec2-net-utils.spec and can be installed using standard
+tools from the distributions (e.g. dpkg-checkbuilddeps and apt, or dnf
+builddep, etc)
+
+The post installation scripts in the spec file and or .deb package
+will stop NetworkManager or ifupdown, if running, and initialize
+systemd-networkd and systemd-resolved. The expectation is that
+amazon-ec2-net-utils will take over and initialize a running system,
+without rebooting, such that it is indistinguishable from a system
+that booted with amazon-ec2-net-utils.
+
+### rpm build and installation ###
+
+    $ mkdir -p rpmbuild/BUILD
+    $ git -C amazon-ec2-net-utils/ archive main | (cd rpmbuild/BUILD/ && tar xvf -)
+    $ rpmbuild -bb rpmbuild/BUILD/amazon-ec2-net-utils.spec
+    $ sudo dnf install rpmbuild/RPMS/noarch/amazon-ec2-net-utils-2.0.0-1.al2022.noarch.rpm
+
+### dpkg build and installation ###
+
+    $ dpkg-buildpackage -uc -us -b
+    $ sudo apt install ../amazon-ec2-net-utils_2.0.0-1_all.deb
+
+### Installation verification ###
+
+    $ # inspect the state of the system to verify that networkd is running:
+    $ networkctl # should report all physical interfaces as "routable" and "configured"
+    $ networkctl status eth0 # should report "/run/systemd/network/70-eth0.network" as the network conf file
+    $ resolvectl # show status of systemd-resolved
+
+**Example:**
+
+    [ec2-user@ip-10-0-0-114 ~]$ networkctl
+    IDX LINK TYPE     OPERATIONAL SETUP
+     1 lo   loopback carrier     unmanaged
+     2 eth0 ether    routable    configured
+
+    2 links listed.
+    [ec2-user@ip-10-0-0-114 ~]$ networkctl status eth0
+    ● 2: eth0
+                        Link File: /usr/lib/systemd/network/99-default.link
+                     Network File: /run/systemd/network/70-eth0.network
+                             Type: ether
+                            State: routable (configured)
+                Alternative Names: enp0s5
+                                   ens5
+                             Path: pci-0000:00:05.0
+                           Driver: ena
+                           Vendor: Amazon.com, Inc.
+                            Model: Elastic Network Adapter (ENA)
+                       HW Address: 02:c9:76:e3:18:0b
+                              MTU: 9001 (min: 128, max: 9216)
+                            QDisc: mq
+     IPv6 Address Generation Mode: eui64
+             Queue Length (Tx/Rx): 2/2
+                          Address: 10.0.0.114 (DHCP4 via 10.0.0.1)
+                                   fe80::c9:76ff:fee3:180b
+                          Gateway: 10.0.0.1
+                              DNS: 10.0.0.2
+                Activation Policy: up
+                  DHCP4 Client ID: IAID:0xed10bdb8/DUID
+                DHCP6 Client DUID: DUID-EN/Vendor:0000ab11a9aa54876c81082a0000
+
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: Link UP
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: Gained carrier
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: Gained IPv6LL
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: DHCPv4 address 10.0.0.114/24 via 10.0.0.1
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: Re-configuring with /run/systemd/network/70-eth0.net>
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: DHCP lease lost
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: DHCPv6 lease lost
+    Sep 01 17:44:54 ip-10-0-0-114.us-west-2.compute.internal systemd-networkd[2042]: eth0: DHCPv4 address 10.0.0.114/24 via 10.0.0.1
+    [ec2-user@ip-10-0-0-114 ~]$ resolvectl
+    Global
+          Protocols: LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported
+    resolv.conf mode: uplink
+
+    Link 2 (eth0)
+       Current Scopes: DNS
+            Protocols: +DefaultRoute -LLMNR -mDNS -DNSOverTLS DNSSEC=no/unsupported
+    Current DNS Server: 10.0.0.2
+          DNS Servers: 10.0.0.2
+
+## TODO ##
+
+There are a few remaining small tasks left to complete, and one larger consideration:
+
+* The primary body of executable code is currently written in
+  bash. While it is intended to be usable as-is, we might consider
+  rewriting some or all of it in something else. Preferred options
+  would be Rust, Go, or C, probably in that order.  Testing
+
+* The systemd .network file templates are currently embedded directly
+  in the shell script code.  Ideally they get moved out of the code to
+  facilitiate easier examination and modification, potentially as
+  configuration files.
diff --git a/amazon-ec2-net-utils.spec b/amazon-ec2-net-utils.spec
@@ -0,0 +1,73 @@
+Name:    amazon-ec2-net-utils
+Version: 2.0.0
+Release: 1%{?dist}
+Summary: utilities for managing network interfaces in Amazon EC2
+
+License: Apache 2.0
+URL:     https://github.com/aws/amazon-ec2-net-utils/
+Source0: amazon-ec2-net-utils-%{version}.tar.xz
+
+BuildArch: noarch
+
+BuildRequires: make
+Requires: systemd-networkd, udev, curl, iproute
+
+%description
+amazon-ec2-net-utils-ng provides udev integration and helper utilities
+to manage network configuration in the Amazon EC2 cloud environment
+
+%prep
+
+%autosetup -n %{name}-%{version}
+
+%install
+make install DESTDIR=%{buildroot} PREFIX=/usr
+
+%files
+%{_sysconfdir}/sysctl.d/90-ipv6-dad.conf
+%{_sysconfdir}/systemd/network/80-ec2.network
+%{_sysconfdir}/systemd/system/[email protected]
+%{_sysconfdir}/systemd/system/[email protected]
+%{_sysconfdir}/systemd/system/[email protected]
+
+%{_sysconfdir}/udev/rules.d/98-eni.rules
+%{_sysconfdir}/udev/rules.d/99-vpc-policy-routes.rules
+%{_bindir}/setup-policy-routes
+
+%post
+
+setup_policy_routes() {
+    local iface node
+    for node in /sys/class/net/*; do
+	iface=$(basename $node)
+	unset ID_NET_DRIVER
+	eval $(udevadm info --export --query=property /sys/class/net/$iface)
+	case $ID_NET_DRIVER in
+	    ena|ixgbevf|vif)
+		systemctl start policy-routes@${iface}.service
+		systemctl start refresh-policy-routes@${iface}.timer
+		;;
+	    *)
+		echo "Skipping $iface with driver $ID_NET_DRIVER"
+		;;
+	esac
+    done
+}
+
+if [ $1 == 1 ]; then
+    systemctl enable systemd-networkd.service
+    systemctl enable systemd-resolved.service
+    systemctl disable NetworkManager-wait-online.service
+    systemctl disable NetworkManager.service
+    [ -f /etc/resolv.conf ] && mv /etc/resolv.conf /etc/resolv.conf.old
+    ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
+    if [ -d /run/systemd ]; then
+	systemctl stop NetworkManager.service
+	systemctl start systemd-networkd.service
+	setup_policy_routes
+	systemctl start systemd-resolved.service
+    fi
+fi
+
+%changelog
+