From f6ce3b604f10f9056f251a28da6f56bde60fa4e9 Mon Sep 17 00:00:00 2001 From: romner-set Date: Thu, 10 Oct 2024 20:35:14 +0200 Subject: [PATCH] nixos/vpp: init --- .../manual/release-notes/rl-2505.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/networking/vpp.nix | 543 ++++++++++++++++++ 3 files changed, 546 insertions(+) create mode 100644 nixos/modules/services/networking/vpp.nix diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index 83ec8b2de680f..d1434bd61c632 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -16,6 +16,8 @@ - [Omnom](https://github.com/asciimoo/omnom), a webpage bookmarking and snapshotting service. Available as [services.omnom](options.html#opt-services.omnom.enable). +- [FD.io's Vector Packet Processor](https://fd.io/docs/vpp/master/), a userland network stack for high-performance routing and switching on general purpose computers. Available as [services.vpp](#opt-services.vpp.instances). + - [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](#opt-services.amazon-cloudwatch-agent.enable). - [agorakit](https://github.com/agorakit/agorakit), an organization tool for citizens' collectives. Available with [services.agorakit](#opt-services.agorakit.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 1d6f11953e469..c04adf3eb2fb5 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1273,6 +1273,7 @@ ./services/networking/v2raya.nix ./services/networking/veilid.nix ./services/networking/vdirsyncer.nix + ./services/networking/vpp.nix ./services/networking/vsftpd.nix ./services/networking/wasabibackend.nix ./services/networking/websockify.nix diff --git a/nixos/modules/services/networking/vpp.nix b/nixos/modules/services/networking/vpp.nix new file mode 100644 index 0000000000000..06cd51856f8d4 --- /dev/null +++ b/nixos/modules/services/networking/vpp.nix @@ -0,0 +1,543 @@ +{ + config, + lib, + pkgs, + ... +}: +let + inherit (lib) + mkEnableOption + mkPackageOption + mkOption + types + mkIf + ; + cfg = config.services.vpp; + + # Converts the settings option to a string + parseConfig = + cfg: + let + inherit (lib.lists) sort concatMap filter; + + sortedAttrs = + set: + sort ( + l: r: + if l == "extraConfig" then + false # Always put extraConfig last + else if builtins.isAttrs set.${l} == builtins.isAttrs set.${r} then + l < r + else + builtins.isAttrs set.${r} # Attrsets should be last, makes for a nice config + # This last case occurs when any side (but not both) is an attrset + # The order of these is correct when the attrset is on the right + # which we're just returning + ) (builtins.attrNames set); + + # Specifies an attrset that encodes the value according to its type + encode = + name: value: + { + null = [ ]; + bool = lib.optional value name; + int = [ "${name} ${builtins.toString value}" ]; + + # extraConfig should be inserted verbatim + string = [ (if name == "extraConfig" then value else "${name} ${value}") ]; + + # Values like `foo = [ "bar" "baz" ];` should be transformed into + # foo bar + # foo baz + list = concatMap (encode name) value; + + # Values like `foo = { bar = { baz = true; }; bar2 = { baz2 = true; }; };` should be transformed into + # foo bar { + # baz + # } + # foo bar2 { + # baz2 + # } + set = concatMap ( + subname: + lib.optionals (value.${subname} != null) ( + [ "${name} ${subname} {" ] ++ (map (line: " ${line}") (toLines value.${subname})) ++ [ "}" ] + ) + ) (filter (v: v != null) (builtins.attrNames value)); + } + .${builtins.typeOf value}; + + # One level "above" encode, acts upon a set and uses encode on each name,value pair + toLines = set: concatMap (name: encode name set.${name}) (sortedAttrs set); + + # Moves top-level attrsets into a dummy "" value, so that `section = {...}` is transformed to `section {...}` + parseSections = + set: + builtins.mapAttrs (name: value: { + "" = value; + }) set; + in + lib.strings.concatStringsSep "\n" (toLines (parseSections cfg)); + + semanticTypes = with types; rec { + vppAtom = nullOr (oneOf [ + int + bool + str + ]); + vppAttr = attrsOf vppAll; + vppAll = + (oneOf [ + vppAtom + (listOf vppAtom) + vppAttr + ]) + // { + # Since this is a recursive type and the description by default contains + # the description of its subtypes, infinite recursion would occur without + # explicitly breaking this cycle + description = "vpp values (atoms (null, str, int, bool), list of atoms, or attrsets of vpp values)"; + }; + }; + + vppOpts = + { name, ... }: + { + options = { + enable = mkEnableOption "FD.io's Vector Packet Processor, a high-performance userspace network stack."; + + name = mkOption { + type = types.str; + default = name; + description = '' + Name is used as a suffix for the service name. + By default it takes the value you use for `` in: + {option}`services.vpp.instances.` + ''; + }; + + package = mkPackageOption pkgs "vpp" { }; + + kernelModule = mkOption { + type = types.nullOr types.str; + default = "uio_pci_generic"; + example = "vfio-pci"; + description = "UIO kernel driver module to load before starting this VPP instance. Set to `null` to disable this functionality."; + }; + + group = mkOption { + type = types.str; + default = "vpp"; + example = "vpp-main"; + description = "Group that grants users in it privileges to control this instance via {command}`vppctl`."; + }; + + settings = mkOption { + description = '' + Configuration for VPP, see for details. + + Nix value declared here will be translated directly to the config format VPP uses. Attributes + called `extraConfig` will be inserted verbatim into the resulting config file. + + Top-level attrsets, lists and primitives get parsed as expected, i.e. `foo = { bar = true; };` + becomes `foo { bar }` in the final config file. Nested attrsets on the other hand, such as + {option}`dpdk.dev`, get parsed differently. For example, this config: + ```nix + settings = { + dpdk = { + dev = { + "foo".name = "bar"; + "baz".name = "qux"; + }; + }; + }; + ``` + Would get parsed into the following: + ``` + dpdk { + dev foo { name bar } + dev baz { name qux } + } + ``` + Notice how `dev.foo` becomes `dev foo { }` instead of `dev { foo { } }`. + ''; + defaultText = '' + unix = { + nodaemon = true; + nosyslog = true; + cli-listen = "/run/vpp/\${name}-cli.sock"; + gid = config.services.vpp.instances.\${name}.group; + startup-config = config.services.vpp.instances.\${name}.startupConfigFile; + cli-prompt = "vpp-\${name}"; + }; + api-segment = { + prefix = "\${name}"; + gid = config.services.vpp.instances.\${name}.group; + }; + ''; + example = lib.literalExpression '' + { + unix.cli-listen = "/run/vpp/cli.sock"; + + plugins = { + plugin."rdma_plugin.so".disable = true; + }; + + cpu = { + main-core = 0; + workers = 4; + }; + + dpdk = { + dev."0000:01:00.0" = { + name = "sfp0"; + num-rx-queues = 4; + }; + dev."0000:01:00.1" = { }; + ## In the case of many devices that don't need special config, they can also be defined in a list: + # dev = [ "0000:01:00.0" "0000:01:00.1" ]; + ## And, since `x = [ y ];` is functionally identical to `x = y;`, this is also possible: + # dev = [ + # { + # default.num-rx-queues = 4; + # + # "0000:01:00.0" = { + # name = "uplink"; + # num-rx-queues = 8; + # }; + # } + # [ "0000:02:00.0" "0000:02:00.1" ] + # ]; + + blacklist = [ + "8086:10fb" + ]; + + no-multi-seg = true; + no-tx-checksum-offload = true; + + extraConfig = "dev 0000:02:00.0"; + }; + } + ''; + type = types.submodule { + freeformType = semanticTypes.vppAttr; + options = + let + # Most of the defaults don't need to be changed except under very specific circumstances and + # all have their values documented in defaultText, so they're not visible to reduce clutter + mkDefaultOption = + value: + mkOption { + default = value; + type = semanticTypes.vppAll; + visible = false; + }; + in + { + ## Config defaults + api-segment = { + prefix = mkDefaultOption name; + gid = mkDefaultOption cfg.instances.${name}.group; + }; + + unix = { + nodaemon = mkDefaultOption true; + nosyslog = mkDefaultOption true; + gid = mkDefaultOption cfg.instances.${name}.group; + startup-config = mkDefaultOption (builtins.toString cfg.instances.${name}.startupConfigFile); + + cli-prompt = mkDefaultOption "vpp-${name}"; + cli-listen = mkOption { + # Visible in documentation unlike other defaults, since changing this + # setting will likely be pretty common on single-instance configs + type = types.str; + default = "/run/vpp/${name}-cli.sock"; + example = "localhost:5002"; + description = '' + Address in the format IPADDR:PORT or a socket path the CLI should listen on. If you only run + a single instance of VPP, it's recommended to change this to `/run/vpp/cli.sock` so + {command}`vppctl` can be used without specifying this path using the `-s` argument. + ''; + }; + }; + + ## User-facing option documentation + plugins.plugin = mkOption { + type = types.attrsOf ( + types.submodule { + freeformType = semanticTypes.vppAll; + options.enable = mkOption { + type = types.nullOr types.bool; + example = true; + default = null; + description = '' + Whether to enable this plugin. Because of how the config is parsed, `false` + has no effect. If you want to explicitly turn a plugin off use {option}`disable`. + ''; + }; + options.disable = mkOption { + type = types.nullOr types.bool; + example = true; + default = null; + description = '' + Whether to disable this plugin. Because of how the config is parsed, `false` + has no effect. If you want to explicitly turn a plugin on use {option}`enable`. + ''; + }; + } + ); + example = { + default.disable = true; + "dpdk_plugin.so".enable = true; + }; + default = { }; + description = '' + Configuration for specific plugins, usually used to turn a plugin on or off. + Special value `default` applies to all plugins. + ''; + }; + + dpdk = { + dev = mkOption { + type = types.attrsOf ( + types.submodule { + freeformType = semanticTypes.vppAll; + + options.name = mkOption { + type = types.nullOr types.str; + example = "eth0"; + default = null; + description = "Override the name of this interface as seen from the CLI."; + }; + options.num-rx-queues = mkOption { + type = types.nullOr types.ints.positive; + example = 4; + default = null; + description = '' + Number of receive queues on this interface. Useful for multi-threaded operation. + Use the `cpu.*` options to setup workers if you want to use this option. + ''; + }; + } + ); + example = { + "0000:01:00.0".name = "eth0"; + }; + default = { }; + description = '' + Which DPDK network interfaces VPP should bind to, can also be specified as a list if no + per-device configuration is needed. Special value `default` applies to all interfaces. + ''; + }; + + blacklist = mkOption { + type = types.oneOf [ + types.str + (types.listOf types.str) + ]; + example = "8086:10fb"; + default = [ ]; + description = "Device types to blacklist, using PCI vendor:device syntax."; + }; + }; + + cpu = { + main-core = mkOption { + type = types.nullOr types.ints.unsigned; + example = 0; + default = null; + description = '' + Logical CPU core where the main thread runs, if unset VPP will use core 1 if available. + ''; + }; + + corelist-workers = mkOption { + type = types.nullOr types.str; + example = "2-3,18-19"; + default = null; + description = '' + Explicitly set logical CPUs on which VPP worker threads will run. + + {option}`skip-cores` and {option}`workers` are incompatible with {option}`corelist-workers`. + ''; + }; + + skip-cores = mkOption { + type = types.nullOr types.ints.positive; + example = 8; + default = null; + description = '' + Set number of logical CPU cores to skip when using the {option}`workers` option. + + {option}`skip-cores` and {option}`workers` are incompatible with {option}`corelist-workers`. + ''; + }; + workers = mkOption { + type = types.nullOr types.ints.positive; + example = 4; + default = null; + description = '' + Set number of workers to be created and automatically assigned. Workers will be pinned to + N consecutive CPU cores while skipping {option}`skip-cores` CPU core(s) and main thread's core. + + {option}`skip-cores` and {option}`workers` are incompatible with {option}`corelist-workers`. + ''; + }; + }; + }; + }; + }; + + settingsFile = mkOption { + type = types.path; + example = "/etc/vpp/startup.conf"; + default = pkgs.writeText "vpp-${name}.conf" (parseConfig cfg.instances.${name}.settings); + defaultText = '' + pkgs.writeText "vpp-\${name}.conf" (parseConfig config.services.vpp.instances.\${name}.settings); + ''; + description = '' + Configuration file passed to {command}`vpp -c`. + It is recommended to use the {option}`settings` option instead. + + Setting this option will override the config file + auto-generated from the {option}`settings` option. + ''; + }; + + startupConfig = mkOption { + type = types.str; + default = ""; + example = '' + set interface state TenGigabitEthernet1/0/0 up + set interface state TenGigabitEthernet1/0/1 up + set interface ip address TenGigabitEthernet1/0/0 2001:db8::1/64 + set interface ip address TenGigabitEthernet1/0/1 2001:db8:1234::1/64 + ''; + description = '' + Script to run on startup in {command}`vppctl`, passed to + VPP's `startup-config` option in the `unix` section as a file. + + This is used to configure things like IP addresses and routes. To configure + interfaces, plugins, etc., see the {option}`settings` option. + ''; + }; + + startupConfigFile = mkOption { + type = types.path; + example = "./vpp-startup.conf"; + default = pkgs.writeText "vpp-${name}-startup.conf" cfg.instances.${name}.startupConfig; + defaultText = '' + pkgs.writeText "vpp-\${name}-startup.conf" (builtins.toString config.services.vpp.instances.\${name}.startupConfig); + ''; + description = '' + File to run as a script on startup in {command}`vppctl`, passed + to VPP's `startup-config` option in the `unix` section. + + Setting this option will override {option}`startupConfig`. + ''; + }; + }; + }; +in +{ + options.services.vpp = { + hugepages = { + autoSetup = mkOption { + default = false; + example = true; + description = '' + Whether to automatically setup hugepages for use with FD.io's Vector Packet Processor. + + If any programs other than VPP use hugepages or you want to use 1GB hugepages, it's recommended + to keep this `false` and set them up manually using {option}`boot.kernel.sysctl` or {option}`boot.kernelParams`. + ''; + }; + + count = mkOption { + type = types.ints.positive; + default = 1024; + example = 512; + description = "Number of 2MB hugepages to setup on the system if {option}`services.vpp.hugepages.autoSetup` is enabled."; + }; + }; + + instances = mkOption { + default = { }; + type = types.attrsOf (types.submodule vppOpts); + description = '' + VPP supports multiple instances for testing or other purposes. + If you don't require multiple instances of VPP you can define just the one. + ''; + example = { + main = { + enable = true; + group = "vpp-main"; + settings = { }; + }; + test = { + enable = false; + group = "vpp-test"; + settings = { }; + }; + }; + }; + }; + + config = + let + #TODO: systemd hardening + mkInstanceServiceConfig = instance: { + description = "Vector Packet Processing Process"; + after = [ + "syslog.target" + "network.target" + "auditd.service" + ]; + serviceConfig = { + ExecStartPre = + [ + "-${pkgs.coreutils}/bin/rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api" + ] + ++ (lib.optional ( + instance.kernelModule != null + ) "-/run/current-system/sw/bin/modprobe ${instance.kernelModule}"); + ExecStart = "${instance.package}/bin/vpp -c ${instance.settingsFile}"; + Type = "simple"; + Restart = "on-failure"; + RestartSec = "5s"; + RuntimeDirectory = "vpp"; + }; + wantedBy = [ "multi-user.target" ]; + }; + instances = lib.attrValues cfg.instances; + in + { + boot.kernel.sysctl = mkIf cfg.hugepages.autoSetup { + # defaults for 2MB hugepages, see https://fd.io/docs/vpp/master/gettingstarted/running/index.html#huge-pages + "vm.nr_hugepages" = cfg.hugepages.count; + "vm.max_map_count" = cfg.hugepages.count * 2; + "kernel.shmmax" = cfg.hugepages.count * 2097152; # * 2 * 1024 * 1024 + }; + + users.groups = lib.mkMerge ( + map ( + instance: + lib.mkIf instance.enable { + ${instance.group} = { }; + } + ) instances + ); + + systemd.services = lib.mkMerge ( + map ( + instance: + lib.mkIf instance.enable { + "vpp-${instance.name}" = mkInstanceServiceConfig instance; + } + ) instances + ); + + meta.maintainers = with lib.maintainers; [ romner-set ]; + }; +}