From 9273b4bfe07e67ec16c103ee68633b3666a26fa6 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:25:12 +0300 Subject: [PATCH 01/69] feat: Provision & provision k8s opts Two new opts that represent the two scripts used in the provision phase (provision linux and provision k8s). Using go embed to include any necessary config files then run the commands on a node using libssh Signed-off-by: aerosouund --- .../opts/k8sprovision/conf/001-calico.conf | 2 + .../opts/k8sprovision/conf/002-dhclient.conf | 2 + .../opts/k8sprovision/conf/adv-audit.yaml | 17 + .../gocli/opts/k8sprovision/conf/cni.diff | 109 + .../gocli/opts/k8sprovision/conf/cni.yaml | 3872 +++++++++++++++++ .../opts/k8sprovision/conf/cni_ipv6.diff | 119 + .../opts/k8sprovision/conf/crio-yum.repo | 6 + .../gocli/opts/k8sprovision/conf/extra-images | 23 + .../opts/k8sprovision/conf/fetch-images.sh | 54 + .../gocli/opts/k8sprovision/conf/k8s.conf | 5 + .../gocli/opts/k8sprovision/conf/kubeadm.conf | 54 + .../opts/k8sprovision/conf/kubeadm_ipv6.conf | 70 + .../opts/k8sprovision/conf/kubernetes.repo | 6 + .../gocli/opts/k8sprovision/conf/psa.yaml | 20 + .../opts/k8sprovision/conf/registries.conf | 8 + .../gocli/opts/k8sprovision/conf/storage.conf | 4 + .../gocli/opts/k8sprovision/k8sprovision.go | 301 ++ .../opts/k8sprovision/k8sprovision_test.go | 39 + ...add-security-context-deployment-patch.yaml | 6 + .../gocli/opts/k8sprovision/patches/etcd.yaml | 4 + .../k8sprovision/patches/kube-apiserver.yaml | 4 + .../patches/kube-controller-manager.yaml | 4 + .../k8sprovision/patches/kube-scheduler.yaml | 4 + .../gocli/opts/k8sprovision/testconfig.go | 122 + .../gocli/opts/labelnodes/labelnodes.go | 2 +- .../gocli/opts/labelnodes/labelnodes_test.go | 39 + .../gocli/opts/labelnodes/testconfig.go | 9 + .../gocli/opts/provision/provision.go | 2 +- .../gocli/opts/provision/provision_test.go | 39 + .../gocli/opts/provision/testconfig.go | 35 + .../github.com/pmezard/go-difflib/LICENSE | 27 - .../pmezard/go-difflib/difflib/difflib.go | 772 ---- .../github.com/stretchr/testify/LICENSE | 21 - .../testify/assert/assertion_compare.go | 480 -- .../testify/assert/assertion_format.go | 815 ---- .../testify/assert/assertion_format.go.tmpl | 5 - .../testify/assert/assertion_forward.go | 1621 ------- .../testify/assert/assertion_forward.go.tmpl | 5 - .../testify/assert/assertion_order.go | 81 - .../stretchr/testify/assert/assertions.go | 2105 --------- .../github.com/stretchr/testify/assert/doc.go | 46 - .../stretchr/testify/assert/errors.go | 10 - .../testify/assert/forward_assertions.go | 16 - .../testify/assert/http_assertions.go | 165 - 44 files changed, 4979 insertions(+), 6171 deletions(-) create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/001-calico.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/002-dhclient.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/adv-audit.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/cni.diff create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/cni.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/cni_ipv6.diff create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/crio-yum.repo create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/extra-images create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/fetch-images.sh create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/k8s.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/kubeadm.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/kubeadm_ipv6.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/kubernetes.repo create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/psa.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/registries.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/conf/storage.conf create mode 100644 cluster-provision/gocli/opts/k8sprovision/k8sprovision.go create mode 100644 cluster-provision/gocli/opts/k8sprovision/k8sprovision_test.go create mode 100644 cluster-provision/gocli/opts/k8sprovision/patches/add-security-context-deployment-patch.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/patches/etcd.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/patches/kube-apiserver.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/patches/kube-controller-manager.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/patches/kube-scheduler.yaml create mode 100644 cluster-provision/gocli/opts/k8sprovision/testconfig.go create mode 100644 cluster-provision/gocli/opts/labelnodes/labelnodes_test.go create mode 100644 cluster-provision/gocli/opts/labelnodes/testconfig.go create mode 100644 cluster-provision/gocli/opts/provision/provision_test.go create mode 100644 cluster-provision/gocli/opts/provision/testconfig.go delete mode 100644 cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/LICENSE delete mode 100644 cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/difflib/difflib.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/LICENSE delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_compare.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_order.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertions.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/doc.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/errors.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/forward_assertions.go delete mode 100644 cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/http_assertions.go diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/001-calico.conf b/cluster-provision/gocli/opts/k8sprovision/conf/001-calico.conf new file mode 100644 index 0000000000..1722715699 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/001-calico.conf @@ -0,0 +1,2 @@ +[keyfile] +unmanaged-devices=interface-name:cali*;interface-name:tunl* \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/002-dhclient.conf b/cluster-provision/gocli/opts/k8sprovision/conf/002-dhclient.conf new file mode 100644 index 0000000000..47fec0ab03 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/002-dhclient.conf @@ -0,0 +1,2 @@ +[main] +dhcp=dhclient \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/adv-audit.yaml b/cluster-provision/gocli/opts/k8sprovision/conf/adv-audit.yaml new file mode 100644 index 0000000000..7eb84c98f9 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/adv-audit.yaml @@ -0,0 +1,17 @@ +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Request + users: ["kubernetes-admin"] + resources: + - group: kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancereplicasets + - virtualmachineinstancepresets + - virtualmachineinstancemigrations + omitStages: + - RequestReceived + - ResponseStarted + - Panic \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/cni.diff b/cluster-provision/gocli/opts/k8sprovision/conf/cni.diff new file mode 100644 index 0000000000..9074ef4de7 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/cni.diff @@ -0,0 +1,109 @@ +--- a/cluster-provision/k8s/1.19/manifests/cni.do-not-change.yaml ++++ b/cluster-provision/k8s/1.19/manifests/cni.do-not-change.yaml +@@ -32,7 +32,12 @@ data: + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { +- "type": "calico-ipam" ++ "type": "calico-ipam", ++ "assign_ipv4": "true", ++ "assign_ipv6": "true" ++ }, ++ "container_settings": { ++ "allow_ip_forwarding": true + }, + "policy": { + "type": "k8s" +@@ -3533,7 +3538,7 @@ spec: + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam +- image: docker.io/calico/cni:v3.18.0 ++ image: quay.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: +@@ -3560,7 +3565,7 @@ spec: + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni +- image: docker.io/calico/cni:v3.18.0 ++ image: quay.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: +@@ -3601,7 +3606,7 @@ spec: + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver +- image: docker.io/calico/pod2daemon-flexvol:v3.18.0 ++ image: quay.io/calico/pod2daemon-flexvol:v3.18.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver +@@ -3612,7 +3617,7 @@ spec: + # container programs network policy and routes on each + # host. + - name: calico-node +- image: docker.io/calico/node:v3.18.0 ++ image: quay.io/calico/node:v3.18.0 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. +@@ -3671,6 +3676,8 @@ spec: + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" ++ - name: IP6 ++ value: "autodetect" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" +@@ -3679,12 +3686,14 @@ spec: + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT +- value: "false" ++ value: "true" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" ++ - name: CALICO_IPV6POOL_NAT_OUTGOING ++ value: "true" + securityContext: + privileged: true + resources: +@@ -3818,6 +3818,8 @@ spec: + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule ++ - key: node-role.kubernetes.io/control-plane ++ effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: +@@ -3820,9 +3829,12 @@ spec: + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical ++ securityContext: ++ seLinuxOptions: ++ type: spc_t + containers: + - name: calico-kube-controllers +- image: docker.io/calico/kube-controllers:v3.18.0 ++ image: quay.io/calico/kube-controllers:v3.18.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS +@@ -3847,7 +3859,7 @@ + + # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +-apiVersion: policy/v1beta1 ++apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + name: calico-kube-controllers diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/cni.yaml b/cluster-provision/gocli/opts/k8sprovision/conf/cni.yaml new file mode 100644 index 0000000000..bcc4ee0c02 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/cni.yaml @@ -0,0 +1,3872 @@ +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } + +--- +# Source: calico/templates/kdd-crds.yaml + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + type: string + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the source address to use on programmed device + routes. By default the source address is left blank, leaving the + kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a comma-delimited list of + UDP/TCP ports that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. Each + port should be specified as tcp: or udp:. + For back-compatibility, if the protocol is not specified, it defaults + to "tcp". To disable all inbound host ports, use the value none. + The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol and port, both + must be specified. + properties: + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a comma-delimited list + of UDP/TCP ports that Felix will allow outgoing traffic from host + endpoints to irrespective of the security policy. This is useful + to avoid accidentally cutting off a host with incorrect configuration. + Each port should be specified as tcp: or udp:. + For back-compatibility, if the protocol is not specified, it defaults + to "tcp". To disable all outbound host ports, use the value none. + The default value opens etcd''s standard ports to ensure that Felix + does not get cut off from etcd as well as allowing DHCP and DNS. + [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, + udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol and port, both + must be specified. + properties: + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is legacy. + type: string + iptablesFilterAllowAction: + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeTableRange: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRange specifies the indices of the route tables + that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + type: boolean + vxlanEnabled: + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled. + [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the Wireguard interface. [Default: wg.calico]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by Wireguard. [Default: 51820]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the Wireguard interface. + See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + type: string + allocations: + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + type: string + deleted: + type: boolean + strictAffinity: + type: boolean + unallocated: + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 112 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When nat-outgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml + +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- + +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: docker.io/calico/pod2daemon-flexvol:v3.18.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.18.0 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.18.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system + +--- + +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + +--- +# Source: calico/templates/calico-etcd-secrets.yaml + +--- +# Source: calico/templates/calico-typha.yaml + +--- +# Source: calico/templates/configure-canal.yaml + + diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/cni_ipv6.diff b/cluster-provision/gocli/opts/k8sprovision/conf/cni_ipv6.diff new file mode 100644 index 0000000000..45cae03d34 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/cni_ipv6.diff @@ -0,0 +1,119 @@ +--- a/cluster-provision/k8s/1.24/manifests/cni.do-not-change.yaml ++++ b/cluster-provision/k8s/1.24/manifests/cni.do-not-change.yaml +@@ -32,7 +32,12 @@ + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { +- "type": "calico-ipam" ++ "type": "calico-ipam", ++ "assign_ipv4": "false", ++ "assign_ipv6": "true" ++ }, ++ "container_settings": { ++ "allow_ip_forwarding": true + }, + "policy": { + "type": "k8s" +@@ -3533,7 +3538,7 @@ + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam +- image: docker.io/calico/cni:v3.18.0 ++ image: quay.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: +@@ -3560,7 +3565,7 @@ + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni +- image: docker.io/calico/cni:v3.18.0 ++ image: quay.io/calico/cni:v3.18.0 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: +@@ -3601,7 +3606,7 @@ + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver +- image: docker.io/calico/pod2daemon-flexvol:v3.18.0 ++ image: quay.io/calico/pod2daemon-flexvol:v3.18.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver +@@ -3612,7 +3617,7 @@ + # container programs network policy and routes on each + # host. + - name: calico-node +- image: docker.io/calico/node:v3.18.0 ++ image: quay.io/calico/node:v3.18.0 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. +@@ -3641,10 +3646,10 @@ + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP +- value: "autodetect" ++ value: "none" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP +- value: "Always" ++ value: "Never" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" +@@ -3671,6 +3676,8 @@ + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" ++ - name: IP6 ++ value: "autodetect" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" +@@ -3679,12 +3686,16 @@ + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT +- value: "false" ++ value: "true" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" ++ - name: CALICO_IPV6POOL_NAT_OUTGOING ++ value: "true" ++ - name: CALICO_ROUTER_ID ++ value: "hash" + securityContext: + privileged: true + resources: +@@ -3818,11 +3829,16 @@ + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule ++ - key: node-role.kubernetes.io/control-plane ++ effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical ++ securityContext: ++ seLinuxOptions: ++ type: spc_t + containers: + - name: calico-kube-controllers +- image: docker.io/calico/kube-controllers:v3.18.0 ++ image: quay.io/calico/kube-controllers:v3.18.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS +@@ -3847,7 +3863,7 @@ + + # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +-apiVersion: policy/v1beta1 ++apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + name: calico-kube-controllers diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/crio-yum.repo b/cluster-provision/gocli/opts/k8sprovision/conf/crio-yum.repo new file mode 100644 index 0000000000..9d6816ada3 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/crio-yum.repo @@ -0,0 +1,6 @@ +[isv_kubernetes_addons_cri-o_stable_v1.VERSION] +name=CRI-O v1.VERSION (Stable) (rpm) +type=rpm-md +baseurl=https://storage.googleapis.com/kubevirtci-crio-mirror/isv_kubernetes_addons_cri-o_stable_v1.VERSION +gpgcheck=0 +enabled=1 \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/extra-images b/cluster-provision/gocli/opts/k8sprovision/conf/extra-images new file mode 100644 index 0000000000..473ecf1347 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/extra-images @@ -0,0 +1,23 @@ +quay.io/kubevirtci/install-cni:1.15.0 +quay.io/kubevirtci/operator:1.15.0 +quay.io/kubevirtci/pilot:1.15.0 +quay.io/kubevirtci/proxyv2:1.15.0 +quay.io/calico/cni:v3.18.0 +quay.io/calico/kube-controllers:v3.18.0 +quay.io/calico/node:v3.18.0 +quay.io/calico/pod2daemon-flexvol:v3.18.0 +quay.io/prometheus-operator/prometheus-config-reloader:v0.75.1 +docker.io/grafana/grafana:11.1.0 +ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:dee1979d92f0a31598a6e3569ac7004be7d29e7ca9e31db23753ef263110dc04 +ghcr.io/kubevirt/kubesecondarydns@sha256:77132adb5f840ceb0aadd408731a5c8b01a4b427a78084ab5e4e9b961195cb02 +quay.io/kubevirt/bridge-marker@sha256:5d24c6d1ecb0556896b7b81c7e5260b54173858425777b7a84df8a706c07e6d2 +quay.io/kubevirt/cdi-apiserver:v1.58.1 +quay.io/kubevirt/cdi-controller:v1.58.1 +quay.io/kubevirt/cdi-operator:v1.58.1 +quay.io/kubevirt/cdi-uploadproxy:v1.58.1 +quay.io/kubevirt/cluster-network-addons-operator:v0.87.0 +quay.io/kubevirt/cni-default-plugins@sha256:825e3f9fec1996c54a52cec806154945b38f76476b160d554c36e38dfffe5e61 +quay.io/kubevirt/kubemacpool@sha256:afba7d0c4a95d2d4924f6ee6ef16bbe59117877383819057f01809150829cb0c +quay.io/kubevirt/macvtap-cni@sha256:434420511e09b2b5ede785a2c9062b6658ffbc26fbdd4629ce06110f9039c600 +quay.io/kubevirt/ovs-cni-plugin@sha256:5f7290e2294255ab2547c3b4bf48cc2d75531ec5a43e600366e9b2719bef983f +quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901 \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/fetch-images.sh b/cluster-provision/gocli/opts/k8sprovision/conf/fetch-images.sh new file mode 100644 index 0000000000..18fba6833b --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/fetch-images.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +function usage() { + cat < [source-image-list] + + Fetches all images from the cluster provision source and manifests. Returns a list that is sorted and + without double entries. + + If source-image-list is provided this is taken as an input and added to the result. + +EOF +} + +function check_args() { + if [ "$#" -lt 1 ]; then + usage + exit 1 + fi + if [ ! -d "$1" ]; then + usage + echo "Directory $1 does not exist" + exit 1 + fi +} + +function main() { + check_args "$@" + + temp_file=$(mktemp) + trap 'rm -f "${temp_file}"' EXIT SIGINT SIGTERM + + provision_dir="$1" + image_regex='([a-z0-9\_\.]+[/-]?)+(@sha256)?:[a-z0-9\_\.\-]+' + image_regex_w_double_quotes='"?'"${image_regex}"'"?' + + ( + # Avoid bailing out because of nothing found in scripts part + set +e + find "$provision_dir" -type f -name '*.sh' -print0 | + xargs -0 grep -iE '(docker|podman)[ _]pull[^ ]+ '"${image_regex_w_double_quotes}" + find "$provision_dir" -type f -name '*.yaml' -print0 | + xargs -0 grep -iE '(image|value): '"${image_regex_w_double_quotes}" + set -e + # last `grep -v` is necessary to avoid trying to pre pull istio "images", as the regex also matches on values + # from the generated istio deployment manifest + ) | grep -ioE "${image_regex_w_double_quotes}"'$' | grep -v '.svc:' >>"${temp_file}" + + sed -E 's/"//g' "${temp_file}" | sort | uniq +} + +main "$@" \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/k8s.conf b/cluster-provision/gocli/opts/k8sprovision/conf/k8s.conf new file mode 100644 index 0000000000..ab1c5d11b9 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/k8s.conf @@ -0,0 +1,5 @@ +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 +net.ipv6.conf.all.disable_ipv6 = 0 +net.ipv6.conf.all.forwarding = 1 +net.bridge.bridge-nf-call-ip6tables = 1 \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm.conf b/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm.conf new file mode 100644 index 0000000000..838aece1b0 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm.conf @@ -0,0 +1,54 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + token: abcdef.1234567890123456 + ttl: 24h0m0s + usages: + - signing + - authentication +kind: InitConfiguration +patches: + directory: /provision/kubeadm-patches +--- +apiServer: + extraArgs: + allow-privileged: "true" + audit-log-format: json + audit-log-path: /var/log/k8s-audit/k8s-audit.log + audit-policy-file: /etc/kubernetes/audit/adv-audit.yaml + enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota + admission-control-config-file: /etc/kubernetes/psa.yaml + extraVolumes: + - hostPath: /etc/kubernetes/psa.yaml + mountPath: /etc/kubernetes/psa.yaml + name: psa + - hostPath: /etc/kubernetes/audit + mountPath: /etc/kubernetes/audit + name: audit-conf + readOnly: true + - hostPath: /var/log/k8s-audit + mountPath: /var/log/k8s-audit + name: audit-log + timeoutForControlPlane: 4m0s +apiVersion: kubeadm.k8s.io/v1beta3 +certificatesDir: /etc/kubernetes/pki +clusterName: kubernetes +controllerManager: + extraArgs: + node-cidr-mask-size-ipv6: "116" +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kind: ClusterConfiguration +kubernetesVersion: vVERSION +networking: + dnsDomain: cluster.local + podSubnet: 10.244.0.0/16,fd10:244::/112 + serviceSubnet: 10.96.0.0/12,fd10:96::/108 +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +clusterCIDR: 10.244.0.0/16,fd10:244::/112 +mode: iptables diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm_ipv6.conf b/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm_ipv6.conf new file mode 100644 index 0000000000..90db90e86e --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/kubeadm_ipv6.conf @@ -0,0 +1,70 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + token: abcdef.1234567890123456 + ttl: 24h0m0s + usages: + - signing + - authentication +kind: InitConfiguration +patches: + directory: /provision/kubeadm-patches +localAPIEndpoint: + advertiseAddress: "::" + bindPort: 6443 +nodeRegistration: + kubeletExtraArgs: + node-ip: "::" +--- +apiServer: + extraArgs: + allow-privileged: "true" + audit-log-format: json + audit-log-path: /var/log/k8s-audit/k8s-audit.log + audit-policy-file: /etc/kubernetes/audit/adv-audit.yaml + enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota + bind-address: "::" + admission-control-config-file: /etc/kubernetes/psa.yaml + extraVolumes: + - hostPath: /etc/kubernetes/psa.yaml + mountPath: /etc/kubernetes/psa.yaml + name: psa + - hostPath: /etc/kubernetes/audit + mountPath: /etc/kubernetes/audit + name: audit-conf + readOnly: true + - hostPath: /var/log/k8s-audit + mountPath: /var/log/k8s-audit + name: audit-log + timeoutForControlPlane: 4m0s +apiVersion: kubeadm.k8s.io/v1beta3 +certificatesDir: /etc/kubernetes/pki +clusterName: kubernetes +controllerManager: + extraArgs: + bind-address: "::" + node-cidr-mask-size-ipv6: "116" +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kind: ClusterConfiguration +kubernetesVersion: vVERSION +networking: + dnsDomain: cluster.local + podSubnet: fd00:10:244::/112 + serviceSubnet: fd00:10:96::/112 +scheduler: + extraArgs: + bind-address: "::1" +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +clusterCIDR: fd00:10:244::/112 +mode: iptables +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +address: "::" +healthzBindAddress: "::" diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/kubernetes.repo b/cluster-provision/gocli/opts/k8sprovision/conf/kubernetes.repo new file mode 100644 index 0000000000..add2c15a20 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/kubernetes.repo @@ -0,0 +1,6 @@ +[kubernetes] +name=Kubernetes Release +baseurl=https://pkgs.k8s.io/core:/stable:/v1.VERSION/rpm +enabled=1 +gpgcheck=0 +repo_gpgcheck=0 \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/psa.yaml b/cluster-provision/gocli/opts/k8sprovision/conf/psa.yaml new file mode 100644 index 0000000000..a50d37db23 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/psa.yaml @@ -0,0 +1,20 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "privileged" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + # Hopefuly this will not be needed in future. Add your favorite namespace to be ignored and your operator not broken + # You also need to modify psa.sh + namespaces: ["kube-system", "default", "istio-operator" ,"istio-system", "nfs-csi", "monitoring", "rook-ceph", "cluster-network-addons", "sonobuoy"] \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/registries.conf b/cluster-provision/gocli/opts/k8sprovision/conf/registries.conf new file mode 100644 index 0000000000..9d3563970e --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/registries.conf @@ -0,0 +1,8 @@ +[registries.search] +registries = ["registry.access.redhat.com", "registry.fedoraproject.org", "quay.io", "docker.io"] + +[registries.insecure] +registries = ["registry:5000"] + +[registries.block] +registries = [] \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/conf/storage.conf b/cluster-provision/gocli/opts/k8sprovision/conf/storage.conf new file mode 100644 index 0000000000..ec4c9fee9c --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/conf/storage.conf @@ -0,0 +1,4 @@ +[storage] +driver = "vfs" +runroot = "/var/run/containers/storage" +graphroot = "/mnt/containers-storage" \ No newline at end of file diff --git a/cluster-provision/gocli/opts/k8sprovision/k8sprovision.go b/cluster-provision/gocli/opts/k8sprovision/k8sprovision.go new file mode 100644 index 0000000000..e09783bd3a --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/k8sprovision.go @@ -0,0 +1,301 @@ +package k8sprovision + +import ( + "embed" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/sirupsen/logrus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +//go:embed conf/* +var f embed.FS + +//go:embed patches/* +var patchFs embed.FS + +type k8sProvisioner struct { + version string + slim bool + sshClient libssh.Client +} + +func NewK8sProvisioner(sshClient libssh.Client, version string, slim bool) *k8sProvisioner { + return &k8sProvisioner{ + version: version, + slim: slim, + sshClient: sshClient, + } +} + +func (k *k8sProvisioner) Exec() error { + crio, err := f.ReadFile("conf/crio-yum.repo") + if err != nil { + return err + } + + registries, err := f.ReadFile("conf/registries.conf") + if err != nil { + return err + } + + storage, err := f.ReadFile("conf/storage.conf") + if err != nil { + return err + } + + k8sRepo, err := f.ReadFile("conf/kubernetes.repo") + if err != nil { + return err + } + + cniPatch, err := f.ReadFile("conf/cni.diff") + if err != nil { + return err + } + + cniV6Patch, err := f.ReadFile("conf/cni_ipv6.diff") + if err != nil { + return err + } + + k8sConf, err := f.ReadFile("conf/k8s.conf") + if err != nil { + return err + } + + calico, err := f.ReadFile("conf/001-calico.conf") + if err != nil { + return err + } + + dhclient, err := f.ReadFile("conf/002-dhclient.conf") + if err != nil { + return err + } + + secContextPatch, err := patchFs.ReadFile("patches/add-security-context-deployment-patch.yaml") + if err != nil { + return err + } + + etcdPatch, err := patchFs.ReadFile("patches/etcd.yaml") + if err != nil { + return err + } + + apiServerPatch, err := patchFs.ReadFile("patches/kube-apiserver.yaml") + if err != nil { + return err + } + + controllerManagerPatch, err := patchFs.ReadFile("patches/kube-controller-manager.yaml") + if err != nil { + return err + } + + schedulerPatch, err := patchFs.ReadFile("patches/kube-scheduler.yaml") + if err != nil { + return err + } + + packagesVersion, err := k.getPackagesVersion() + if err != nil { + return err + } + + advAudit, err := f.ReadFile("conf/adv-audit.yaml") + if err != nil { + return err + } + + psa, err := f.ReadFile("conf/psa.yaml") + if err != nil { + return err + } + + kubeAdm, err := f.ReadFile("conf/kubeadm.conf") + if err != nil { + return err + } + + kubeAdm6, err := f.ReadFile("conf/kubeadm_ipv6.conf") + if err != nil { + return err + } + + k8sMinor := strings.Split(k.version, ".")[1] + + crioWithVersion := strings.Replace(string(crio), "VERSION", k8sMinor, -1) + k8sRepoWithVersion := strings.Replace(string(k8sRepo), "VERSION", k8sMinor, -1) + kubeAdmConf := strings.Replace(string(kubeAdm), "VERSION", k.version, -1) + kubeAdm6Conf := strings.Replace(string(kubeAdm6), "VERSION", k.version, -1) + + cmds := []string{ + "echo '" + crioWithVersion + "' | tee /etc/yum.repos.d/devel_kubic_libcontainers_stable_cri-o_v1." + k8sMinor + ".repo >> /dev/null", + "dnf install -y cri-o", + "echo '" + string(registries) + "' | tee /etc/containers/registries.conf >> /dev/null", + "echo '" + string(storage) + "' | tee /etc/containers/storage.conf >> /dev/null", + "systemctl restart crio", + "systemctl enable --now crio", + "echo '" + k8sRepoWithVersion + "' | tee /etc/yum.repos.d/kubernetes.repo >> /dev/null", + fmt.Sprintf("dnf install --skip-broken --nobest --nogpgcheck --disableexcludes=kubernetes -y kubectl-%[1]s kubeadm-%[1]s kubelet-%[1]s kubernetes-cni", packagesVersion), + "kubeadm config images pull --kubernetes-version " + k.version, + `image_regex='([a-z0-9\_\.]+[/-]?)+(@sha256)?:[a-z0-9\_\.\-]+' image_regex_w_double_quotes='"?'"${image_regex}"'"?' find /tmp -type f -name '*.yaml' -print0 | xargs -0 grep -iE '(image|value): '"${image_regex_w_double_quotes}" > /tmp/images`, + } + + for _, cmd := range cmds { + if err := k.sshClient.Command(cmd); err != nil { + return err + } + } + + images, err := k.sshClient.CommandWithNoStdOut(`image_regex='([a-z0-9\_\.]+[/-]?)+(@sha256)?:[a-z0-9\_\.\-]+' && image_regex_w_double_quotes='"?'"${image_regex}"'"?' && grep -ioE "${image_regex_w_double_quotes}" /tmp/images`) + if err != nil { + return err + } + + if !k.slim { + imagesList := strings.Split(images, "\n") + for _, image := range imagesList { + err := k.pullImageRetry(image) + if err != nil { + logrus.Infof("Failed to pull image: %s, it will not be available offline", image) + } + } + + extraImg, err := f.ReadFile("conf/extra-images") + if err != nil { + return err + } + + imagesList = strings.Split(string(extraImg), "\n") + for _, image := range imagesList { + err := k.pullImageRetry(image) + if err != nil { + logrus.Infof("Failed to pull image: %s, it will not be available offline", image) + } + } + } + + cmds = []string{ + "mkdir /provision", + "yum install -y patch || true", + "dnf install -y patch || true", + "cp /tmp/cni.do-not-change.yaml /provision/cni.yaml", + "mv /tmp/cni.do-not-change.yaml /provision/cni_ipv6.yaml", + "echo '" + string(cniPatch) + "' | tee /tmp/cni_patch.diff >> /dev/null", + "echo '" + string(cniV6Patch) + "' | tee /tmp/cni_v6_patch.diff >> /dev/null", + "patch /provision/cni.yaml /tmp/cni_patch.diff", + "patch /provision/cni_ipv6.yaml /tmp/cni_v6_patch.diff", + "cp /tmp/local-volume.yaml /provision/local-volume.yaml", + "echo 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --fail-swap-on=false --kubelet-cgroups=/systemd/system.slice' >> /etc/sysconfig/kubelet", + `echo "vm.unprivileged_userfaultfd = 1" > /etc/sysctl.d/enable-userfaultfd.conf`, + "modprobe bridge", + "modprobe overlay", + "modprobe br_netfilter", + "echo '" + string(k8sConf) + "' | tee /etc/sysctl.d/k8s.conf >> /dev/null", + "sysctl --system", + "echo bridge >> /etc/modules-load.d/k8s.conf", + "echo br_netfilter >> /etc/modules-load.d/k8s.conf", + "echo overlay >> /etc/modules-load.d/k8s.conf", + "rm -f /etc/cni/net.d/*", + "systemctl daemon-reload", + "systemctl enable crio kubelet --now", + "echo '" + string(calico) + "' | tee /etc/NetworkManager/conf.d/001-calico.conf >> /dev/null", + "echo '" + string(dhclient) + "' | tee /etc/NetworkManager/conf.d/002-dhclient.conf >> /dev/null", + `echo "net.netfilter.nf_conntrack_max=1000000" >> /etc/sysctl.conf`, + "sysctl --system", + "systemctl restart NetworkManager", + `nmcli connection modify "System eth0" ipv6.method auto ipv6.addr-gen-mode eui64`, + `nmcli connection up "System eth0"`, + "sysctl --system", + "mkdir -p /provision/kubeadm-patches", + "echo '" + string(secContextPatch) + "' | tee /provision/kubeadm-patches/add-security-context-deployment-patch.yaml >> /dev/null", + "echo '" + string(etcdPatch) + "' | tee /provision/kubeadm-patches/etcd.yaml >> /dev/null", + "echo '" + string(apiServerPatch) + "' | tee /provision/kubeadm-patches/kube-apiserver.yaml >> /dev/null", + "echo '" + string(controllerManagerPatch) + "' | tee /provision/kubeadm-patches/kube-controller-manager.yaml >> /dev/null", + "echo '" + string(schedulerPatch) + "' | tee /provision/kubeadm-patches/kube-scheduler.yaml >> /dev/null", + "mkdir /etc/kubernetes/audit", + "echo '" + string(advAudit) + "' | tee /etc/kubernetes/audit/adv-audit.yaml >> /dev/null", + "echo '" + string(psa) + "' | tee /etc/kubernetes/psa.yaml >> /dev/null", + "echo '" + kubeAdmConf + "' | tee /etc/kubernetes/kubeadm.conf >> /dev/null", + "echo '" + kubeAdm6Conf + "' | tee /etc/kubernetes/kubeadm_ipv6.conf >> /dev/null", + "until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done", + "swapoff -a", + "systemctl restart kubelet", + "kubeadm init --config /etc/kubernetes/kubeadm.conf -v5", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf patch deployment coredns -n kube-system -p '" + string(secContextPatch) + "'", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f /provision/cni.yaml", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf wait --for=condition=Ready pods --all -n kube-system --timeout=300s", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system", + "kubeadm reset --force", + `for i in {1..10}; do mkdir -p /var/local/kubevirt-storage/local-volume/disk${i} && mkdir -p /mnt/local-storage/local/disk${i} && echo "/var/local/kubevirt-storage/local-volume/disk${i} /mnt/local-storage/local/disk${i} none defaults,bind 0 0" >> /etc/fstab; done`, + "chmod -R 777 /var/local/kubevirt-storage/local-volume", + "chcon -R unconfined_u:object_r:svirt_sandbox_file_t:s0 /mnt/local-storage/", + "mkdir -p /var/provision/kubevirt.io/tests", + "chcon -t container_file_t /var/provision/kubevirt.io/tests", + `echo "tmpfs /var/provision/kubevirt.io/tests tmpfs rw,context=system_u:object_r:container_file_t:s0 0 1" >> /etc/fstab`, + "cp -rf /tmp/kwok /opt/", + "rm -f /etc/sysconfig/network-scripts/ifcfg-*", + "nmcli connection add con-name eth0 ifname eth0 type ethernet", + "rm -f /etc/machine-id ; touch /etc/machine-id", + } + + for _, cmd := range cmds { + if err := k.sshClient.Command(cmd); err != nil { + return err + } + } + + return nil +} + +func (k *k8sProvisioner) pullImageRetry(image string) error { + maxRetries := 5 + downloaded := false + + for i := 0; i < maxRetries; i++ { + if err := k.sshClient.Command("crictl pull " + image); err != nil { + logrus.Infof("Attempt [%d]: Failed to download image %s: %s, sleeping 3 seconds and trying again", i+1, image, err.Error()) + time.Sleep(time.Second * 3) + } else { + downloaded = true + break + } + } + + if !downloaded { + return fmt.Errorf("reached max retries to download for %s", image) + } + return nil +} + +func (k *k8sProvisioner) getPackagesVersion() (string, error) { + packagesVersion := k.version + if strings.HasSuffix(k.version, "alpha") || strings.HasSuffix(k.version, "beta") || strings.HasSuffix(k.version, "rc") { + k8sversion := strings.Split(k.version, ".") + + url := fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/stable-%s.%s.txt", k8sversion[0], k8sversion[1]) + resp, err := http.Get(url) + if err != nil { + return packagesVersion, nil + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading the response body:", err) + return packagesVersion, nil + } + if string(body) != "" { + packagesVersion = strings.TrimPrefix(string(body), "v") + } + } + return packagesVersion, nil +} diff --git a/cluster-provision/gocli/opts/k8sprovision/k8sprovision_test.go b/cluster-provision/gocli/opts/k8sprovision/k8sprovision_test.go new file mode 100644 index 0000000000..59f59f8053 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/k8sprovision_test.go @@ -0,0 +1,39 @@ +package k8sprovision + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func TestK8sProvision(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "K8s provision test suite") +} + +var _ = Describe("K8s provision", func() { + var ( + mockCtrl *gomock.Controller + sshClient *kubevirtcimocks.MockSSHClient + opt *k8sProvisioner + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) + opt := NewK8sProvisioner(sshClient, "1.30", true) + AddExpectCalls(sshClient, opt.version, opt.slim) + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + It("should provision k8s successfully", func() { + err := opt.Exec() + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/cluster-provision/gocli/opts/k8sprovision/patches/add-security-context-deployment-patch.yaml b/cluster-provision/gocli/opts/k8sprovision/patches/add-security-context-deployment-patch.yaml new file mode 100644 index 0000000000..38d9b0e65c --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/patches/add-security-context-deployment-patch.yaml @@ -0,0 +1,6 @@ +spec: + template: + spec: + securityContext: + seLinuxOptions: + type: spc_t diff --git a/cluster-provision/gocli/opts/k8sprovision/patches/etcd.yaml b/cluster-provision/gocli/opts/k8sprovision/patches/etcd.yaml new file mode 100644 index 0000000000..58ec1a7a53 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/patches/etcd.yaml @@ -0,0 +1,4 @@ +spec: + securityContext: + seLinuxOptions: + type: spc_t diff --git a/cluster-provision/gocli/opts/k8sprovision/patches/kube-apiserver.yaml b/cluster-provision/gocli/opts/k8sprovision/patches/kube-apiserver.yaml new file mode 100644 index 0000000000..58ec1a7a53 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/patches/kube-apiserver.yaml @@ -0,0 +1,4 @@ +spec: + securityContext: + seLinuxOptions: + type: spc_t diff --git a/cluster-provision/gocli/opts/k8sprovision/patches/kube-controller-manager.yaml b/cluster-provision/gocli/opts/k8sprovision/patches/kube-controller-manager.yaml new file mode 100644 index 0000000000..58ec1a7a53 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/patches/kube-controller-manager.yaml @@ -0,0 +1,4 @@ +spec: + securityContext: + seLinuxOptions: + type: spc_t diff --git a/cluster-provision/gocli/opts/k8sprovision/patches/kube-scheduler.yaml b/cluster-provision/gocli/opts/k8sprovision/patches/kube-scheduler.yaml new file mode 100644 index 0000000000..58ec1a7a53 --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/patches/kube-scheduler.yaml @@ -0,0 +1,4 @@ +spec: + securityContext: + seLinuxOptions: + type: spc_t diff --git a/cluster-provision/gocli/opts/k8sprovision/testconfig.go b/cluster-provision/gocli/opts/k8sprovision/testconfig.go new file mode 100644 index 0000000000..66dd52b02c --- /dev/null +++ b/cluster-provision/gocli/opts/k8sprovision/testconfig.go @@ -0,0 +1,122 @@ +package k8sprovision + +import ( + "fmt" + "strings" + + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func AddExpectCalls(sshClient *kubevirtcimocks.MockSSHClient, version string, slim bool) { + crio, _ := f.ReadFile("conf/crio-yum.repo") + registries, _ := f.ReadFile("conf/registries.conf") + storage, _ := f.ReadFile("conf/storage.conf") + k8sRepo, _ := f.ReadFile("conf/kubernetes.repo") + cniPatch, _ := f.ReadFile("conf/cni.diff") + cniV6Patch, _ := f.ReadFile("conf/cni_ipv6.diff") + k8sConf, _ := f.ReadFile("conf/k8s.conf") + calico, _ := f.ReadFile("conf/001-calico.conf") + dhclient, _ := f.ReadFile("conf/002-dhclient.conf") + secContextPatch, _ := patchFs.ReadFile("patches/add-security-context-deployment-patch.yaml") + etcdPatch, _ := patchFs.ReadFile("patches/etcd.yaml") + apiServerPatch, _ := patchFs.ReadFile("patches/kube-apiserver.yaml") + controllerManagerPatch, _ := patchFs.ReadFile("patches/kube-controller-manager.yaml") + schedulerPatch, _ := patchFs.ReadFile("patches/kube-scheduler.yaml") + + packagesVersion := "1.30" + + advAudit, _ := f.ReadFile("conf/adv-audit.yaml") + psa, _ := f.ReadFile("conf/psa.yaml") + kubeAdm, _ := f.ReadFile("conf/kubeadm.conf") + kubeAdm6, _ := f.ReadFile("conf/kubeadm_ipv6.conf") + + k8sMinor := strings.Split(version, ".")[1] + k8sRepoWithVersion := strings.Replace(string(k8sRepo), "VERSION", k8sMinor, 1) + kubeAdmConf := strings.Replace(string(kubeAdm), "VERSION", version, 1) + kubeAdm6Conf := strings.Replace(string(kubeAdm6), "VERSION", version, 1) + + cmds := []string{ + "echo '" + string(crio) + "' | tee /etc/yum.repos.d/devel_kubic_libcontainers_stable_cri-o_v1.28.repo >> /dev/null", + "dnf install -y cri-o", + "echo '" + string(registries) + "' | tee /etc/containers/registries.conf >> /dev/null", + "echo '" + string(storage) + "' | tee /etc/containers/storage.conf >> /dev/null", + "systemctl restart crio", + "systemctl enable --now crio", + "echo '" + k8sRepoWithVersion + "' | tee /etc/yum.repos.d/kubernetes.repo >> /dev/null", + fmt.Sprintf("dnf install --skip-broken --nobest --nogpgcheck --disableexcludes=kubernetes -y kubectl-%[1]s kubeadm-%[1]s kubelet-%[1]s kubernetes-cni", packagesVersion), + "kubeadm config images pull --kubernetes-version " + version, + `image_regex='([a-z0-9\_\.]+[/-]?)+(@sha256)?:[a-z0-9\_\.\-]+' image_regex_w_double_quotes='"?'"${image_regex}"'"?' find /tmp -type f -name '*.yaml' -print0 | xargs -0 grep -iE '(image|value): '"${image_regex_w_double_quotes}" > /tmp/images`, + } + + for _, cmd := range cmds { + sshClient.EXPECT().Command(cmd) + } + + sshClient.EXPECT().CommandWithNoStdOut(`image_regex='([a-z0-9\_\.]+[/-]?)+(@sha256)?:[a-z0-9\_\.\-]+' && image_regex_w_double_quotes='"?'"${image_regex}"'"?' && grep -ioE "${image_regex_w_double_quotes}" /tmp/images`).Return("nginx:latest", nil) + + cmds = []string{ + "mkdir /provision", + "yum install -y patch || true", + "dnf install -y patch || true", + "cp /tmp/cni.do-not-change.yaml /provision/cni.yaml", + "mv /tmp/cni.do-not-change.yaml /provision/cni_ipv6.yaml", + "echo '" + string(cniPatch) + "' | tee /tmp/cni_patch.diff >> /dev/null", + "echo '" + string(cniV6Patch) + "' | tee /tmp/cni_v6_patch.diff >> /dev/null", + "patch /provision/cni.yaml /tmp/cni_patch.diff", + "patch /provision/cni_ipv6.yaml /tmp/cni_v6_patch.diff", + "cp /tmp/local-volume.yaml /provision/local-volume.yaml", + `echo "vm.unprivileged_userfaultfd = 1" > /etc/sysctl.d/enable-userfaultfd.conf`, + "modprobe bridge", + "modprobe overlay", + "modprobe br_netfilter", + "echo '" + string(k8sConf) + "' | tee /etc/sysctl.d/k8s.conf >> /dev/null", + "sysctl --system", + "echo bridge >> /etc/modules-load.d/k8s.conf", + "echo br_netfilter >> /etc/modules-load.d/k8s.conf", + "echo overlay >> /etc/modules-load.d/k8s.conf", + "rm -f /etc/cni/net.d/*", + "systemctl daemon-reload", + "systemctl enable crio kubelet --now", + "echo '" + string(calico) + "' | tee /etc/NetworkManager/conf.d/001-calico.conf >> /dev/null", + "echo '" + string(dhclient) + "' | tee /etc/NetworkManager/conf.d/002-dhclient.conf >> /dev/null", + `echo "net.netfilter.nf_conntrack_max=1000000" >> /etc/sysctl.conf`, + "sysctl --system", + "systemctl restart NetworkManager", + `nmcli connection modify "System eth0" ipv6.method auto ipv6.addr-gen-mode eui64`, + `nmcli connection up "System eth0"`, + "sysctl --system", + "echo bridge >> /etc/modules-load.d/k8s.conf", + "echo br_netfilter >> /etc/modules-load.d/k8s.conf", + "echo overlay >> /etc/modules-load.d/k8s.conf", + "mkdir -p /provision/kubeadm-patches", + "echo '" + string(secContextPatch) + "' | tee /provision/kubeadm-patches/add-security-context-deployment-patch.yaml >> /dev/null", + "echo '" + string(etcdPatch) + "' | tee /provision/kubeadm-patches/etcd.yaml >> /dev/null", + "echo '" + string(apiServerPatch) + "' | tee /provision/kubeadm-patches/kube-apiserver.yaml >> /dev/null", + "echo '" + string(controllerManagerPatch) + "' | tee /provision/kubeadm-patches/kube-controller-manager.yaml >> /dev/null", + "echo '" + string(schedulerPatch) + "' | tee /provision/kubeadm-patches/kube-scheduler.yaml >> /dev/null", + "mkdir /etc/kubernetes/audit", + "echo '" + string(advAudit) + "' | tee /etc/kubernetes/audit/adv-audit.yaml >> /dev/null", + "echo '" + string(psa) + "' | tee /etc/kubernetes/psa.yaml >> /dev/null", + "echo '" + kubeAdmConf + "' | tee /etc/kubernetes/kubeadm.conf >> /dev/null", + "echo '" + kubeAdm6Conf + "' | tee /etc/kubernetes/kubeadm_ipv6.conf >> /dev/null", + "until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done", + "swapoff -a", + "systemctl restart kubelet", + "kubeadm init --config /etc/kubernetes/kubeadm.conf -v5", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf patch deployment coredns -n kube-system -p '" + string(secContextPatch) + "'", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f /provision/cni.yaml", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf wait --for=condition=Ready pods --all -n kube-system --timeout=300s", + "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system", + "kubeadm reset --force", + "mkdir -p /var/provision/kubevirt.io/tests", + "chcon -t container_file_t /var/provision/kubevirt.io/tests", + `echo "tmpfs /var/provision/kubevirt.io/tests tmpfs rw,context=system_u:object_r:container_file_t:s0 0 1" >> /etc/fstab`, + "rm -f /etc/sysconfig/network-scripts/ifcfg-*", + "nmcli connection add con-name eth0 ifname eth0 type ethernet", + "rm -f /etc/machine-id ; touch /etc/machine-id", + } + + for _, cmd := range cmds { + sshClient.EXPECT().Command(cmd) + } +} diff --git a/cluster-provision/gocli/opts/labelnodes/labelnodes.go b/cluster-provision/gocli/opts/labelnodes/labelnodes.go index 75c0393cf1..d3203756ff 100644 --- a/cluster-provision/gocli/opts/labelnodes/labelnodes.go +++ b/cluster-provision/gocli/opts/labelnodes/labelnodes.go @@ -9,7 +9,7 @@ type nodeLabler struct { labelSelector string } -func NewNodeLabler(sc libssh.Client, p uint16, l string) *nodeLabler { +func NewNodeLabler(sc libssh.Client, l string) *nodeLabler { return &nodeLabler{ sshClient: sc, labelSelector: l, diff --git a/cluster-provision/gocli/opts/labelnodes/labelnodes_test.go b/cluster-provision/gocli/opts/labelnodes/labelnodes_test.go new file mode 100644 index 0000000000..4aa810966e --- /dev/null +++ b/cluster-provision/gocli/opts/labelnodes/labelnodes_test.go @@ -0,0 +1,39 @@ +package labelnodes + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func TestNodeLabel(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "NodeLabeler test suite") +} + +var _ = Describe("NodeLabeler", func() { + var ( + mockCtrl *gomock.Controller + sshClient *kubevirtcimocks.MockSSHClient + opt *nodeLabler + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) + opt = NewNodeLabler(sshClient, "node-role.kubernetes.io/control-plane") + AddExpectCalls(sshClient, "node-role.kubernetes.io/control-plane") + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + It("should label nodes successfully", func() { + err := opt.Exec() + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/cluster-provision/gocli/opts/labelnodes/testconfig.go b/cluster-provision/gocli/opts/labelnodes/testconfig.go new file mode 100644 index 0000000000..1e1ee46b86 --- /dev/null +++ b/cluster-provision/gocli/opts/labelnodes/testconfig.go @@ -0,0 +1,9 @@ +package labelnodes + +import ( + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func AddExpectCalls(sshClient *kubevirtcimocks.MockSSHClient, label string) { + sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf label node -l " + label + " node-role.kubernetes.io/worker=''") +} diff --git a/cluster-provision/gocli/opts/provision/provision.go b/cluster-provision/gocli/opts/provision/provision.go index f59cb5d45f..0038dc49b0 100644 --- a/cluster-provision/gocli/opts/provision/provision.go +++ b/cluster-provision/gocli/opts/provision/provision.go @@ -22,7 +22,7 @@ func NewLinuxProvisioner(sc libssh.Client) *linuxProvisioner { func (l *linuxProvisioner) Exec() error { cmds := []string{ - `echo '` + string(sharedVars) + `' | tee /var/lib/kubevirtci/shared_vars.sh > /dev/null`, + `mkdir -p /var/lib/kubevirtci && echo '` + string(sharedVars) + `' | tee /var/lib/kubevirtci/shared_vars.sh > /dev/null`, `dnf install -y "kernel-modules-$(uname -r)"`, "dnf install -y cloud-utils-growpart", `if growpart /dev/vda 1; then resize2fs /dev/vda1; fi`, diff --git a/cluster-provision/gocli/opts/provision/provision_test.go b/cluster-provision/gocli/opts/provision/provision_test.go new file mode 100644 index 0000000000..bfb8435eb1 --- /dev/null +++ b/cluster-provision/gocli/opts/provision/provision_test.go @@ -0,0 +1,39 @@ +package provision + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func TestLinuxProvisioner(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Linux provision test suite") +} + +var _ = Describe("Linux provision", func() { + var ( + mockCtrl *gomock.Controller + sshClient *kubevirtcimocks.MockSSHClient + opt *linuxProvisioner + ) + + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) + opt = NewLinuxProvisioner(sshClient) + AddExpectCalls(sshClient) + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + It("should provision linux successfully", func() { + err := opt.Exec() + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/cluster-provision/gocli/opts/provision/testconfig.go b/cluster-provision/gocli/opts/provision/testconfig.go new file mode 100644 index 0000000000..ea74be7b13 --- /dev/null +++ b/cluster-provision/gocli/opts/provision/testconfig.go @@ -0,0 +1,35 @@ +package provision + +import ( + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func AddExpectCalls(sshClient *kubevirtcimocks.MockSSHClient) { + cmds := []string{ + `mkdir -p /var/lib/kubevirtci && echo '` + string(sharedVars) + `' | tee /var/lib/kubevirtci/shared_vars.sh > /dev/null`, + `dnf install -y "kernel-modules-$(uname -r)"`, + "dnf install -y cloud-utils-growpart", + `if growpart /dev/vda 1; then resize2fs /dev/vda1; fi`, + "dnf install -y patch", + "systemctl stop firewalld || :", + "systemctl disable firewalld || :", + "dnf -y remove firewalld", + "dnf -y install iscsi-initiator-utils", + "dnf -y install nftables", + "dnf -y install lvm2", + `echo 'ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="vd[a-z]", ATTR{queue/rotational}="0"' > /etc/udev/rules.d/60-force-ssd-rotational.rules`, + "dnf install -y iproute-tc", + "mkdir -p /opt/istio-1.15.0/bin", + `curl "https://storage.googleapis.com/kubevirtci-istioctl-mirror/istio-1.15.0/bin/istioctl" -o "/opt/istio-1.15.0/bin/istioctl"`, + `chmod +x /opt/istio-1.15.0/bin/istioctl`, + "dnf install -y container-selinux", + "dnf install -y libseccomp-devel", + "dnf install -y centos-release-nfv-openvswitch", + "dnf install -y openvswitch2.16", + "dnf install -y NetworkManager NetworkManager-ovs NetworkManager-config-server", + } + + for _, cmd := range cmds { + sshClient.EXPECT().Command(cmd) + } +} diff --git a/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/LICENSE b/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/LICENSE deleted file mode 100644 index c67dad612a..0000000000 --- a/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2013, Patrick Mezard -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - The names of its contributors may not be used to endorse or promote -products derived from this software without specific prior written -permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/difflib/difflib.go b/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/difflib/difflib.go deleted file mode 100644 index 003e99fadb..0000000000 --- a/cluster-provision/gocli/vendor/github.com/pmezard/go-difflib/difflib/difflib.go +++ /dev/null @@ -1,772 +0,0 @@ -// Package difflib is a partial port of Python difflib module. -// -// It provides tools to compare sequences of strings and generate textual diffs. -// -// The following class and functions have been ported: -// -// - SequenceMatcher -// -// - unified_diff -// -// - context_diff -// -// Getting unified diffs was the main goal of the port. Keep in mind this code -// is mostly suitable to output text differences in a human friendly way, there -// are no guarantees generated diffs are consumable by patch(1). -package difflib - -import ( - "bufio" - "bytes" - "fmt" - "io" - "strings" -) - -func min(a, b int) int { - if a < b { - return a - } - return b -} - -func max(a, b int) int { - if a > b { - return a - } - return b -} - -func calculateRatio(matches, length int) float64 { - if length > 0 { - return 2.0 * float64(matches) / float64(length) - } - return 1.0 -} - -type Match struct { - A int - B int - Size int -} - -type OpCode struct { - Tag byte - I1 int - I2 int - J1 int - J2 int -} - -// SequenceMatcher compares sequence of strings. The basic -// algorithm predates, and is a little fancier than, an algorithm -// published in the late 1980's by Ratcliff and Obershelp under the -// hyperbolic name "gestalt pattern matching". The basic idea is to find -// the longest contiguous matching subsequence that contains no "junk" -// elements (R-O doesn't address junk). The same idea is then applied -// recursively to the pieces of the sequences to the left and to the right -// of the matching subsequence. This does not yield minimal edit -// sequences, but does tend to yield matches that "look right" to people. -// -// SequenceMatcher tries to compute a "human-friendly diff" between two -// sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the -// longest *contiguous* & junk-free matching subsequence. That's what -// catches peoples' eyes. The Windows(tm) windiff has another interesting -// notion, pairing up elements that appear uniquely in each sequence. -// That, and the method here, appear to yield more intuitive difference -// reports than does diff. This method appears to be the least vulnerable -// to synching up on blocks of "junk lines", though (like blank lines in -// ordinary text files, or maybe "

" lines in HTML files). That may be -// because this is the only method of the 3 that has a *concept* of -// "junk" . -// -// Timing: Basic R-O is cubic time worst case and quadratic time expected -// case. SequenceMatcher is quadratic time for the worst case and has -// expected-case behavior dependent in a complicated way on how many -// elements the sequences have in common; best case time is linear. -type SequenceMatcher struct { - a []string - b []string - b2j map[string][]int - IsJunk func(string) bool - autoJunk bool - bJunk map[string]struct{} - matchingBlocks []Match - fullBCount map[string]int - bPopular map[string]struct{} - opCodes []OpCode -} - -func NewMatcher(a, b []string) *SequenceMatcher { - m := SequenceMatcher{autoJunk: true} - m.SetSeqs(a, b) - return &m -} - -func NewMatcherWithJunk(a, b []string, autoJunk bool, - isJunk func(string) bool) *SequenceMatcher { - - m := SequenceMatcher{IsJunk: isJunk, autoJunk: autoJunk} - m.SetSeqs(a, b) - return &m -} - -// Set two sequences to be compared. -func (m *SequenceMatcher) SetSeqs(a, b []string) { - m.SetSeq1(a) - m.SetSeq2(b) -} - -// Set the first sequence to be compared. The second sequence to be compared is -// not changed. -// -// SequenceMatcher computes and caches detailed information about the second -// sequence, so if you want to compare one sequence S against many sequences, -// use .SetSeq2(s) once and call .SetSeq1(x) repeatedly for each of the other -// sequences. -// -// See also SetSeqs() and SetSeq2(). -func (m *SequenceMatcher) SetSeq1(a []string) { - if &a == &m.a { - return - } - m.a = a - m.matchingBlocks = nil - m.opCodes = nil -} - -// Set the second sequence to be compared. The first sequence to be compared is -// not changed. -func (m *SequenceMatcher) SetSeq2(b []string) { - if &b == &m.b { - return - } - m.b = b - m.matchingBlocks = nil - m.opCodes = nil - m.fullBCount = nil - m.chainB() -} - -func (m *SequenceMatcher) chainB() { - // Populate line -> index mapping - b2j := map[string][]int{} - for i, s := range m.b { - indices := b2j[s] - indices = append(indices, i) - b2j[s] = indices - } - - // Purge junk elements - m.bJunk = map[string]struct{}{} - if m.IsJunk != nil { - junk := m.bJunk - for s, _ := range b2j { - if m.IsJunk(s) { - junk[s] = struct{}{} - } - } - for s, _ := range junk { - delete(b2j, s) - } - } - - // Purge remaining popular elements - popular := map[string]struct{}{} - n := len(m.b) - if m.autoJunk && n >= 200 { - ntest := n/100 + 1 - for s, indices := range b2j { - if len(indices) > ntest { - popular[s] = struct{}{} - } - } - for s, _ := range popular { - delete(b2j, s) - } - } - m.bPopular = popular - m.b2j = b2j -} - -func (m *SequenceMatcher) isBJunk(s string) bool { - _, ok := m.bJunk[s] - return ok -} - -// Find longest matching block in a[alo:ahi] and b[blo:bhi]. -// -// If IsJunk is not defined: -// -// Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where -// alo <= i <= i+k <= ahi -// blo <= j <= j+k <= bhi -// and for all (i',j',k') meeting those conditions, -// k >= k' -// i <= i' -// and if i == i', j <= j' -// -// In other words, of all maximal matching blocks, return one that -// starts earliest in a, and of all those maximal matching blocks that -// start earliest in a, return the one that starts earliest in b. -// -// If IsJunk is defined, first the longest matching block is -// determined as above, but with the additional restriction that no -// junk element appears in the block. Then that block is extended as -// far as possible by matching (only) junk elements on both sides. So -// the resulting block never matches on junk except as identical junk -// happens to be adjacent to an "interesting" match. -// -// If no blocks match, return (alo, blo, 0). -func (m *SequenceMatcher) findLongestMatch(alo, ahi, blo, bhi int) Match { - // CAUTION: stripping common prefix or suffix would be incorrect. - // E.g., - // ab - // acab - // Longest matching block is "ab", but if common prefix is - // stripped, it's "a" (tied with "b"). UNIX(tm) diff does so - // strip, so ends up claiming that ab is changed to acab by - // inserting "ca" in the middle. That's minimal but unintuitive: - // "it's obvious" that someone inserted "ac" at the front. - // Windiff ends up at the same place as diff, but by pairing up - // the unique 'b's and then matching the first two 'a's. - besti, bestj, bestsize := alo, blo, 0 - - // find longest junk-free match - // during an iteration of the loop, j2len[j] = length of longest - // junk-free match ending with a[i-1] and b[j] - j2len := map[int]int{} - for i := alo; i != ahi; i++ { - // look at all instances of a[i] in b; note that because - // b2j has no junk keys, the loop is skipped if a[i] is junk - newj2len := map[int]int{} - for _, j := range m.b2j[m.a[i]] { - // a[i] matches b[j] - if j < blo { - continue - } - if j >= bhi { - break - } - k := j2len[j-1] + 1 - newj2len[j] = k - if k > bestsize { - besti, bestj, bestsize = i-k+1, j-k+1, k - } - } - j2len = newj2len - } - - // Extend the best by non-junk elements on each end. In particular, - // "popular" non-junk elements aren't in b2j, which greatly speeds - // the inner loop above, but also means "the best" match so far - // doesn't contain any junk *or* popular non-junk elements. - for besti > alo && bestj > blo && !m.isBJunk(m.b[bestj-1]) && - m.a[besti-1] == m.b[bestj-1] { - besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 - } - for besti+bestsize < ahi && bestj+bestsize < bhi && - !m.isBJunk(m.b[bestj+bestsize]) && - m.a[besti+bestsize] == m.b[bestj+bestsize] { - bestsize += 1 - } - - // Now that we have a wholly interesting match (albeit possibly - // empty!), we may as well suck up the matching junk on each - // side of it too. Can't think of a good reason not to, and it - // saves post-processing the (possibly considerable) expense of - // figuring out what to do with it. In the case of an empty - // interesting match, this is clearly the right thing to do, - // because no other kind of match is possible in the regions. - for besti > alo && bestj > blo && m.isBJunk(m.b[bestj-1]) && - m.a[besti-1] == m.b[bestj-1] { - besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 - } - for besti+bestsize < ahi && bestj+bestsize < bhi && - m.isBJunk(m.b[bestj+bestsize]) && - m.a[besti+bestsize] == m.b[bestj+bestsize] { - bestsize += 1 - } - - return Match{A: besti, B: bestj, Size: bestsize} -} - -// Return list of triples describing matching subsequences. -// -// Each triple is of the form (i, j, n), and means that -// a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in -// i and in j. It's also guaranteed that if (i, j, n) and (i', j', n') are -// adjacent triples in the list, and the second is not the last triple in the -// list, then i+n != i' or j+n != j'. IOW, adjacent triples never describe -// adjacent equal blocks. -// -// The last triple is a dummy, (len(a), len(b), 0), and is the only -// triple with n==0. -func (m *SequenceMatcher) GetMatchingBlocks() []Match { - if m.matchingBlocks != nil { - return m.matchingBlocks - } - - var matchBlocks func(alo, ahi, blo, bhi int, matched []Match) []Match - matchBlocks = func(alo, ahi, blo, bhi int, matched []Match) []Match { - match := m.findLongestMatch(alo, ahi, blo, bhi) - i, j, k := match.A, match.B, match.Size - if match.Size > 0 { - if alo < i && blo < j { - matched = matchBlocks(alo, i, blo, j, matched) - } - matched = append(matched, match) - if i+k < ahi && j+k < bhi { - matched = matchBlocks(i+k, ahi, j+k, bhi, matched) - } - } - return matched - } - matched := matchBlocks(0, len(m.a), 0, len(m.b), nil) - - // It's possible that we have adjacent equal blocks in the - // matching_blocks list now. - nonAdjacent := []Match{} - i1, j1, k1 := 0, 0, 0 - for _, b := range matched { - // Is this block adjacent to i1, j1, k1? - i2, j2, k2 := b.A, b.B, b.Size - if i1+k1 == i2 && j1+k1 == j2 { - // Yes, so collapse them -- this just increases the length of - // the first block by the length of the second, and the first - // block so lengthened remains the block to compare against. - k1 += k2 - } else { - // Not adjacent. Remember the first block (k1==0 means it's - // the dummy we started with), and make the second block the - // new block to compare against. - if k1 > 0 { - nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) - } - i1, j1, k1 = i2, j2, k2 - } - } - if k1 > 0 { - nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) - } - - nonAdjacent = append(nonAdjacent, Match{len(m.a), len(m.b), 0}) - m.matchingBlocks = nonAdjacent - return m.matchingBlocks -} - -// Return list of 5-tuples describing how to turn a into b. -// -// Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple -// has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the -// tuple preceding it, and likewise for j1 == the previous j2. -// -// The tags are characters, with these meanings: -// -// 'r' (replace): a[i1:i2] should be replaced by b[j1:j2] -// -// 'd' (delete): a[i1:i2] should be deleted, j1==j2 in this case. -// -// 'i' (insert): b[j1:j2] should be inserted at a[i1:i1], i1==i2 in this case. -// -// 'e' (equal): a[i1:i2] == b[j1:j2] -func (m *SequenceMatcher) GetOpCodes() []OpCode { - if m.opCodes != nil { - return m.opCodes - } - i, j := 0, 0 - matching := m.GetMatchingBlocks() - opCodes := make([]OpCode, 0, len(matching)) - for _, m := range matching { - // invariant: we've pumped out correct diffs to change - // a[:i] into b[:j], and the next matching block is - // a[ai:ai+size] == b[bj:bj+size]. So we need to pump - // out a diff to change a[i:ai] into b[j:bj], pump out - // the matching block, and move (i,j) beyond the match - ai, bj, size := m.A, m.B, m.Size - tag := byte(0) - if i < ai && j < bj { - tag = 'r' - } else if i < ai { - tag = 'd' - } else if j < bj { - tag = 'i' - } - if tag > 0 { - opCodes = append(opCodes, OpCode{tag, i, ai, j, bj}) - } - i, j = ai+size, bj+size - // the list of matching blocks is terminated by a - // sentinel with size 0 - if size > 0 { - opCodes = append(opCodes, OpCode{'e', ai, i, bj, j}) - } - } - m.opCodes = opCodes - return m.opCodes -} - -// Isolate change clusters by eliminating ranges with no changes. -// -// Return a generator of groups with up to n lines of context. -// Each group is in the same format as returned by GetOpCodes(). -func (m *SequenceMatcher) GetGroupedOpCodes(n int) [][]OpCode { - if n < 0 { - n = 3 - } - codes := m.GetOpCodes() - if len(codes) == 0 { - codes = []OpCode{OpCode{'e', 0, 1, 0, 1}} - } - // Fixup leading and trailing groups if they show no changes. - if codes[0].Tag == 'e' { - c := codes[0] - i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 - codes[0] = OpCode{c.Tag, max(i1, i2-n), i2, max(j1, j2-n), j2} - } - if codes[len(codes)-1].Tag == 'e' { - c := codes[len(codes)-1] - i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 - codes[len(codes)-1] = OpCode{c.Tag, i1, min(i2, i1+n), j1, min(j2, j1+n)} - } - nn := n + n - groups := [][]OpCode{} - group := []OpCode{} - for _, c := range codes { - i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 - // End the current group and start a new one whenever - // there is a large range with no changes. - if c.Tag == 'e' && i2-i1 > nn { - group = append(group, OpCode{c.Tag, i1, min(i2, i1+n), - j1, min(j2, j1+n)}) - groups = append(groups, group) - group = []OpCode{} - i1, j1 = max(i1, i2-n), max(j1, j2-n) - } - group = append(group, OpCode{c.Tag, i1, i2, j1, j2}) - } - if len(group) > 0 && !(len(group) == 1 && group[0].Tag == 'e') { - groups = append(groups, group) - } - return groups -} - -// Return a measure of the sequences' similarity (float in [0,1]). -// -// Where T is the total number of elements in both sequences, and -// M is the number of matches, this is 2.0*M / T. -// Note that this is 1 if the sequences are identical, and 0 if -// they have nothing in common. -// -// .Ratio() is expensive to compute if you haven't already computed -// .GetMatchingBlocks() or .GetOpCodes(), in which case you may -// want to try .QuickRatio() or .RealQuickRation() first to get an -// upper bound. -func (m *SequenceMatcher) Ratio() float64 { - matches := 0 - for _, m := range m.GetMatchingBlocks() { - matches += m.Size - } - return calculateRatio(matches, len(m.a)+len(m.b)) -} - -// Return an upper bound on ratio() relatively quickly. -// -// This isn't defined beyond that it is an upper bound on .Ratio(), and -// is faster to compute. -func (m *SequenceMatcher) QuickRatio() float64 { - // viewing a and b as multisets, set matches to the cardinality - // of their intersection; this counts the number of matches - // without regard to order, so is clearly an upper bound - if m.fullBCount == nil { - m.fullBCount = map[string]int{} - for _, s := range m.b { - m.fullBCount[s] = m.fullBCount[s] + 1 - } - } - - // avail[x] is the number of times x appears in 'b' less the - // number of times we've seen it in 'a' so far ... kinda - avail := map[string]int{} - matches := 0 - for _, s := range m.a { - n, ok := avail[s] - if !ok { - n = m.fullBCount[s] - } - avail[s] = n - 1 - if n > 0 { - matches += 1 - } - } - return calculateRatio(matches, len(m.a)+len(m.b)) -} - -// Return an upper bound on ratio() very quickly. -// -// This isn't defined beyond that it is an upper bound on .Ratio(), and -// is faster to compute than either .Ratio() or .QuickRatio(). -func (m *SequenceMatcher) RealQuickRatio() float64 { - la, lb := len(m.a), len(m.b) - return calculateRatio(min(la, lb), la+lb) -} - -// Convert range to the "ed" format -func formatRangeUnified(start, stop int) string { - // Per the diff spec at http://www.unix.org/single_unix_specification/ - beginning := start + 1 // lines start numbering with one - length := stop - start - if length == 1 { - return fmt.Sprintf("%d", beginning) - } - if length == 0 { - beginning -= 1 // empty ranges begin at line just before the range - } - return fmt.Sprintf("%d,%d", beginning, length) -} - -// Unified diff parameters -type UnifiedDiff struct { - A []string // First sequence lines - FromFile string // First file name - FromDate string // First file time - B []string // Second sequence lines - ToFile string // Second file name - ToDate string // Second file time - Eol string // Headers end of line, defaults to LF - Context int // Number of context lines -} - -// Compare two sequences of lines; generate the delta as a unified diff. -// -// Unified diffs are a compact way of showing line changes and a few -// lines of context. The number of context lines is set by 'n' which -// defaults to three. -// -// By default, the diff control lines (those with ---, +++, or @@) are -// created with a trailing newline. This is helpful so that inputs -// created from file.readlines() result in diffs that are suitable for -// file.writelines() since both the inputs and outputs have trailing -// newlines. -// -// For inputs that do not have trailing newlines, set the lineterm -// argument to "" so that the output will be uniformly newline free. -// -// The unidiff format normally has a header for filenames and modification -// times. Any or all of these may be specified using strings for -// 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. -// The modification times are normally expressed in the ISO 8601 format. -func WriteUnifiedDiff(writer io.Writer, diff UnifiedDiff) error { - buf := bufio.NewWriter(writer) - defer buf.Flush() - wf := func(format string, args ...interface{}) error { - _, err := buf.WriteString(fmt.Sprintf(format, args...)) - return err - } - ws := func(s string) error { - _, err := buf.WriteString(s) - return err - } - - if len(diff.Eol) == 0 { - diff.Eol = "\n" - } - - started := false - m := NewMatcher(diff.A, diff.B) - for _, g := range m.GetGroupedOpCodes(diff.Context) { - if !started { - started = true - fromDate := "" - if len(diff.FromDate) > 0 { - fromDate = "\t" + diff.FromDate - } - toDate := "" - if len(diff.ToDate) > 0 { - toDate = "\t" + diff.ToDate - } - if diff.FromFile != "" || diff.ToFile != "" { - err := wf("--- %s%s%s", diff.FromFile, fromDate, diff.Eol) - if err != nil { - return err - } - err = wf("+++ %s%s%s", diff.ToFile, toDate, diff.Eol) - if err != nil { - return err - } - } - } - first, last := g[0], g[len(g)-1] - range1 := formatRangeUnified(first.I1, last.I2) - range2 := formatRangeUnified(first.J1, last.J2) - if err := wf("@@ -%s +%s @@%s", range1, range2, diff.Eol); err != nil { - return err - } - for _, c := range g { - i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 - if c.Tag == 'e' { - for _, line := range diff.A[i1:i2] { - if err := ws(" " + line); err != nil { - return err - } - } - continue - } - if c.Tag == 'r' || c.Tag == 'd' { - for _, line := range diff.A[i1:i2] { - if err := ws("-" + line); err != nil { - return err - } - } - } - if c.Tag == 'r' || c.Tag == 'i' { - for _, line := range diff.B[j1:j2] { - if err := ws("+" + line); err != nil { - return err - } - } - } - } - } - return nil -} - -// Like WriteUnifiedDiff but returns the diff a string. -func GetUnifiedDiffString(diff UnifiedDiff) (string, error) { - w := &bytes.Buffer{} - err := WriteUnifiedDiff(w, diff) - return string(w.Bytes()), err -} - -// Convert range to the "ed" format. -func formatRangeContext(start, stop int) string { - // Per the diff spec at http://www.unix.org/single_unix_specification/ - beginning := start + 1 // lines start numbering with one - length := stop - start - if length == 0 { - beginning -= 1 // empty ranges begin at line just before the range - } - if length <= 1 { - return fmt.Sprintf("%d", beginning) - } - return fmt.Sprintf("%d,%d", beginning, beginning+length-1) -} - -type ContextDiff UnifiedDiff - -// Compare two sequences of lines; generate the delta as a context diff. -// -// Context diffs are a compact way of showing line changes and a few -// lines of context. The number of context lines is set by diff.Context -// which defaults to three. -// -// By default, the diff control lines (those with *** or ---) are -// created with a trailing newline. -// -// For inputs that do not have trailing newlines, set the diff.Eol -// argument to "" so that the output will be uniformly newline free. -// -// The context diff format normally has a header for filenames and -// modification times. Any or all of these may be specified using -// strings for diff.FromFile, diff.ToFile, diff.FromDate, diff.ToDate. -// The modification times are normally expressed in the ISO 8601 format. -// If not specified, the strings default to blanks. -func WriteContextDiff(writer io.Writer, diff ContextDiff) error { - buf := bufio.NewWriter(writer) - defer buf.Flush() - var diffErr error - wf := func(format string, args ...interface{}) { - _, err := buf.WriteString(fmt.Sprintf(format, args...)) - if diffErr == nil && err != nil { - diffErr = err - } - } - ws := func(s string) { - _, err := buf.WriteString(s) - if diffErr == nil && err != nil { - diffErr = err - } - } - - if len(diff.Eol) == 0 { - diff.Eol = "\n" - } - - prefix := map[byte]string{ - 'i': "+ ", - 'd': "- ", - 'r': "! ", - 'e': " ", - } - - started := false - m := NewMatcher(diff.A, diff.B) - for _, g := range m.GetGroupedOpCodes(diff.Context) { - if !started { - started = true - fromDate := "" - if len(diff.FromDate) > 0 { - fromDate = "\t" + diff.FromDate - } - toDate := "" - if len(diff.ToDate) > 0 { - toDate = "\t" + diff.ToDate - } - if diff.FromFile != "" || diff.ToFile != "" { - wf("*** %s%s%s", diff.FromFile, fromDate, diff.Eol) - wf("--- %s%s%s", diff.ToFile, toDate, diff.Eol) - } - } - - first, last := g[0], g[len(g)-1] - ws("***************" + diff.Eol) - - range1 := formatRangeContext(first.I1, last.I2) - wf("*** %s ****%s", range1, diff.Eol) - for _, c := range g { - if c.Tag == 'r' || c.Tag == 'd' { - for _, cc := range g { - if cc.Tag == 'i' { - continue - } - for _, line := range diff.A[cc.I1:cc.I2] { - ws(prefix[cc.Tag] + line) - } - } - break - } - } - - range2 := formatRangeContext(first.J1, last.J2) - wf("--- %s ----%s", range2, diff.Eol) - for _, c := range g { - if c.Tag == 'r' || c.Tag == 'i' { - for _, cc := range g { - if cc.Tag == 'd' { - continue - } - for _, line := range diff.B[cc.J1:cc.J2] { - ws(prefix[cc.Tag] + line) - } - } - break - } - } - } - return diffErr -} - -// Like WriteContextDiff but returns the diff a string. -func GetContextDiffString(diff ContextDiff) (string, error) { - w := &bytes.Buffer{} - err := WriteContextDiff(w, diff) - return string(w.Bytes()), err -} - -// Split a string on "\n" while preserving them. The output can be used -// as input for UnifiedDiff and ContextDiff structures. -func SplitLines(s string) []string { - lines := strings.SplitAfter(s, "\n") - lines[len(lines)-1] += "\n" - return lines -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/LICENSE b/cluster-provision/gocli/vendor/github.com/stretchr/testify/LICENSE deleted file mode 100644 index 4b0421cf9e..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_compare.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_compare.go deleted file mode 100644 index 4d4b4aad6f..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_compare.go +++ /dev/null @@ -1,480 +0,0 @@ -package assert - -import ( - "bytes" - "fmt" - "reflect" - "time" -) - -type CompareType int - -const ( - compareLess CompareType = iota - 1 - compareEqual - compareGreater -) - -var ( - intType = reflect.TypeOf(int(1)) - int8Type = reflect.TypeOf(int8(1)) - int16Type = reflect.TypeOf(int16(1)) - int32Type = reflect.TypeOf(int32(1)) - int64Type = reflect.TypeOf(int64(1)) - - uintType = reflect.TypeOf(uint(1)) - uint8Type = reflect.TypeOf(uint8(1)) - uint16Type = reflect.TypeOf(uint16(1)) - uint32Type = reflect.TypeOf(uint32(1)) - uint64Type = reflect.TypeOf(uint64(1)) - - uintptrType = reflect.TypeOf(uintptr(1)) - - float32Type = reflect.TypeOf(float32(1)) - float64Type = reflect.TypeOf(float64(1)) - - stringType = reflect.TypeOf("") - - timeType = reflect.TypeOf(time.Time{}) - bytesType = reflect.TypeOf([]byte{}) -) - -func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) { - obj1Value := reflect.ValueOf(obj1) - obj2Value := reflect.ValueOf(obj2) - - // throughout this switch we try and avoid calling .Convert() if possible, - // as this has a pretty big performance impact - switch kind { - case reflect.Int: - { - intobj1, ok := obj1.(int) - if !ok { - intobj1 = obj1Value.Convert(intType).Interface().(int) - } - intobj2, ok := obj2.(int) - if !ok { - intobj2 = obj2Value.Convert(intType).Interface().(int) - } - if intobj1 > intobj2 { - return compareGreater, true - } - if intobj1 == intobj2 { - return compareEqual, true - } - if intobj1 < intobj2 { - return compareLess, true - } - } - case reflect.Int8: - { - int8obj1, ok := obj1.(int8) - if !ok { - int8obj1 = obj1Value.Convert(int8Type).Interface().(int8) - } - int8obj2, ok := obj2.(int8) - if !ok { - int8obj2 = obj2Value.Convert(int8Type).Interface().(int8) - } - if int8obj1 > int8obj2 { - return compareGreater, true - } - if int8obj1 == int8obj2 { - return compareEqual, true - } - if int8obj1 < int8obj2 { - return compareLess, true - } - } - case reflect.Int16: - { - int16obj1, ok := obj1.(int16) - if !ok { - int16obj1 = obj1Value.Convert(int16Type).Interface().(int16) - } - int16obj2, ok := obj2.(int16) - if !ok { - int16obj2 = obj2Value.Convert(int16Type).Interface().(int16) - } - if int16obj1 > int16obj2 { - return compareGreater, true - } - if int16obj1 == int16obj2 { - return compareEqual, true - } - if int16obj1 < int16obj2 { - return compareLess, true - } - } - case reflect.Int32: - { - int32obj1, ok := obj1.(int32) - if !ok { - int32obj1 = obj1Value.Convert(int32Type).Interface().(int32) - } - int32obj2, ok := obj2.(int32) - if !ok { - int32obj2 = obj2Value.Convert(int32Type).Interface().(int32) - } - if int32obj1 > int32obj2 { - return compareGreater, true - } - if int32obj1 == int32obj2 { - return compareEqual, true - } - if int32obj1 < int32obj2 { - return compareLess, true - } - } - case reflect.Int64: - { - int64obj1, ok := obj1.(int64) - if !ok { - int64obj1 = obj1Value.Convert(int64Type).Interface().(int64) - } - int64obj2, ok := obj2.(int64) - if !ok { - int64obj2 = obj2Value.Convert(int64Type).Interface().(int64) - } - if int64obj1 > int64obj2 { - return compareGreater, true - } - if int64obj1 == int64obj2 { - return compareEqual, true - } - if int64obj1 < int64obj2 { - return compareLess, true - } - } - case reflect.Uint: - { - uintobj1, ok := obj1.(uint) - if !ok { - uintobj1 = obj1Value.Convert(uintType).Interface().(uint) - } - uintobj2, ok := obj2.(uint) - if !ok { - uintobj2 = obj2Value.Convert(uintType).Interface().(uint) - } - if uintobj1 > uintobj2 { - return compareGreater, true - } - if uintobj1 == uintobj2 { - return compareEqual, true - } - if uintobj1 < uintobj2 { - return compareLess, true - } - } - case reflect.Uint8: - { - uint8obj1, ok := obj1.(uint8) - if !ok { - uint8obj1 = obj1Value.Convert(uint8Type).Interface().(uint8) - } - uint8obj2, ok := obj2.(uint8) - if !ok { - uint8obj2 = obj2Value.Convert(uint8Type).Interface().(uint8) - } - if uint8obj1 > uint8obj2 { - return compareGreater, true - } - if uint8obj1 == uint8obj2 { - return compareEqual, true - } - if uint8obj1 < uint8obj2 { - return compareLess, true - } - } - case reflect.Uint16: - { - uint16obj1, ok := obj1.(uint16) - if !ok { - uint16obj1 = obj1Value.Convert(uint16Type).Interface().(uint16) - } - uint16obj2, ok := obj2.(uint16) - if !ok { - uint16obj2 = obj2Value.Convert(uint16Type).Interface().(uint16) - } - if uint16obj1 > uint16obj2 { - return compareGreater, true - } - if uint16obj1 == uint16obj2 { - return compareEqual, true - } - if uint16obj1 < uint16obj2 { - return compareLess, true - } - } - case reflect.Uint32: - { - uint32obj1, ok := obj1.(uint32) - if !ok { - uint32obj1 = obj1Value.Convert(uint32Type).Interface().(uint32) - } - uint32obj2, ok := obj2.(uint32) - if !ok { - uint32obj2 = obj2Value.Convert(uint32Type).Interface().(uint32) - } - if uint32obj1 > uint32obj2 { - return compareGreater, true - } - if uint32obj1 == uint32obj2 { - return compareEqual, true - } - if uint32obj1 < uint32obj2 { - return compareLess, true - } - } - case reflect.Uint64: - { - uint64obj1, ok := obj1.(uint64) - if !ok { - uint64obj1 = obj1Value.Convert(uint64Type).Interface().(uint64) - } - uint64obj2, ok := obj2.(uint64) - if !ok { - uint64obj2 = obj2Value.Convert(uint64Type).Interface().(uint64) - } - if uint64obj1 > uint64obj2 { - return compareGreater, true - } - if uint64obj1 == uint64obj2 { - return compareEqual, true - } - if uint64obj1 < uint64obj2 { - return compareLess, true - } - } - case reflect.Float32: - { - float32obj1, ok := obj1.(float32) - if !ok { - float32obj1 = obj1Value.Convert(float32Type).Interface().(float32) - } - float32obj2, ok := obj2.(float32) - if !ok { - float32obj2 = obj2Value.Convert(float32Type).Interface().(float32) - } - if float32obj1 > float32obj2 { - return compareGreater, true - } - if float32obj1 == float32obj2 { - return compareEqual, true - } - if float32obj1 < float32obj2 { - return compareLess, true - } - } - case reflect.Float64: - { - float64obj1, ok := obj1.(float64) - if !ok { - float64obj1 = obj1Value.Convert(float64Type).Interface().(float64) - } - float64obj2, ok := obj2.(float64) - if !ok { - float64obj2 = obj2Value.Convert(float64Type).Interface().(float64) - } - if float64obj1 > float64obj2 { - return compareGreater, true - } - if float64obj1 == float64obj2 { - return compareEqual, true - } - if float64obj1 < float64obj2 { - return compareLess, true - } - } - case reflect.String: - { - stringobj1, ok := obj1.(string) - if !ok { - stringobj1 = obj1Value.Convert(stringType).Interface().(string) - } - stringobj2, ok := obj2.(string) - if !ok { - stringobj2 = obj2Value.Convert(stringType).Interface().(string) - } - if stringobj1 > stringobj2 { - return compareGreater, true - } - if stringobj1 == stringobj2 { - return compareEqual, true - } - if stringobj1 < stringobj2 { - return compareLess, true - } - } - // Check for known struct types we can check for compare results. - case reflect.Struct: - { - // All structs enter here. We're not interested in most types. - if !obj1Value.CanConvert(timeType) { - break - } - - // time.Time can be compared! - timeObj1, ok := obj1.(time.Time) - if !ok { - timeObj1 = obj1Value.Convert(timeType).Interface().(time.Time) - } - - timeObj2, ok := obj2.(time.Time) - if !ok { - timeObj2 = obj2Value.Convert(timeType).Interface().(time.Time) - } - - return compare(timeObj1.UnixNano(), timeObj2.UnixNano(), reflect.Int64) - } - case reflect.Slice: - { - // We only care about the []byte type. - if !obj1Value.CanConvert(bytesType) { - break - } - - // []byte can be compared! - bytesObj1, ok := obj1.([]byte) - if !ok { - bytesObj1 = obj1Value.Convert(bytesType).Interface().([]byte) - - } - bytesObj2, ok := obj2.([]byte) - if !ok { - bytesObj2 = obj2Value.Convert(bytesType).Interface().([]byte) - } - - return CompareType(bytes.Compare(bytesObj1, bytesObj2)), true - } - case reflect.Uintptr: - { - uintptrObj1, ok := obj1.(uintptr) - if !ok { - uintptrObj1 = obj1Value.Convert(uintptrType).Interface().(uintptr) - } - uintptrObj2, ok := obj2.(uintptr) - if !ok { - uintptrObj2 = obj2Value.Convert(uintptrType).Interface().(uintptr) - } - if uintptrObj1 > uintptrObj2 { - return compareGreater, true - } - if uintptrObj1 == uintptrObj2 { - return compareEqual, true - } - if uintptrObj1 < uintptrObj2 { - return compareLess, true - } - } - } - - return compareEqual, false -} - -// Greater asserts that the first element is greater than the second -// -// assert.Greater(t, 2, 1) -// assert.Greater(t, float64(2), float64(1)) -// assert.Greater(t, "b", "a") -func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return compareTwoValues(t, e1, e2, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...) -} - -// GreaterOrEqual asserts that the first element is greater than or equal to the second -// -// assert.GreaterOrEqual(t, 2, 1) -// assert.GreaterOrEqual(t, 2, 2) -// assert.GreaterOrEqual(t, "b", "a") -// assert.GreaterOrEqual(t, "b", "b") -func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return compareTwoValues(t, e1, e2, []CompareType{compareGreater, compareEqual}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...) -} - -// Less asserts that the first element is less than the second -// -// assert.Less(t, 1, 2) -// assert.Less(t, float64(1), float64(2)) -// assert.Less(t, "a", "b") -func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return compareTwoValues(t, e1, e2, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...) -} - -// LessOrEqual asserts that the first element is less than or equal to the second -// -// assert.LessOrEqual(t, 1, 2) -// assert.LessOrEqual(t, 2, 2) -// assert.LessOrEqual(t, "a", "b") -// assert.LessOrEqual(t, "b", "b") -func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return compareTwoValues(t, e1, e2, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...) -} - -// Positive asserts that the specified element is positive -// -// assert.Positive(t, 1) -// assert.Positive(t, 1.23) -func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - zero := reflect.Zero(reflect.TypeOf(e)) - return compareTwoValues(t, e, zero.Interface(), []CompareType{compareGreater}, "\"%v\" is not positive", msgAndArgs...) -} - -// Negative asserts that the specified element is negative -// -// assert.Negative(t, -1) -// assert.Negative(t, -1.23) -func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - zero := reflect.Zero(reflect.TypeOf(e)) - return compareTwoValues(t, e, zero.Interface(), []CompareType{compareLess}, "\"%v\" is not negative", msgAndArgs...) -} - -func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - e1Kind := reflect.ValueOf(e1).Kind() - e2Kind := reflect.ValueOf(e2).Kind() - if e1Kind != e2Kind { - return Fail(t, "Elements should be the same type", msgAndArgs...) - } - - compareResult, isComparable := compare(e1, e2, e1Kind) - if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) - } - - if !containsValue(allowedComparesResults, compareResult) { - return Fail(t, fmt.Sprintf(failMessage, e1, e2), msgAndArgs...) - } - - return true -} - -func containsValue(values []CompareType, value CompareType) bool { - for _, v := range values { - if v == value { - return true - } - } - - return false -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go deleted file mode 100644 index 3ddab109ad..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go +++ /dev/null @@ -1,815 +0,0 @@ -// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. - -package assert - -import ( - http "net/http" - url "net/url" - time "time" -) - -// Conditionf uses a Comparison to assert a complex condition. -func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Condition(t, comp, append([]interface{}{msg}, args...)...) -} - -// Containsf asserts that the specified string, list(array, slice...) or map contains the -// specified substring or element. -// -// assert.Containsf(t, "Hello World", "World", "error message %s", "formatted") -// assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") -// assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") -func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Contains(t, s, contains, append([]interface{}{msg}, args...)...) -} - -// DirExistsf checks whether a directory exists in the given path. It also fails -// if the path is a file rather a directory or there is an error checking whether it exists. -func DirExistsf(t TestingT, path string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return DirExists(t, path, append([]interface{}{msg}, args...)...) -} - -// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified -// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, -// the number of appearances of each of them in both lists should match. -// -// assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") -func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return ElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) -} - -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// assert.Emptyf(t, obj, "error message %s", "formatted") -func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Empty(t, object, append([]interface{}{msg}, args...)...) -} - -// Equalf asserts that two objects are equal. -// -// assert.Equalf(t, 123, 123, "error message %s", "formatted") -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). Function equality -// cannot be determined and will always fail. -func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Equal(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// EqualErrorf asserts that a function returned an error (i.e. not `nil`) -// and that it is equal to the provided error. -// -// actualObj, err := SomeFunction() -// assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") -func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return EqualError(t, theError, errString, append([]interface{}{msg}, args...)...) -} - -// EqualExportedValuesf asserts that the types of two objects are equal and their public -// fields are also equal. This is useful for comparing structs that have private fields -// that could potentially differ. -// -// type S struct { -// Exported int -// notExported int -// } -// assert.EqualExportedValuesf(t, S{1, 2}, S{1, 3}, "error message %s", "formatted") => true -// assert.EqualExportedValuesf(t, S{1, 2}, S{2, 3}, "error message %s", "formatted") => false -func EqualExportedValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return EqualExportedValues(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// EqualValuesf asserts that two objects are equal or convertible to the same types -// and equal. -// -// assert.EqualValuesf(t, uint32(123), int32(123), "error message %s", "formatted") -func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return EqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// Errorf asserts that a function returned an error (i.e. not `nil`). -// -// actualObj, err := SomeFunction() -// if assert.Errorf(t, err, "error message %s", "formatted") { -// assert.Equal(t, expectedErrorf, err) -// } -func Errorf(t TestingT, err error, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Error(t, err, append([]interface{}{msg}, args...)...) -} - -// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. -// This is a wrapper for errors.As. -func ErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return ErrorAs(t, err, target, append([]interface{}{msg}, args...)...) -} - -// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) -// and that the error contains the specified substring. -// -// actualObj, err := SomeFunction() -// assert.ErrorContainsf(t, err, expectedErrorSubString, "error message %s", "formatted") -func ErrorContainsf(t TestingT, theError error, contains string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return ErrorContains(t, theError, contains, append([]interface{}{msg}, args...)...) -} - -// ErrorIsf asserts that at least one of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return ErrorIs(t, err, target, append([]interface{}{msg}, args...)...) -} - -// Eventuallyf asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. -// -// assert.Eventuallyf(t, func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") -func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Eventually(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) -} - -// EventuallyWithTf asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. In contrast to Eventually, -// it supplies a CollectT to the condition function, so that the condition -// function can use the CollectT to call other assertions. -// The condition is considered "met" if no errors are raised in a tick. -// The supplied CollectT collects all errors from one tick (if there are any). -// If the condition is not met before waitFor, the collected errors of -// the last tick are copied to t. -// -// externalValue := false -// go func() { -// time.Sleep(8*time.Second) -// externalValue = true -// }() -// assert.EventuallyWithTf(t, func(c *assert.CollectT, "error message %s", "formatted") { -// // add assertions as needed; any assertion failure will fail the current tick -// assert.True(c, externalValue, "expected 'externalValue' to be true") -// }, 1*time.Second, 10*time.Second, "external state has not changed to 'true'; still false") -func EventuallyWithTf(t TestingT, condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return EventuallyWithT(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) -} - -// Exactlyf asserts that two objects are equal in value and type. -// -// assert.Exactlyf(t, int32(123), int64(123), "error message %s", "formatted") -func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Exactly(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// Failf reports a failure through -func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, failureMessage, append([]interface{}{msg}, args...)...) -} - -// FailNowf fails test -func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return FailNow(t, failureMessage, append([]interface{}{msg}, args...)...) -} - -// Falsef asserts that the specified value is false. -// -// assert.Falsef(t, myBool, "error message %s", "formatted") -func Falsef(t TestingT, value bool, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return False(t, value, append([]interface{}{msg}, args...)...) -} - -// FileExistsf checks whether a file exists in the given path. It also fails if -// the path points to a directory or there is an error when trying to check the file. -func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return FileExists(t, path, append([]interface{}{msg}, args...)...) -} - -// Greaterf asserts that the first element is greater than the second -// -// assert.Greaterf(t, 2, 1, "error message %s", "formatted") -// assert.Greaterf(t, float64(2), float64(1), "error message %s", "formatted") -// assert.Greaterf(t, "b", "a", "error message %s", "formatted") -func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Greater(t, e1, e2, append([]interface{}{msg}, args...)...) -} - -// GreaterOrEqualf asserts that the first element is greater than or equal to the second -// -// assert.GreaterOrEqualf(t, 2, 1, "error message %s", "formatted") -// assert.GreaterOrEqualf(t, 2, 2, "error message %s", "formatted") -// assert.GreaterOrEqualf(t, "b", "a", "error message %s", "formatted") -// assert.GreaterOrEqualf(t, "b", "b", "error message %s", "formatted") -func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return GreaterOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) -} - -// HTTPBodyContainsf asserts that a specified handler returns a -// body that contains a string. -// -// assert.HTTPBodyContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPBodyContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) -} - -// HTTPBodyNotContainsf asserts that a specified handler returns a -// body that does not contain a string. -// -// assert.HTTPBodyNotContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPBodyNotContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) -} - -// HTTPErrorf asserts that a specified handler returns an error status code. -// -// assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPError(t, handler, method, url, values, append([]interface{}{msg}, args...)...) -} - -// HTTPRedirectf asserts that a specified handler returns a redirect status code. -// -// assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPRedirect(t, handler, method, url, values, append([]interface{}{msg}, args...)...) -} - -// HTTPStatusCodef asserts that a specified handler returns a specified status code. -// -// assert.HTTPStatusCodef(t, myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPStatusCodef(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPStatusCode(t, handler, method, url, values, statuscode, append([]interface{}{msg}, args...)...) -} - -// HTTPSuccessf asserts that a specified handler returns a success status code. -// -// assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return HTTPSuccess(t, handler, method, url, values, append([]interface{}{msg}, args...)...) -} - -// Implementsf asserts that an object is implemented by the specified interface. -// -// assert.Implementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") -func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Implements(t, interfaceObject, object, append([]interface{}{msg}, args...)...) -} - -// InDeltaf asserts that the two numerals are within delta of each other. -// -// assert.InDeltaf(t, math.Pi, 22/7.0, 0.01, "error message %s", "formatted") -func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return InDelta(t, expected, actual, delta, append([]interface{}{msg}, args...)...) -} - -// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. -func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return InDeltaMapValues(t, expected, actual, delta, append([]interface{}{msg}, args...)...) -} - -// InDeltaSlicef is the same as InDelta, except it compares two slices. -func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return InDeltaSlice(t, expected, actual, delta, append([]interface{}{msg}, args...)...) -} - -// InEpsilonf asserts that expected and actual have a relative error less than epsilon -func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return InEpsilon(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) -} - -// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. -func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return InEpsilonSlice(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) -} - -// IsDecreasingf asserts that the collection is decreasing -// -// assert.IsDecreasingf(t, []int{2, 1, 0}, "error message %s", "formatted") -// assert.IsDecreasingf(t, []float{2, 1}, "error message %s", "formatted") -// assert.IsDecreasingf(t, []string{"b", "a"}, "error message %s", "formatted") -func IsDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return IsDecreasing(t, object, append([]interface{}{msg}, args...)...) -} - -// IsIncreasingf asserts that the collection is increasing -// -// assert.IsIncreasingf(t, []int{1, 2, 3}, "error message %s", "formatted") -// assert.IsIncreasingf(t, []float{1, 2}, "error message %s", "formatted") -// assert.IsIncreasingf(t, []string{"a", "b"}, "error message %s", "formatted") -func IsIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return IsIncreasing(t, object, append([]interface{}{msg}, args...)...) -} - -// IsNonDecreasingf asserts that the collection is not decreasing -// -// assert.IsNonDecreasingf(t, []int{1, 1, 2}, "error message %s", "formatted") -// assert.IsNonDecreasingf(t, []float{1, 2}, "error message %s", "formatted") -// assert.IsNonDecreasingf(t, []string{"a", "b"}, "error message %s", "formatted") -func IsNonDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return IsNonDecreasing(t, object, append([]interface{}{msg}, args...)...) -} - -// IsNonIncreasingf asserts that the collection is not increasing -// -// assert.IsNonIncreasingf(t, []int{2, 1, 1}, "error message %s", "formatted") -// assert.IsNonIncreasingf(t, []float{2, 1}, "error message %s", "formatted") -// assert.IsNonIncreasingf(t, []string{"b", "a"}, "error message %s", "formatted") -func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return IsNonIncreasing(t, object, append([]interface{}{msg}, args...)...) -} - -// IsTypef asserts that the specified objects are of the same type. -func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return IsType(t, expectedType, object, append([]interface{}{msg}, args...)...) -} - -// JSONEqf asserts that two JSON strings are equivalent. -// -// assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return JSONEq(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// Lenf asserts that the specified object has specific length. -// Lenf also fails if the object has a type that len() not accept. -// -// assert.Lenf(t, mySlice, 3, "error message %s", "formatted") -func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Len(t, object, length, append([]interface{}{msg}, args...)...) -} - -// Lessf asserts that the first element is less than the second -// -// assert.Lessf(t, 1, 2, "error message %s", "formatted") -// assert.Lessf(t, float64(1), float64(2), "error message %s", "formatted") -// assert.Lessf(t, "a", "b", "error message %s", "formatted") -func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Less(t, e1, e2, append([]interface{}{msg}, args...)...) -} - -// LessOrEqualf asserts that the first element is less than or equal to the second -// -// assert.LessOrEqualf(t, 1, 2, "error message %s", "formatted") -// assert.LessOrEqualf(t, 2, 2, "error message %s", "formatted") -// assert.LessOrEqualf(t, "a", "b", "error message %s", "formatted") -// assert.LessOrEqualf(t, "b", "b", "error message %s", "formatted") -func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return LessOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) -} - -// Negativef asserts that the specified element is negative -// -// assert.Negativef(t, -1, "error message %s", "formatted") -// assert.Negativef(t, -1.23, "error message %s", "formatted") -func Negativef(t TestingT, e interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Negative(t, e, append([]interface{}{msg}, args...)...) -} - -// Neverf asserts that the given condition doesn't satisfy in waitFor time, -// periodically checking the target function each tick. -// -// assert.Neverf(t, func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") -func Neverf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Never(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) -} - -// Nilf asserts that the specified object is nil. -// -// assert.Nilf(t, err, "error message %s", "formatted") -func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Nil(t, object, append([]interface{}{msg}, args...)...) -} - -// NoDirExistsf checks whether a directory does not exist in the given path. -// It fails if the path points to an existing _directory_ only. -func NoDirExistsf(t TestingT, path string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NoDirExists(t, path, append([]interface{}{msg}, args...)...) -} - -// NoErrorf asserts that a function returned no error (i.e. `nil`). -// -// actualObj, err := SomeFunction() -// if assert.NoErrorf(t, err, "error message %s", "formatted") { -// assert.Equal(t, expectedObj, actualObj) -// } -func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NoError(t, err, append([]interface{}{msg}, args...)...) -} - -// NoFileExistsf checks whether a file does not exist in a given path. It fails -// if the path points to an existing _file_ only. -func NoFileExistsf(t TestingT, path string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NoFileExists(t, path, append([]interface{}{msg}, args...)...) -} - -// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the -// specified substring or element. -// -// assert.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted") -// assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") -// assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") -func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotContains(t, s, contains, append([]interface{}{msg}, args...)...) -} - -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// if assert.NotEmptyf(t, obj, "error message %s", "formatted") { -// assert.Equal(t, "two", obj[1]) -// } -func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotEmpty(t, object, append([]interface{}{msg}, args...)...) -} - -// NotEqualf asserts that the specified values are NOT equal. -// -// assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted") -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). -func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotEqual(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// NotEqualValuesf asserts that two objects are not equal even when converted to the same type -// -// assert.NotEqualValuesf(t, obj1, obj2, "error message %s", "formatted") -func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotEqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// NotErrorIsf asserts that at none of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func NotErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotErrorIs(t, err, target, append([]interface{}{msg}, args...)...) -} - -// NotImplementsf asserts that an object does not implement the specified interface. -// -// assert.NotImplementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") -func NotImplementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotImplements(t, interfaceObject, object, append([]interface{}{msg}, args...)...) -} - -// NotNilf asserts that the specified object is not nil. -// -// assert.NotNilf(t, err, "error message %s", "formatted") -func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotNil(t, object, append([]interface{}{msg}, args...)...) -} - -// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. -// -// assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") -func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotPanics(t, f, append([]interface{}{msg}, args...)...) -} - -// NotRegexpf asserts that a specified regexp does not match a string. -// -// assert.NotRegexpf(t, regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") -// assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") -func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotRegexp(t, rx, str, append([]interface{}{msg}, args...)...) -} - -// NotSamef asserts that two pointers do not reference the same object. -// -// assert.NotSamef(t, ptr1, ptr2, "error message %s", "formatted") -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func NotSamef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotSame(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. -// -// assert.NotSubsetf(t, [1, 3, 4], [1, 2], "error message %s", "formatted") -// assert.NotSubsetf(t, {"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") -func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotSubset(t, list, subset, append([]interface{}{msg}, args...)...) -} - -// NotZerof asserts that i is not the zero value for its type. -func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return NotZero(t, i, append([]interface{}{msg}, args...)...) -} - -// Panicsf asserts that the code inside the specified PanicTestFunc panics. -// -// assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") -func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Panics(t, f, append([]interface{}{msg}, args...)...) -} - -// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc -// panics, and that the recovered panic value is an error that satisfies the -// EqualError comparison. -// -// assert.PanicsWithErrorf(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -func PanicsWithErrorf(t TestingT, errString string, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return PanicsWithError(t, errString, f, append([]interface{}{msg}, args...)...) -} - -// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that -// the recovered panic value equals the expected panic value. -// -// assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return PanicsWithValue(t, expected, f, append([]interface{}{msg}, args...)...) -} - -// Positivef asserts that the specified element is positive -// -// assert.Positivef(t, 1, "error message %s", "formatted") -// assert.Positivef(t, 1.23, "error message %s", "formatted") -func Positivef(t TestingT, e interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Positive(t, e, append([]interface{}{msg}, args...)...) -} - -// Regexpf asserts that a specified regexp matches a string. -// -// assert.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") -// assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") -func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Regexp(t, rx, str, append([]interface{}{msg}, args...)...) -} - -// Samef asserts that two pointers reference the same object. -// -// assert.Samef(t, ptr1, ptr2, "error message %s", "formatted") -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func Samef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Same(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. -// -// assert.Subsetf(t, [1, 2, 3], [1, 2], "error message %s", "formatted") -// assert.Subsetf(t, {"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") -func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Subset(t, list, subset, append([]interface{}{msg}, args...)...) -} - -// Truef asserts that the specified value is true. -// -// assert.Truef(t, myBool, "error message %s", "formatted") -func Truef(t TestingT, value bool, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return True(t, value, append([]interface{}{msg}, args...)...) -} - -// WithinDurationf asserts that the two times are within duration delta of each other. -// -// assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...) -} - -// WithinRangef asserts that a time is within a time range (inclusive). -// -// assert.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") -func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return WithinRange(t, actual, start, end, append([]interface{}{msg}, args...)...) -} - -// YAMLEqf asserts that two YAML strings are equivalent. -func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return YAMLEq(t, expected, actual, append([]interface{}{msg}, args...)...) -} - -// Zerof asserts that i is the zero value for its type. -func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Zero(t, i, append([]interface{}{msg}, args...)...) -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl deleted file mode 100644 index d2bb0b8177..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -{{.CommentFormat}} -func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool { - if h, ok := t.(tHelper); ok { h.Helper() } - return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}}) -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go deleted file mode 100644 index a84e09bd40..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go +++ /dev/null @@ -1,1621 +0,0 @@ -// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. - -package assert - -import ( - http "net/http" - url "net/url" - time "time" -) - -// Condition uses a Comparison to assert a complex condition. -func (a *Assertions) Condition(comp Comparison, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Condition(a.t, comp, msgAndArgs...) -} - -// Conditionf uses a Comparison to assert a complex condition. -func (a *Assertions) Conditionf(comp Comparison, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Conditionf(a.t, comp, msg, args...) -} - -// Contains asserts that the specified string, list(array, slice...) or map contains the -// specified substring or element. -// -// a.Contains("Hello World", "World") -// a.Contains(["Hello", "World"], "World") -// a.Contains({"Hello": "World"}, "Hello") -func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Contains(a.t, s, contains, msgAndArgs...) -} - -// Containsf asserts that the specified string, list(array, slice...) or map contains the -// specified substring or element. -// -// a.Containsf("Hello World", "World", "error message %s", "formatted") -// a.Containsf(["Hello", "World"], "World", "error message %s", "formatted") -// a.Containsf({"Hello": "World"}, "Hello", "error message %s", "formatted") -func (a *Assertions) Containsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Containsf(a.t, s, contains, msg, args...) -} - -// DirExists checks whether a directory exists in the given path. It also fails -// if the path is a file rather a directory or there is an error checking whether it exists. -func (a *Assertions) DirExists(path string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return DirExists(a.t, path, msgAndArgs...) -} - -// DirExistsf checks whether a directory exists in the given path. It also fails -// if the path is a file rather a directory or there is an error checking whether it exists. -func (a *Assertions) DirExistsf(path string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return DirExistsf(a.t, path, msg, args...) -} - -// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified -// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, -// the number of appearances of each of them in both lists should match. -// -// a.ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]) -func (a *Assertions) ElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ElementsMatch(a.t, listA, listB, msgAndArgs...) -} - -// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified -// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, -// the number of appearances of each of them in both lists should match. -// -// a.ElementsMatchf([1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") -func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ElementsMatchf(a.t, listA, listB, msg, args...) -} - -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// a.Empty(obj) -func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Empty(a.t, object, msgAndArgs...) -} - -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// a.Emptyf(obj, "error message %s", "formatted") -func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Emptyf(a.t, object, msg, args...) -} - -// Equal asserts that two objects are equal. -// -// a.Equal(123, 123) -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). Function equality -// cannot be determined and will always fail. -func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Equal(a.t, expected, actual, msgAndArgs...) -} - -// EqualError asserts that a function returned an error (i.e. not `nil`) -// and that it is equal to the provided error. -// -// actualObj, err := SomeFunction() -// a.EqualError(err, expectedErrorString) -func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualError(a.t, theError, errString, msgAndArgs...) -} - -// EqualErrorf asserts that a function returned an error (i.e. not `nil`) -// and that it is equal to the provided error. -// -// actualObj, err := SomeFunction() -// a.EqualErrorf(err, expectedErrorString, "error message %s", "formatted") -func (a *Assertions) EqualErrorf(theError error, errString string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualErrorf(a.t, theError, errString, msg, args...) -} - -// EqualExportedValues asserts that the types of two objects are equal and their public -// fields are also equal. This is useful for comparing structs that have private fields -// that could potentially differ. -// -// type S struct { -// Exported int -// notExported int -// } -// a.EqualExportedValues(S{1, 2}, S{1, 3}) => true -// a.EqualExportedValues(S{1, 2}, S{2, 3}) => false -func (a *Assertions) EqualExportedValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualExportedValues(a.t, expected, actual, msgAndArgs...) -} - -// EqualExportedValuesf asserts that the types of two objects are equal and their public -// fields are also equal. This is useful for comparing structs that have private fields -// that could potentially differ. -// -// type S struct { -// Exported int -// notExported int -// } -// a.EqualExportedValuesf(S{1, 2}, S{1, 3}, "error message %s", "formatted") => true -// a.EqualExportedValuesf(S{1, 2}, S{2, 3}, "error message %s", "formatted") => false -func (a *Assertions) EqualExportedValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualExportedValuesf(a.t, expected, actual, msg, args...) -} - -// EqualValues asserts that two objects are equal or convertible to the same types -// and equal. -// -// a.EqualValues(uint32(123), int32(123)) -func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualValues(a.t, expected, actual, msgAndArgs...) -} - -// EqualValuesf asserts that two objects are equal or convertible to the same types -// and equal. -// -// a.EqualValuesf(uint32(123), int32(123), "error message %s", "formatted") -func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EqualValuesf(a.t, expected, actual, msg, args...) -} - -// Equalf asserts that two objects are equal. -// -// a.Equalf(123, 123, "error message %s", "formatted") -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). Function equality -// cannot be determined and will always fail. -func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Equalf(a.t, expected, actual, msg, args...) -} - -// Error asserts that a function returned an error (i.e. not `nil`). -// -// actualObj, err := SomeFunction() -// if a.Error(err) { -// assert.Equal(t, expectedError, err) -// } -func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Error(a.t, err, msgAndArgs...) -} - -// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. -// This is a wrapper for errors.As. -func (a *Assertions) ErrorAs(err error, target interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorAs(a.t, err, target, msgAndArgs...) -} - -// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. -// This is a wrapper for errors.As. -func (a *Assertions) ErrorAsf(err error, target interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorAsf(a.t, err, target, msg, args...) -} - -// ErrorContains asserts that a function returned an error (i.e. not `nil`) -// and that the error contains the specified substring. -// -// actualObj, err := SomeFunction() -// a.ErrorContains(err, expectedErrorSubString) -func (a *Assertions) ErrorContains(theError error, contains string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorContains(a.t, theError, contains, msgAndArgs...) -} - -// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) -// and that the error contains the specified substring. -// -// actualObj, err := SomeFunction() -// a.ErrorContainsf(err, expectedErrorSubString, "error message %s", "formatted") -func (a *Assertions) ErrorContainsf(theError error, contains string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorContainsf(a.t, theError, contains, msg, args...) -} - -// ErrorIs asserts that at least one of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func (a *Assertions) ErrorIs(err error, target error, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorIs(a.t, err, target, msgAndArgs...) -} - -// ErrorIsf asserts that at least one of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return ErrorIsf(a.t, err, target, msg, args...) -} - -// Errorf asserts that a function returned an error (i.e. not `nil`). -// -// actualObj, err := SomeFunction() -// if a.Errorf(err, "error message %s", "formatted") { -// assert.Equal(t, expectedErrorf, err) -// } -func (a *Assertions) Errorf(err error, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Errorf(a.t, err, msg, args...) -} - -// Eventually asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. -// -// a.Eventually(func() bool { return true; }, time.Second, 10*time.Millisecond) -func (a *Assertions) Eventually(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Eventually(a.t, condition, waitFor, tick, msgAndArgs...) -} - -// EventuallyWithT asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. In contrast to Eventually, -// it supplies a CollectT to the condition function, so that the condition -// function can use the CollectT to call other assertions. -// The condition is considered "met" if no errors are raised in a tick. -// The supplied CollectT collects all errors from one tick (if there are any). -// If the condition is not met before waitFor, the collected errors of -// the last tick are copied to t. -// -// externalValue := false -// go func() { -// time.Sleep(8*time.Second) -// externalValue = true -// }() -// a.EventuallyWithT(func(c *assert.CollectT) { -// // add assertions as needed; any assertion failure will fail the current tick -// assert.True(c, externalValue, "expected 'externalValue' to be true") -// }, 1*time.Second, 10*time.Second, "external state has not changed to 'true'; still false") -func (a *Assertions) EventuallyWithT(condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EventuallyWithT(a.t, condition, waitFor, tick, msgAndArgs...) -} - -// EventuallyWithTf asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. In contrast to Eventually, -// it supplies a CollectT to the condition function, so that the condition -// function can use the CollectT to call other assertions. -// The condition is considered "met" if no errors are raised in a tick. -// The supplied CollectT collects all errors from one tick (if there are any). -// If the condition is not met before waitFor, the collected errors of -// the last tick are copied to t. -// -// externalValue := false -// go func() { -// time.Sleep(8*time.Second) -// externalValue = true -// }() -// a.EventuallyWithTf(func(c *assert.CollectT, "error message %s", "formatted") { -// // add assertions as needed; any assertion failure will fail the current tick -// assert.True(c, externalValue, "expected 'externalValue' to be true") -// }, 1*time.Second, 10*time.Second, "external state has not changed to 'true'; still false") -func (a *Assertions) EventuallyWithTf(condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return EventuallyWithTf(a.t, condition, waitFor, tick, msg, args...) -} - -// Eventuallyf asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. -// -// a.Eventuallyf(func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") -func (a *Assertions) Eventuallyf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Eventuallyf(a.t, condition, waitFor, tick, msg, args...) -} - -// Exactly asserts that two objects are equal in value and type. -// -// a.Exactly(int32(123), int64(123)) -func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Exactly(a.t, expected, actual, msgAndArgs...) -} - -// Exactlyf asserts that two objects are equal in value and type. -// -// a.Exactlyf(int32(123), int64(123), "error message %s", "formatted") -func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Exactlyf(a.t, expected, actual, msg, args...) -} - -// Fail reports a failure through -func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Fail(a.t, failureMessage, msgAndArgs...) -} - -// FailNow fails test -func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return FailNow(a.t, failureMessage, msgAndArgs...) -} - -// FailNowf fails test -func (a *Assertions) FailNowf(failureMessage string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return FailNowf(a.t, failureMessage, msg, args...) -} - -// Failf reports a failure through -func (a *Assertions) Failf(failureMessage string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Failf(a.t, failureMessage, msg, args...) -} - -// False asserts that the specified value is false. -// -// a.False(myBool) -func (a *Assertions) False(value bool, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return False(a.t, value, msgAndArgs...) -} - -// Falsef asserts that the specified value is false. -// -// a.Falsef(myBool, "error message %s", "formatted") -func (a *Assertions) Falsef(value bool, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Falsef(a.t, value, msg, args...) -} - -// FileExists checks whether a file exists in the given path. It also fails if -// the path points to a directory or there is an error when trying to check the file. -func (a *Assertions) FileExists(path string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return FileExists(a.t, path, msgAndArgs...) -} - -// FileExistsf checks whether a file exists in the given path. It also fails if -// the path points to a directory or there is an error when trying to check the file. -func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return FileExistsf(a.t, path, msg, args...) -} - -// Greater asserts that the first element is greater than the second -// -// a.Greater(2, 1) -// a.Greater(float64(2), float64(1)) -// a.Greater("b", "a") -func (a *Assertions) Greater(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Greater(a.t, e1, e2, msgAndArgs...) -} - -// GreaterOrEqual asserts that the first element is greater than or equal to the second -// -// a.GreaterOrEqual(2, 1) -// a.GreaterOrEqual(2, 2) -// a.GreaterOrEqual("b", "a") -// a.GreaterOrEqual("b", "b") -func (a *Assertions) GreaterOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return GreaterOrEqual(a.t, e1, e2, msgAndArgs...) -} - -// GreaterOrEqualf asserts that the first element is greater than or equal to the second -// -// a.GreaterOrEqualf(2, 1, "error message %s", "formatted") -// a.GreaterOrEqualf(2, 2, "error message %s", "formatted") -// a.GreaterOrEqualf("b", "a", "error message %s", "formatted") -// a.GreaterOrEqualf("b", "b", "error message %s", "formatted") -func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return GreaterOrEqualf(a.t, e1, e2, msg, args...) -} - -// Greaterf asserts that the first element is greater than the second -// -// a.Greaterf(2, 1, "error message %s", "formatted") -// a.Greaterf(float64(2), float64(1), "error message %s", "formatted") -// a.Greaterf("b", "a", "error message %s", "formatted") -func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Greaterf(a.t, e1, e2, msg, args...) -} - -// HTTPBodyContains asserts that a specified handler returns a -// body that contains a string. -// -// a.HTTPBodyContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPBodyContains(a.t, handler, method, url, values, str, msgAndArgs...) -} - -// HTTPBodyContainsf asserts that a specified handler returns a -// body that contains a string. -// -// a.HTTPBodyContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPBodyContainsf(a.t, handler, method, url, values, str, msg, args...) -} - -// HTTPBodyNotContains asserts that a specified handler returns a -// body that does not contain a string. -// -// a.HTTPBodyNotContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPBodyNotContains(a.t, handler, method, url, values, str, msgAndArgs...) -} - -// HTTPBodyNotContainsf asserts that a specified handler returns a -// body that does not contain a string. -// -// a.HTTPBodyNotContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPBodyNotContainsf(a.t, handler, method, url, values, str, msg, args...) -} - -// HTTPError asserts that a specified handler returns an error status code. -// -// a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPError(a.t, handler, method, url, values, msgAndArgs...) -} - -// HTTPErrorf asserts that a specified handler returns an error status code. -// -// a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPErrorf(a.t, handler, method, url, values, msg, args...) -} - -// HTTPRedirect asserts that a specified handler returns a redirect status code. -// -// a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPRedirect(a.t, handler, method, url, values, msgAndArgs...) -} - -// HTTPRedirectf asserts that a specified handler returns a redirect status code. -// -// a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPRedirectf(a.t, handler, method, url, values, msg, args...) -} - -// HTTPStatusCode asserts that a specified handler returns a specified status code. -// -// a.HTTPStatusCode(myHandler, "GET", "/notImplemented", nil, 501) -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPStatusCode(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPStatusCode(a.t, handler, method, url, values, statuscode, msgAndArgs...) -} - -// HTTPStatusCodef asserts that a specified handler returns a specified status code. -// -// a.HTTPStatusCodef(myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPStatusCodef(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPStatusCodef(a.t, handler, method, url, values, statuscode, msg, args...) -} - -// HTTPSuccess asserts that a specified handler returns a success status code. -// -// a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPSuccess(a.t, handler, method, url, values, msgAndArgs...) -} - -// HTTPSuccessf asserts that a specified handler returns a success status code. -// -// a.HTTPSuccessf(myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return HTTPSuccessf(a.t, handler, method, url, values, msg, args...) -} - -// Implements asserts that an object is implemented by the specified interface. -// -// a.Implements((*MyInterface)(nil), new(MyObject)) -func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Implements(a.t, interfaceObject, object, msgAndArgs...) -} - -// Implementsf asserts that an object is implemented by the specified interface. -// -// a.Implementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") -func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Implementsf(a.t, interfaceObject, object, msg, args...) -} - -// InDelta asserts that the two numerals are within delta of each other. -// -// a.InDelta(math.Pi, 22/7.0, 0.01) -func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDelta(a.t, expected, actual, delta, msgAndArgs...) -} - -// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. -func (a *Assertions) InDeltaMapValues(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDeltaMapValues(a.t, expected, actual, delta, msgAndArgs...) -} - -// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. -func (a *Assertions) InDeltaMapValuesf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDeltaMapValuesf(a.t, expected, actual, delta, msg, args...) -} - -// InDeltaSlice is the same as InDelta, except it compares two slices. -func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) -} - -// InDeltaSlicef is the same as InDelta, except it compares two slices. -func (a *Assertions) InDeltaSlicef(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDeltaSlicef(a.t, expected, actual, delta, msg, args...) -} - -// InDeltaf asserts that the two numerals are within delta of each other. -// -// a.InDeltaf(math.Pi, 22/7.0, 0.01, "error message %s", "formatted") -func (a *Assertions) InDeltaf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InDeltaf(a.t, expected, actual, delta, msg, args...) -} - -// InEpsilon asserts that expected and actual have a relative error less than epsilon -func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) -} - -// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. -func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InEpsilonSlice(a.t, expected, actual, epsilon, msgAndArgs...) -} - -// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. -func (a *Assertions) InEpsilonSlicef(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InEpsilonSlicef(a.t, expected, actual, epsilon, msg, args...) -} - -// InEpsilonf asserts that expected and actual have a relative error less than epsilon -func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return InEpsilonf(a.t, expected, actual, epsilon, msg, args...) -} - -// IsDecreasing asserts that the collection is decreasing -// -// a.IsDecreasing([]int{2, 1, 0}) -// a.IsDecreasing([]float{2, 1}) -// a.IsDecreasing([]string{"b", "a"}) -func (a *Assertions) IsDecreasing(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsDecreasing(a.t, object, msgAndArgs...) -} - -// IsDecreasingf asserts that the collection is decreasing -// -// a.IsDecreasingf([]int{2, 1, 0}, "error message %s", "formatted") -// a.IsDecreasingf([]float{2, 1}, "error message %s", "formatted") -// a.IsDecreasingf([]string{"b", "a"}, "error message %s", "formatted") -func (a *Assertions) IsDecreasingf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsDecreasingf(a.t, object, msg, args...) -} - -// IsIncreasing asserts that the collection is increasing -// -// a.IsIncreasing([]int{1, 2, 3}) -// a.IsIncreasing([]float{1, 2}) -// a.IsIncreasing([]string{"a", "b"}) -func (a *Assertions) IsIncreasing(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsIncreasing(a.t, object, msgAndArgs...) -} - -// IsIncreasingf asserts that the collection is increasing -// -// a.IsIncreasingf([]int{1, 2, 3}, "error message %s", "formatted") -// a.IsIncreasingf([]float{1, 2}, "error message %s", "formatted") -// a.IsIncreasingf([]string{"a", "b"}, "error message %s", "formatted") -func (a *Assertions) IsIncreasingf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsIncreasingf(a.t, object, msg, args...) -} - -// IsNonDecreasing asserts that the collection is not decreasing -// -// a.IsNonDecreasing([]int{1, 1, 2}) -// a.IsNonDecreasing([]float{1, 2}) -// a.IsNonDecreasing([]string{"a", "b"}) -func (a *Assertions) IsNonDecreasing(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsNonDecreasing(a.t, object, msgAndArgs...) -} - -// IsNonDecreasingf asserts that the collection is not decreasing -// -// a.IsNonDecreasingf([]int{1, 1, 2}, "error message %s", "formatted") -// a.IsNonDecreasingf([]float{1, 2}, "error message %s", "formatted") -// a.IsNonDecreasingf([]string{"a", "b"}, "error message %s", "formatted") -func (a *Assertions) IsNonDecreasingf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsNonDecreasingf(a.t, object, msg, args...) -} - -// IsNonIncreasing asserts that the collection is not increasing -// -// a.IsNonIncreasing([]int{2, 1, 1}) -// a.IsNonIncreasing([]float{2, 1}) -// a.IsNonIncreasing([]string{"b", "a"}) -func (a *Assertions) IsNonIncreasing(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsNonIncreasing(a.t, object, msgAndArgs...) -} - -// IsNonIncreasingf asserts that the collection is not increasing -// -// a.IsNonIncreasingf([]int{2, 1, 1}, "error message %s", "formatted") -// a.IsNonIncreasingf([]float{2, 1}, "error message %s", "formatted") -// a.IsNonIncreasingf([]string{"b", "a"}, "error message %s", "formatted") -func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsNonIncreasingf(a.t, object, msg, args...) -} - -// IsType asserts that the specified objects are of the same type. -func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsType(a.t, expectedType, object, msgAndArgs...) -} - -// IsTypef asserts that the specified objects are of the same type. -func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return IsTypef(a.t, expectedType, object, msg, args...) -} - -// JSONEq asserts that two JSON strings are equivalent. -// -// a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return JSONEq(a.t, expected, actual, msgAndArgs...) -} - -// JSONEqf asserts that two JSON strings are equivalent. -// -// a.JSONEqf(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -func (a *Assertions) JSONEqf(expected string, actual string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return JSONEqf(a.t, expected, actual, msg, args...) -} - -// Len asserts that the specified object has specific length. -// Len also fails if the object has a type that len() not accept. -// -// a.Len(mySlice, 3) -func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Len(a.t, object, length, msgAndArgs...) -} - -// Lenf asserts that the specified object has specific length. -// Lenf also fails if the object has a type that len() not accept. -// -// a.Lenf(mySlice, 3, "error message %s", "formatted") -func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Lenf(a.t, object, length, msg, args...) -} - -// Less asserts that the first element is less than the second -// -// a.Less(1, 2) -// a.Less(float64(1), float64(2)) -// a.Less("a", "b") -func (a *Assertions) Less(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Less(a.t, e1, e2, msgAndArgs...) -} - -// LessOrEqual asserts that the first element is less than or equal to the second -// -// a.LessOrEqual(1, 2) -// a.LessOrEqual(2, 2) -// a.LessOrEqual("a", "b") -// a.LessOrEqual("b", "b") -func (a *Assertions) LessOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return LessOrEqual(a.t, e1, e2, msgAndArgs...) -} - -// LessOrEqualf asserts that the first element is less than or equal to the second -// -// a.LessOrEqualf(1, 2, "error message %s", "formatted") -// a.LessOrEqualf(2, 2, "error message %s", "formatted") -// a.LessOrEqualf("a", "b", "error message %s", "formatted") -// a.LessOrEqualf("b", "b", "error message %s", "formatted") -func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return LessOrEqualf(a.t, e1, e2, msg, args...) -} - -// Lessf asserts that the first element is less than the second -// -// a.Lessf(1, 2, "error message %s", "formatted") -// a.Lessf(float64(1), float64(2), "error message %s", "formatted") -// a.Lessf("a", "b", "error message %s", "formatted") -func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Lessf(a.t, e1, e2, msg, args...) -} - -// Negative asserts that the specified element is negative -// -// a.Negative(-1) -// a.Negative(-1.23) -func (a *Assertions) Negative(e interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Negative(a.t, e, msgAndArgs...) -} - -// Negativef asserts that the specified element is negative -// -// a.Negativef(-1, "error message %s", "formatted") -// a.Negativef(-1.23, "error message %s", "formatted") -func (a *Assertions) Negativef(e interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Negativef(a.t, e, msg, args...) -} - -// Never asserts that the given condition doesn't satisfy in waitFor time, -// periodically checking the target function each tick. -// -// a.Never(func() bool { return false; }, time.Second, 10*time.Millisecond) -func (a *Assertions) Never(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Never(a.t, condition, waitFor, tick, msgAndArgs...) -} - -// Neverf asserts that the given condition doesn't satisfy in waitFor time, -// periodically checking the target function each tick. -// -// a.Neverf(func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") -func (a *Assertions) Neverf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Neverf(a.t, condition, waitFor, tick, msg, args...) -} - -// Nil asserts that the specified object is nil. -// -// a.Nil(err) -func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Nil(a.t, object, msgAndArgs...) -} - -// Nilf asserts that the specified object is nil. -// -// a.Nilf(err, "error message %s", "formatted") -func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Nilf(a.t, object, msg, args...) -} - -// NoDirExists checks whether a directory does not exist in the given path. -// It fails if the path points to an existing _directory_ only. -func (a *Assertions) NoDirExists(path string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoDirExists(a.t, path, msgAndArgs...) -} - -// NoDirExistsf checks whether a directory does not exist in the given path. -// It fails if the path points to an existing _directory_ only. -func (a *Assertions) NoDirExistsf(path string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoDirExistsf(a.t, path, msg, args...) -} - -// NoError asserts that a function returned no error (i.e. `nil`). -// -// actualObj, err := SomeFunction() -// if a.NoError(err) { -// assert.Equal(t, expectedObj, actualObj) -// } -func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoError(a.t, err, msgAndArgs...) -} - -// NoErrorf asserts that a function returned no error (i.e. `nil`). -// -// actualObj, err := SomeFunction() -// if a.NoErrorf(err, "error message %s", "formatted") { -// assert.Equal(t, expectedObj, actualObj) -// } -func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoErrorf(a.t, err, msg, args...) -} - -// NoFileExists checks whether a file does not exist in a given path. It fails -// if the path points to an existing _file_ only. -func (a *Assertions) NoFileExists(path string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoFileExists(a.t, path, msgAndArgs...) -} - -// NoFileExistsf checks whether a file does not exist in a given path. It fails -// if the path points to an existing _file_ only. -func (a *Assertions) NoFileExistsf(path string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NoFileExistsf(a.t, path, msg, args...) -} - -// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the -// specified substring or element. -// -// a.NotContains("Hello World", "Earth") -// a.NotContains(["Hello", "World"], "Earth") -// a.NotContains({"Hello": "World"}, "Earth") -func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotContains(a.t, s, contains, msgAndArgs...) -} - -// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the -// specified substring or element. -// -// a.NotContainsf("Hello World", "Earth", "error message %s", "formatted") -// a.NotContainsf(["Hello", "World"], "Earth", "error message %s", "formatted") -// a.NotContainsf({"Hello": "World"}, "Earth", "error message %s", "formatted") -func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotContainsf(a.t, s, contains, msg, args...) -} - -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// if a.NotEmpty(obj) { -// assert.Equal(t, "two", obj[1]) -// } -func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEmpty(a.t, object, msgAndArgs...) -} - -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// if a.NotEmptyf(obj, "error message %s", "formatted") { -// assert.Equal(t, "two", obj[1]) -// } -func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEmptyf(a.t, object, msg, args...) -} - -// NotEqual asserts that the specified values are NOT equal. -// -// a.NotEqual(obj1, obj2) -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). -func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEqual(a.t, expected, actual, msgAndArgs...) -} - -// NotEqualValues asserts that two objects are not equal even when converted to the same type -// -// a.NotEqualValues(obj1, obj2) -func (a *Assertions) NotEqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEqualValues(a.t, expected, actual, msgAndArgs...) -} - -// NotEqualValuesf asserts that two objects are not equal even when converted to the same type -// -// a.NotEqualValuesf(obj1, obj2, "error message %s", "formatted") -func (a *Assertions) NotEqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEqualValuesf(a.t, expected, actual, msg, args...) -} - -// NotEqualf asserts that the specified values are NOT equal. -// -// a.NotEqualf(obj1, obj2, "error message %s", "formatted") -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). -func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotEqualf(a.t, expected, actual, msg, args...) -} - -// NotErrorIs asserts that at none of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func (a *Assertions) NotErrorIs(err error, target error, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotErrorIs(a.t, err, target, msgAndArgs...) -} - -// NotErrorIsf asserts that at none of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func (a *Assertions) NotErrorIsf(err error, target error, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotErrorIsf(a.t, err, target, msg, args...) -} - -// NotImplements asserts that an object does not implement the specified interface. -// -// a.NotImplements((*MyInterface)(nil), new(MyObject)) -func (a *Assertions) NotImplements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotImplements(a.t, interfaceObject, object, msgAndArgs...) -} - -// NotImplementsf asserts that an object does not implement the specified interface. -// -// a.NotImplementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") -func (a *Assertions) NotImplementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotImplementsf(a.t, interfaceObject, object, msg, args...) -} - -// NotNil asserts that the specified object is not nil. -// -// a.NotNil(err) -func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotNil(a.t, object, msgAndArgs...) -} - -// NotNilf asserts that the specified object is not nil. -// -// a.NotNilf(err, "error message %s", "formatted") -func (a *Assertions) NotNilf(object interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotNilf(a.t, object, msg, args...) -} - -// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. -// -// a.NotPanics(func(){ RemainCalm() }) -func (a *Assertions) NotPanics(f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotPanics(a.t, f, msgAndArgs...) -} - -// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. -// -// a.NotPanicsf(func(){ RemainCalm() }, "error message %s", "formatted") -func (a *Assertions) NotPanicsf(f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotPanicsf(a.t, f, msg, args...) -} - -// NotRegexp asserts that a specified regexp does not match a string. -// -// a.NotRegexp(regexp.MustCompile("starts"), "it's starting") -// a.NotRegexp("^start", "it's not starting") -func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotRegexp(a.t, rx, str, msgAndArgs...) -} - -// NotRegexpf asserts that a specified regexp does not match a string. -// -// a.NotRegexpf(regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") -// a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") -func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotRegexpf(a.t, rx, str, msg, args...) -} - -// NotSame asserts that two pointers do not reference the same object. -// -// a.NotSame(ptr1, ptr2) -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func (a *Assertions) NotSame(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotSame(a.t, expected, actual, msgAndArgs...) -} - -// NotSamef asserts that two pointers do not reference the same object. -// -// a.NotSamef(ptr1, ptr2, "error message %s", "formatted") -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func (a *Assertions) NotSamef(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotSamef(a.t, expected, actual, msg, args...) -} - -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. -// -// a.NotSubset([1, 3, 4], [1, 2]) -// a.NotSubset({"x": 1, "y": 2}, {"z": 3}) -func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotSubset(a.t, list, subset, msgAndArgs...) -} - -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. -// -// a.NotSubsetf([1, 3, 4], [1, 2], "error message %s", "formatted") -// a.NotSubsetf({"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") -func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotSubsetf(a.t, list, subset, msg, args...) -} - -// NotZero asserts that i is not the zero value for its type. -func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotZero(a.t, i, msgAndArgs...) -} - -// NotZerof asserts that i is not the zero value for its type. -func (a *Assertions) NotZerof(i interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return NotZerof(a.t, i, msg, args...) -} - -// Panics asserts that the code inside the specified PanicTestFunc panics. -// -// a.Panics(func(){ GoCrazy() }) -func (a *Assertions) Panics(f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Panics(a.t, f, msgAndArgs...) -} - -// PanicsWithError asserts that the code inside the specified PanicTestFunc -// panics, and that the recovered panic value is an error that satisfies the -// EqualError comparison. -// -// a.PanicsWithError("crazy error", func(){ GoCrazy() }) -func (a *Assertions) PanicsWithError(errString string, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return PanicsWithError(a.t, errString, f, msgAndArgs...) -} - -// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc -// panics, and that the recovered panic value is an error that satisfies the -// EqualError comparison. -// -// a.PanicsWithErrorf("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -func (a *Assertions) PanicsWithErrorf(errString string, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return PanicsWithErrorf(a.t, errString, f, msg, args...) -} - -// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that -// the recovered panic value equals the expected panic value. -// -// a.PanicsWithValue("crazy error", func(){ GoCrazy() }) -func (a *Assertions) PanicsWithValue(expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return PanicsWithValue(a.t, expected, f, msgAndArgs...) -} - -// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that -// the recovered panic value equals the expected panic value. -// -// a.PanicsWithValuef("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -func (a *Assertions) PanicsWithValuef(expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return PanicsWithValuef(a.t, expected, f, msg, args...) -} - -// Panicsf asserts that the code inside the specified PanicTestFunc panics. -// -// a.Panicsf(func(){ GoCrazy() }, "error message %s", "formatted") -func (a *Assertions) Panicsf(f PanicTestFunc, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Panicsf(a.t, f, msg, args...) -} - -// Positive asserts that the specified element is positive -// -// a.Positive(1) -// a.Positive(1.23) -func (a *Assertions) Positive(e interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Positive(a.t, e, msgAndArgs...) -} - -// Positivef asserts that the specified element is positive -// -// a.Positivef(1, "error message %s", "formatted") -// a.Positivef(1.23, "error message %s", "formatted") -func (a *Assertions) Positivef(e interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Positivef(a.t, e, msg, args...) -} - -// Regexp asserts that a specified regexp matches a string. -// -// a.Regexp(regexp.MustCompile("start"), "it's starting") -// a.Regexp("start...$", "it's not starting") -func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Regexp(a.t, rx, str, msgAndArgs...) -} - -// Regexpf asserts that a specified regexp matches a string. -// -// a.Regexpf(regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") -// a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") -func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Regexpf(a.t, rx, str, msg, args...) -} - -// Same asserts that two pointers reference the same object. -// -// a.Same(ptr1, ptr2) -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func (a *Assertions) Same(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Same(a.t, expected, actual, msgAndArgs...) -} - -// Samef asserts that two pointers reference the same object. -// -// a.Samef(ptr1, ptr2, "error message %s", "formatted") -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Samef(a.t, expected, actual, msg, args...) -} - -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. -// -// a.Subset([1, 2, 3], [1, 2]) -// a.Subset({"x": 1, "y": 2}, {"x": 1}) -func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Subset(a.t, list, subset, msgAndArgs...) -} - -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. -// -// a.Subsetf([1, 2, 3], [1, 2], "error message %s", "formatted") -// a.Subsetf({"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") -func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Subsetf(a.t, list, subset, msg, args...) -} - -// True asserts that the specified value is true. -// -// a.True(myBool) -func (a *Assertions) True(value bool, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return True(a.t, value, msgAndArgs...) -} - -// Truef asserts that the specified value is true. -// -// a.Truef(myBool, "error message %s", "formatted") -func (a *Assertions) Truef(value bool, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Truef(a.t, value, msg, args...) -} - -// WithinDuration asserts that the two times are within duration delta of each other. -// -// a.WithinDuration(time.Now(), time.Now(), 10*time.Second) -func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return WithinDuration(a.t, expected, actual, delta, msgAndArgs...) -} - -// WithinDurationf asserts that the two times are within duration delta of each other. -// -// a.WithinDurationf(time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return WithinDurationf(a.t, expected, actual, delta, msg, args...) -} - -// WithinRange asserts that a time is within a time range (inclusive). -// -// a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) -func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return WithinRange(a.t, actual, start, end, msgAndArgs...) -} - -// WithinRangef asserts that a time is within a time range (inclusive). -// -// a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") -func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return WithinRangef(a.t, actual, start, end, msg, args...) -} - -// YAMLEq asserts that two YAML strings are equivalent. -func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return YAMLEq(a.t, expected, actual, msgAndArgs...) -} - -// YAMLEqf asserts that two YAML strings are equivalent. -func (a *Assertions) YAMLEqf(expected string, actual string, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return YAMLEqf(a.t, expected, actual, msg, args...) -} - -// Zero asserts that i is the zero value for its type. -func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Zero(a.t, i, msgAndArgs...) -} - -// Zerof asserts that i is the zero value for its type. -func (a *Assertions) Zerof(i interface{}, msg string, args ...interface{}) bool { - if h, ok := a.t.(tHelper); ok { - h.Helper() - } - return Zerof(a.t, i, msg, args...) -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl deleted file mode 100644 index 188bb9e174..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -{{.CommentWithoutT "a"}} -func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { - if h, ok := a.t.(tHelper); ok { h.Helper() } - return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_order.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_order.go deleted file mode 100644 index 00df62a059..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertion_order.go +++ /dev/null @@ -1,81 +0,0 @@ -package assert - -import ( - "fmt" - "reflect" -) - -// isOrdered checks that collection contains orderable elements. -func isOrdered(t TestingT, object interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool { - objKind := reflect.TypeOf(object).Kind() - if objKind != reflect.Slice && objKind != reflect.Array { - return false - } - - objValue := reflect.ValueOf(object) - objLen := objValue.Len() - - if objLen <= 1 { - return true - } - - value := objValue.Index(0) - valueInterface := value.Interface() - firstValueKind := value.Kind() - - for i := 1; i < objLen; i++ { - prevValue := value - prevValueInterface := valueInterface - - value = objValue.Index(i) - valueInterface = value.Interface() - - compareResult, isComparable := compare(prevValueInterface, valueInterface, firstValueKind) - - if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\" and \"%s\"", reflect.TypeOf(value), reflect.TypeOf(prevValue)), msgAndArgs...) - } - - if !containsValue(allowedComparesResults, compareResult) { - return Fail(t, fmt.Sprintf(failMessage, prevValue, value), msgAndArgs...) - } - } - - return true -} - -// IsIncreasing asserts that the collection is increasing -// -// assert.IsIncreasing(t, []int{1, 2, 3}) -// assert.IsIncreasing(t, []float{1, 2}) -// assert.IsIncreasing(t, []string{"a", "b"}) -func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - return isOrdered(t, object, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...) -} - -// IsNonIncreasing asserts that the collection is not increasing -// -// assert.IsNonIncreasing(t, []int{2, 1, 1}) -// assert.IsNonIncreasing(t, []float{2, 1}) -// assert.IsNonIncreasing(t, []string{"b", "a"}) -func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - return isOrdered(t, object, []CompareType{compareEqual, compareGreater}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...) -} - -// IsDecreasing asserts that the collection is decreasing -// -// assert.IsDecreasing(t, []int{2, 1, 0}) -// assert.IsDecreasing(t, []float{2, 1}) -// assert.IsDecreasing(t, []string{"b", "a"}) -func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - return isOrdered(t, object, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...) -} - -// IsNonDecreasing asserts that the collection is not decreasing -// -// assert.IsNonDecreasing(t, []int{1, 1, 2}) -// assert.IsNonDecreasing(t, []float{1, 2}) -// assert.IsNonDecreasing(t, []string{"a", "b"}) -func IsNonDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - return isOrdered(t, object, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...) -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertions.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertions.go deleted file mode 100644 index 0b7570f21c..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/assertions.go +++ /dev/null @@ -1,2105 +0,0 @@ -package assert - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "math" - "os" - "reflect" - "regexp" - "runtime" - "runtime/debug" - "strings" - "time" - "unicode" - "unicode/utf8" - - "github.com/davecgh/go-spew/spew" - "github.com/pmezard/go-difflib/difflib" - "gopkg.in/yaml.v3" -) - -//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_format.go.tmpl" - -// TestingT is an interface wrapper around *testing.T -type TestingT interface { - Errorf(format string, args ...interface{}) -} - -// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful -// for table driven tests. -type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) bool - -// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful -// for table driven tests. -type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool - -// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful -// for table driven tests. -type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool - -// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful -// for table driven tests. -type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool - -// Comparison is a custom function that returns true on success and false on failure -type Comparison func() (success bool) - -/* - Helper functions -*/ - -// ObjectsAreEqual determines if two objects are considered equal. -// -// This function does no assertion of any kind. -func ObjectsAreEqual(expected, actual interface{}) bool { - if expected == nil || actual == nil { - return expected == actual - } - - exp, ok := expected.([]byte) - if !ok { - return reflect.DeepEqual(expected, actual) - } - - act, ok := actual.([]byte) - if !ok { - return false - } - if exp == nil || act == nil { - return exp == nil && act == nil - } - return bytes.Equal(exp, act) -} - -// copyExportedFields iterates downward through nested data structures and creates a copy -// that only contains the exported struct fields. -func copyExportedFields(expected interface{}) interface{} { - if isNil(expected) { - return expected - } - - expectedType := reflect.TypeOf(expected) - expectedKind := expectedType.Kind() - expectedValue := reflect.ValueOf(expected) - - switch expectedKind { - case reflect.Struct: - result := reflect.New(expectedType).Elem() - for i := 0; i < expectedType.NumField(); i++ { - field := expectedType.Field(i) - isExported := field.IsExported() - if isExported { - fieldValue := expectedValue.Field(i) - if isNil(fieldValue) || isNil(fieldValue.Interface()) { - continue - } - newValue := copyExportedFields(fieldValue.Interface()) - result.Field(i).Set(reflect.ValueOf(newValue)) - } - } - return result.Interface() - - case reflect.Ptr: - result := reflect.New(expectedType.Elem()) - unexportedRemoved := copyExportedFields(expectedValue.Elem().Interface()) - result.Elem().Set(reflect.ValueOf(unexportedRemoved)) - return result.Interface() - - case reflect.Array, reflect.Slice: - var result reflect.Value - if expectedKind == reflect.Array { - result = reflect.New(reflect.ArrayOf(expectedValue.Len(), expectedType.Elem())).Elem() - } else { - result = reflect.MakeSlice(expectedType, expectedValue.Len(), expectedValue.Len()) - } - for i := 0; i < expectedValue.Len(); i++ { - index := expectedValue.Index(i) - if isNil(index) { - continue - } - unexportedRemoved := copyExportedFields(index.Interface()) - result.Index(i).Set(reflect.ValueOf(unexportedRemoved)) - } - return result.Interface() - - case reflect.Map: - result := reflect.MakeMap(expectedType) - for _, k := range expectedValue.MapKeys() { - index := expectedValue.MapIndex(k) - unexportedRemoved := copyExportedFields(index.Interface()) - result.SetMapIndex(k, reflect.ValueOf(unexportedRemoved)) - } - return result.Interface() - - default: - return expected - } -} - -// ObjectsExportedFieldsAreEqual determines if the exported (public) fields of two objects are -// considered equal. This comparison of only exported fields is applied recursively to nested data -// structures. -// -// This function does no assertion of any kind. -// -// Deprecated: Use [EqualExportedValues] instead. -func ObjectsExportedFieldsAreEqual(expected, actual interface{}) bool { - expectedCleaned := copyExportedFields(expected) - actualCleaned := copyExportedFields(actual) - return ObjectsAreEqualValues(expectedCleaned, actualCleaned) -} - -// ObjectsAreEqualValues gets whether two objects are equal, or if their -// values are equal. -func ObjectsAreEqualValues(expected, actual interface{}) bool { - if ObjectsAreEqual(expected, actual) { - return true - } - - expectedValue := reflect.ValueOf(expected) - actualValue := reflect.ValueOf(actual) - if !expectedValue.IsValid() || !actualValue.IsValid() { - return false - } - - expectedType := expectedValue.Type() - actualType := actualValue.Type() - if !expectedType.ConvertibleTo(actualType) { - return false - } - - if !isNumericType(expectedType) || !isNumericType(actualType) { - // Attempt comparison after type conversion - return reflect.DeepEqual( - expectedValue.Convert(actualType).Interface(), actual, - ) - } - - // If BOTH values are numeric, there are chances of false positives due - // to overflow or underflow. So, we need to make sure to always convert - // the smaller type to a larger type before comparing. - if expectedType.Size() >= actualType.Size() { - return actualValue.Convert(expectedType).Interface() == expected - } - - return expectedValue.Convert(actualType).Interface() == actual -} - -// isNumericType returns true if the type is one of: -// int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, -// float32, float64, complex64, complex128 -func isNumericType(t reflect.Type) bool { - return t.Kind() >= reflect.Int && t.Kind() <= reflect.Complex128 -} - -/* CallerInfo is necessary because the assert functions use the testing object -internally, causing it to print the file:line of the assert method, rather than where -the problem actually occurred in calling code.*/ - -// CallerInfo returns an array of strings containing the file and line number -// of each stack frame leading from the current test to the assert call that -// failed. -func CallerInfo() []string { - - var pc uintptr - var ok bool - var file string - var line int - var name string - - callers := []string{} - for i := 0; ; i++ { - pc, file, line, ok = runtime.Caller(i) - if !ok { - // The breaks below failed to terminate the loop, and we ran off the - // end of the call stack. - break - } - - // This is a huge edge case, but it will panic if this is the case, see #180 - if file == "" { - break - } - - f := runtime.FuncForPC(pc) - if f == nil { - break - } - name = f.Name() - - // testing.tRunner is the standard library function that calls - // tests. Subtests are called directly by tRunner, without going through - // the Test/Benchmark/Example function that contains the t.Run calls, so - // with subtests we should break when we hit tRunner, without adding it - // to the list of callers. - if name == "testing.tRunner" { - break - } - - parts := strings.Split(file, "/") - if len(parts) > 1 { - filename := parts[len(parts)-1] - dir := parts[len(parts)-2] - if (dir != "assert" && dir != "mock" && dir != "require") || filename == "mock_test.go" { - callers = append(callers, fmt.Sprintf("%s:%d", file, line)) - } - } - - // Drop the package - segments := strings.Split(name, ".") - name = segments[len(segments)-1] - if isTest(name, "Test") || - isTest(name, "Benchmark") || - isTest(name, "Example") { - break - } - } - - return callers -} - -// Stolen from the `go test` tool. -// isTest tells whether name looks like a test (or benchmark, according to prefix). -// It is a Test (say) if there is a character after Test that is not a lower-case letter. -// We don't want TesticularCancer. -func isTest(name, prefix string) bool { - if !strings.HasPrefix(name, prefix) { - return false - } - if len(name) == len(prefix) { // "Test" is ok - return true - } - r, _ := utf8.DecodeRuneInString(name[len(prefix):]) - return !unicode.IsLower(r) -} - -func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { - if len(msgAndArgs) == 0 || msgAndArgs == nil { - return "" - } - if len(msgAndArgs) == 1 { - msg := msgAndArgs[0] - if msgAsStr, ok := msg.(string); ok { - return msgAsStr - } - return fmt.Sprintf("%+v", msg) - } - if len(msgAndArgs) > 1 { - return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) - } - return "" -} - -// Aligns the provided message so that all lines after the first line start at the same location as the first line. -// Assumes that the first line starts at the correct location (after carriage return, tab, label, spacer and tab). -// The longestLabelLen parameter specifies the length of the longest label in the output (required because this is the -// basis on which the alignment occurs). -func indentMessageLines(message string, longestLabelLen int) string { - outBuf := new(bytes.Buffer) - - for i, scanner := 0, bufio.NewScanner(strings.NewReader(message)); scanner.Scan(); i++ { - // no need to align first line because it starts at the correct location (after the label) - if i != 0 { - // append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab - outBuf.WriteString("\n\t" + strings.Repeat(" ", longestLabelLen+1) + "\t") - } - outBuf.WriteString(scanner.Text()) - } - - return outBuf.String() -} - -type failNower interface { - FailNow() -} - -// FailNow fails test -func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - Fail(t, failureMessage, msgAndArgs...) - - // We cannot extend TestingT with FailNow() and - // maintain backwards compatibility, so we fallback - // to panicking when FailNow is not available in - // TestingT. - // See issue #263 - - if t, ok := t.(failNower); ok { - t.FailNow() - } else { - panic("test failed and t is missing `FailNow()`") - } - return false -} - -// Fail reports a failure through -func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - content := []labeledContent{ - {"Error Trace", strings.Join(CallerInfo(), "\n\t\t\t")}, - {"Error", failureMessage}, - } - - // Add test name if the Go version supports it - if n, ok := t.(interface { - Name() string - }); ok { - content = append(content, labeledContent{"Test", n.Name()}) - } - - message := messageFromMsgAndArgs(msgAndArgs...) - if len(message) > 0 { - content = append(content, labeledContent{"Messages", message}) - } - - t.Errorf("\n%s", ""+labeledOutput(content...)) - - return false -} - -type labeledContent struct { - label string - content string -} - -// labeledOutput returns a string consisting of the provided labeledContent. Each labeled output is appended in the following manner: -// -// \t{{label}}:{{align_spaces}}\t{{content}}\n -// -// The initial carriage return is required to undo/erase any padding added by testing.T.Errorf. The "\t{{label}}:" is for the label. -// If a label is shorter than the longest label provided, padding spaces are added to make all the labels match in length. Once this -// alignment is achieved, "\t{{content}}\n" is added for the output. -// -// If the content of the labeledOutput contains line breaks, the subsequent lines are aligned so that they start at the same location as the first line. -func labeledOutput(content ...labeledContent) string { - longestLabel := 0 - for _, v := range content { - if len(v.label) > longestLabel { - longestLabel = len(v.label) - } - } - var output string - for _, v := range content { - output += "\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n" - } - return output -} - -// Implements asserts that an object is implemented by the specified interface. -// -// assert.Implements(t, (*MyInterface)(nil), new(MyObject)) -func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - interfaceType := reflect.TypeOf(interfaceObject).Elem() - - if object == nil { - return Fail(t, fmt.Sprintf("Cannot check if nil implements %v", interfaceType), msgAndArgs...) - } - if !reflect.TypeOf(object).Implements(interfaceType) { - return Fail(t, fmt.Sprintf("%T must implement %v", object, interfaceType), msgAndArgs...) - } - - return true -} - -// NotImplements asserts that an object does not implement the specified interface. -// -// assert.NotImplements(t, (*MyInterface)(nil), new(MyObject)) -func NotImplements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - interfaceType := reflect.TypeOf(interfaceObject).Elem() - - if object == nil { - return Fail(t, fmt.Sprintf("Cannot check if nil does not implement %v", interfaceType), msgAndArgs...) - } - if reflect.TypeOf(object).Implements(interfaceType) { - return Fail(t, fmt.Sprintf("%T implements %v", object, interfaceType), msgAndArgs...) - } - - return true -} - -// IsType asserts that the specified objects are of the same type. -func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if !ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) { - return Fail(t, fmt.Sprintf("Object expected to be of type %v, but was %v", reflect.TypeOf(expectedType), reflect.TypeOf(object)), msgAndArgs...) - } - - return true -} - -// Equal asserts that two objects are equal. -// -// assert.Equal(t, 123, 123) -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). Function equality -// cannot be determined and will always fail. -func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if err := validateEqualArgs(expected, actual); err != nil { - return Fail(t, fmt.Sprintf("Invalid operation: %#v == %#v (%s)", - expected, actual, err), msgAndArgs...) - } - - if !ObjectsAreEqual(expected, actual) { - diff := diff(expected, actual) - expected, actual = formatUnequalValues(expected, actual) - return Fail(t, fmt.Sprintf("Not equal: \n"+ - "expected: %s\n"+ - "actual : %s%s", expected, actual, diff), msgAndArgs...) - } - - return true - -} - -// validateEqualArgs checks whether provided arguments can be safely used in the -// Equal/NotEqual functions. -func validateEqualArgs(expected, actual interface{}) error { - if expected == nil && actual == nil { - return nil - } - - if isFunction(expected) || isFunction(actual) { - return errors.New("cannot take func type as argument") - } - return nil -} - -// Same asserts that two pointers reference the same object. -// -// assert.Same(t, ptr1, ptr2) -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func Same(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if !samePointers(expected, actual) { - return Fail(t, fmt.Sprintf("Not same: \n"+ - "expected: %p %#v\n"+ - "actual : %p %#v", expected, expected, actual, actual), msgAndArgs...) - } - - return true -} - -// NotSame asserts that two pointers do not reference the same object. -// -// assert.NotSame(t, ptr1, ptr2) -// -// Both arguments must be pointer variables. Pointer variable sameness is -// determined based on the equality of both type and value. -func NotSame(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if samePointers(expected, actual) { - return Fail(t, fmt.Sprintf( - "Expected and actual point to the same object: %p %#v", - expected, expected), msgAndArgs...) - } - return true -} - -// samePointers compares two generic interface objects and returns whether -// they point to the same object -func samePointers(first, second interface{}) bool { - firstPtr, secondPtr := reflect.ValueOf(first), reflect.ValueOf(second) - if firstPtr.Kind() != reflect.Ptr || secondPtr.Kind() != reflect.Ptr { - return false - } - - firstType, secondType := reflect.TypeOf(first), reflect.TypeOf(second) - if firstType != secondType { - return false - } - - // compare pointer addresses - return first == second -} - -// formatUnequalValues takes two values of arbitrary types and returns string -// representations appropriate to be presented to the user. -// -// If the values are not of like type, the returned strings will be prefixed -// with the type name, and the value will be enclosed in parentheses similar -// to a type conversion in the Go grammar. -func formatUnequalValues(expected, actual interface{}) (e string, a string) { - if reflect.TypeOf(expected) != reflect.TypeOf(actual) { - return fmt.Sprintf("%T(%s)", expected, truncatingFormat(expected)), - fmt.Sprintf("%T(%s)", actual, truncatingFormat(actual)) - } - switch expected.(type) { - case time.Duration: - return fmt.Sprintf("%v", expected), fmt.Sprintf("%v", actual) - } - return truncatingFormat(expected), truncatingFormat(actual) -} - -// truncatingFormat formats the data and truncates it if it's too long. -// -// This helps keep formatted error messages lines from exceeding the -// bufio.MaxScanTokenSize max line length that the go testing framework imposes. -func truncatingFormat(data interface{}) string { - value := fmt.Sprintf("%#v", data) - max := bufio.MaxScanTokenSize - 100 // Give us some space the type info too if needed. - if len(value) > max { - value = value[0:max] + "<... truncated>" - } - return value -} - -// EqualValues asserts that two objects are equal or convertible to the same types -// and equal. -// -// assert.EqualValues(t, uint32(123), int32(123)) -func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if !ObjectsAreEqualValues(expected, actual) { - diff := diff(expected, actual) - expected, actual = formatUnequalValues(expected, actual) - return Fail(t, fmt.Sprintf("Not equal: \n"+ - "expected: %s\n"+ - "actual : %s%s", expected, actual, diff), msgAndArgs...) - } - - return true - -} - -// EqualExportedValues asserts that the types of two objects are equal and their public -// fields are also equal. This is useful for comparing structs that have private fields -// that could potentially differ. -// -// type S struct { -// Exported int -// notExported int -// } -// assert.EqualExportedValues(t, S{1, 2}, S{1, 3}) => true -// assert.EqualExportedValues(t, S{1, 2}, S{2, 3}) => false -func EqualExportedValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - aType := reflect.TypeOf(expected) - bType := reflect.TypeOf(actual) - - if aType != bType { - return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...) - } - - if aType.Kind() == reflect.Ptr { - aType = aType.Elem() - } - if bType.Kind() == reflect.Ptr { - bType = bType.Elem() - } - - if aType.Kind() != reflect.Struct { - return Fail(t, fmt.Sprintf("Types expected to both be struct or pointer to struct \n\t%v != %v", aType.Kind(), reflect.Struct), msgAndArgs...) - } - - if bType.Kind() != reflect.Struct { - return Fail(t, fmt.Sprintf("Types expected to both be struct or pointer to struct \n\t%v != %v", bType.Kind(), reflect.Struct), msgAndArgs...) - } - - expected = copyExportedFields(expected) - actual = copyExportedFields(actual) - - if !ObjectsAreEqualValues(expected, actual) { - diff := diff(expected, actual) - expected, actual = formatUnequalValues(expected, actual) - return Fail(t, fmt.Sprintf("Not equal (comparing only exported fields): \n"+ - "expected: %s\n"+ - "actual : %s%s", expected, actual, diff), msgAndArgs...) - } - - return true -} - -// Exactly asserts that two objects are equal in value and type. -// -// assert.Exactly(t, int32(123), int64(123)) -func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - aType := reflect.TypeOf(expected) - bType := reflect.TypeOf(actual) - - if aType != bType { - return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...) - } - - return Equal(t, expected, actual, msgAndArgs...) - -} - -// NotNil asserts that the specified object is not nil. -// -// assert.NotNil(t, err) -func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if !isNil(object) { - return true - } - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, "Expected value not to be nil.", msgAndArgs...) -} - -// isNil checks if a specified object is nil or not, without Failing. -func isNil(object interface{}) bool { - if object == nil { - return true - } - - value := reflect.ValueOf(object) - switch value.Kind() { - case - reflect.Chan, reflect.Func, - reflect.Interface, reflect.Map, - reflect.Ptr, reflect.Slice, reflect.UnsafePointer: - - return value.IsNil() - } - - return false -} - -// Nil asserts that the specified object is nil. -// -// assert.Nil(t, err) -func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if isNil(object) { - return true - } - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...) -} - -// isEmpty gets whether the specified object is considered empty or not. -func isEmpty(object interface{}) bool { - - // get nil case out of the way - if object == nil { - return true - } - - objValue := reflect.ValueOf(object) - - switch objValue.Kind() { - // collection types are empty when they have no element - case reflect.Chan, reflect.Map, reflect.Slice: - return objValue.Len() == 0 - // pointers are empty if nil or if the value they point to is empty - case reflect.Ptr: - if objValue.IsNil() { - return true - } - deref := objValue.Elem().Interface() - return isEmpty(deref) - // for all other types, compare against the zero value - // array types are empty when they match their zero-initialized state - default: - zero := reflect.Zero(objValue.Type()) - return reflect.DeepEqual(object, zero.Interface()) - } -} - -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// assert.Empty(t, obj) -func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - pass := isEmpty(object) - if !pass { - if h, ok := t.(tHelper); ok { - h.Helper() - } - Fail(t, fmt.Sprintf("Should be empty, but was %v", object), msgAndArgs...) - } - - return pass - -} - -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. -// -// if assert.NotEmpty(t, obj) { -// assert.Equal(t, "two", obj[1]) -// } -func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - pass := !isEmpty(object) - if !pass { - if h, ok := t.(tHelper); ok { - h.Helper() - } - Fail(t, fmt.Sprintf("Should NOT be empty, but was %v", object), msgAndArgs...) - } - - return pass - -} - -// getLen tries to get the length of an object. -// It returns (0, false) if impossible. -func getLen(x interface{}) (length int, ok bool) { - v := reflect.ValueOf(x) - defer func() { - ok = recover() == nil - }() - return v.Len(), true -} - -// Len asserts that the specified object has specific length. -// Len also fails if the object has a type that len() not accept. -// -// assert.Len(t, mySlice, 3) -func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - l, ok := getLen(object) - if !ok { - return Fail(t, fmt.Sprintf("\"%v\" could not be applied builtin len()", object), msgAndArgs...) - } - - if l != length { - return Fail(t, fmt.Sprintf("\"%v\" should have %d item(s), but has %d", object, length, l), msgAndArgs...) - } - return true -} - -// True asserts that the specified value is true. -// -// assert.True(t, myBool) -func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { - if !value { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, "Should be true", msgAndArgs...) - } - - return true - -} - -// False asserts that the specified value is false. -// -// assert.False(t, myBool) -func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { - if value { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, "Should be false", msgAndArgs...) - } - - return true - -} - -// NotEqual asserts that the specified values are NOT equal. -// -// assert.NotEqual(t, obj1, obj2) -// -// Pointer variable equality is determined based on the equality of the -// referenced values (as opposed to the memory addresses). -func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if err := validateEqualArgs(expected, actual); err != nil { - return Fail(t, fmt.Sprintf("Invalid operation: %#v != %#v (%s)", - expected, actual, err), msgAndArgs...) - } - - if ObjectsAreEqual(expected, actual) { - return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) - } - - return true - -} - -// NotEqualValues asserts that two objects are not equal even when converted to the same type -// -// assert.NotEqualValues(t, obj1, obj2) -func NotEqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if ObjectsAreEqualValues(expected, actual) { - return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) - } - - return true -} - -// containsElement try loop over the list check if the list includes the element. -// return (false, false) if impossible. -// return (true, false) if element was not found. -// return (true, true) if element was found. -func containsElement(list interface{}, element interface{}) (ok, found bool) { - - listValue := reflect.ValueOf(list) - listType := reflect.TypeOf(list) - if listType == nil { - return false, false - } - listKind := listType.Kind() - defer func() { - if e := recover(); e != nil { - ok = false - found = false - } - }() - - if listKind == reflect.String { - elementValue := reflect.ValueOf(element) - return true, strings.Contains(listValue.String(), elementValue.String()) - } - - if listKind == reflect.Map { - mapKeys := listValue.MapKeys() - for i := 0; i < len(mapKeys); i++ { - if ObjectsAreEqual(mapKeys[i].Interface(), element) { - return true, true - } - } - return true, false - } - - for i := 0; i < listValue.Len(); i++ { - if ObjectsAreEqual(listValue.Index(i).Interface(), element) { - return true, true - } - } - return true, false - -} - -// Contains asserts that the specified string, list(array, slice...) or map contains the -// specified substring or element. -// -// assert.Contains(t, "Hello World", "World") -// assert.Contains(t, ["Hello", "World"], "World") -// assert.Contains(t, {"Hello": "World"}, "Hello") -func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - ok, found := containsElement(s, contains) - if !ok { - return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...) - } - if !found { - return Fail(t, fmt.Sprintf("%#v does not contain %#v", s, contains), msgAndArgs...) - } - - return true - -} - -// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the -// specified substring or element. -// -// assert.NotContains(t, "Hello World", "Earth") -// assert.NotContains(t, ["Hello", "World"], "Earth") -// assert.NotContains(t, {"Hello": "World"}, "Earth") -func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - ok, found := containsElement(s, contains) - if !ok { - return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...) - } - if found { - return Fail(t, fmt.Sprintf("%#v should not contain %#v", s, contains), msgAndArgs...) - } - - return true - -} - -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. -// -// assert.Subset(t, [1, 2, 3], [1, 2]) -// assert.Subset(t, {"x": 1, "y": 2}, {"x": 1}) -func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if subset == nil { - return true // we consider nil to be equal to the nil set - } - - listKind := reflect.TypeOf(list).Kind() - if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) - } - - subsetKind := reflect.TypeOf(subset).Kind() - if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) - } - - if subsetKind == reflect.Map && listKind == reflect.Map { - subsetMap := reflect.ValueOf(subset) - actualMap := reflect.ValueOf(list) - - for _, k := range subsetMap.MapKeys() { - ev := subsetMap.MapIndex(k) - av := actualMap.MapIndex(k) - - if !av.IsValid() { - return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, subset), msgAndArgs...) - } - if !ObjectsAreEqual(ev.Interface(), av.Interface()) { - return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, subset), msgAndArgs...) - } - } - - return true - } - - subsetList := reflect.ValueOf(subset) - for i := 0; i < subsetList.Len(); i++ { - element := subsetList.Index(i).Interface() - ok, found := containsElement(list, element) - if !ok { - return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", list), msgAndArgs...) - } - if !found { - return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, element), msgAndArgs...) - } - } - - return true -} - -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. -// -// assert.NotSubset(t, [1, 3, 4], [1, 2]) -// assert.NotSubset(t, {"x": 1, "y": 2}, {"z": 3}) -func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if subset == nil { - return Fail(t, "nil is the empty set which is a subset of every set", msgAndArgs...) - } - - listKind := reflect.TypeOf(list).Kind() - if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) - } - - subsetKind := reflect.TypeOf(subset).Kind() - if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) - } - - if subsetKind == reflect.Map && listKind == reflect.Map { - subsetMap := reflect.ValueOf(subset) - actualMap := reflect.ValueOf(list) - - for _, k := range subsetMap.MapKeys() { - ev := subsetMap.MapIndex(k) - av := actualMap.MapIndex(k) - - if !av.IsValid() { - return true - } - if !ObjectsAreEqual(ev.Interface(), av.Interface()) { - return true - } - } - - return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) - } - - subsetList := reflect.ValueOf(subset) - for i := 0; i < subsetList.Len(); i++ { - element := subsetList.Index(i).Interface() - ok, found := containsElement(list, element) - if !ok { - return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", list), msgAndArgs...) - } - if !found { - return true - } - } - - return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) -} - -// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified -// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, -// the number of appearances of each of them in both lists should match. -// -// assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) -func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if isEmpty(listA) && isEmpty(listB) { - return true - } - - if !isList(t, listA, msgAndArgs...) || !isList(t, listB, msgAndArgs...) { - return false - } - - extraA, extraB := diffLists(listA, listB) - - if len(extraA) == 0 && len(extraB) == 0 { - return true - } - - return Fail(t, formatListDiff(listA, listB, extraA, extraB), msgAndArgs...) -} - -// isList checks that the provided value is array or slice. -func isList(t TestingT, list interface{}, msgAndArgs ...interface{}) (ok bool) { - kind := reflect.TypeOf(list).Kind() - if kind != reflect.Array && kind != reflect.Slice { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s, expecting array or slice", list, kind), - msgAndArgs...) - } - return true -} - -// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. -// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and -// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. -func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { - aValue := reflect.ValueOf(listA) - bValue := reflect.ValueOf(listB) - - aLen := aValue.Len() - bLen := bValue.Len() - - // Mark indexes in bValue that we already used - visited := make([]bool, bLen) - for i := 0; i < aLen; i++ { - element := aValue.Index(i).Interface() - found := false - for j := 0; j < bLen; j++ { - if visited[j] { - continue - } - if ObjectsAreEqual(bValue.Index(j).Interface(), element) { - visited[j] = true - found = true - break - } - } - if !found { - extraA = append(extraA, element) - } - } - - for j := 0; j < bLen; j++ { - if visited[j] { - continue - } - extraB = append(extraB, bValue.Index(j).Interface()) - } - - return -} - -func formatListDiff(listA, listB interface{}, extraA, extraB []interface{}) string { - var msg bytes.Buffer - - msg.WriteString("elements differ") - if len(extraA) > 0 { - msg.WriteString("\n\nextra elements in list A:\n") - msg.WriteString(spewConfig.Sdump(extraA)) - } - if len(extraB) > 0 { - msg.WriteString("\n\nextra elements in list B:\n") - msg.WriteString(spewConfig.Sdump(extraB)) - } - msg.WriteString("\n\nlistA:\n") - msg.WriteString(spewConfig.Sdump(listA)) - msg.WriteString("\n\nlistB:\n") - msg.WriteString(spewConfig.Sdump(listB)) - - return msg.String() -} - -// Condition uses a Comparison to assert a complex condition. -func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - result := comp() - if !result { - Fail(t, "Condition failed!", msgAndArgs...) - } - return result -} - -// PanicTestFunc defines a func that should be passed to the assert.Panics and assert.NotPanics -// methods, and represents a simple func that takes no arguments, and returns nothing. -type PanicTestFunc func() - -// didPanic returns true if the function passed to it panics. Otherwise, it returns false. -func didPanic(f PanicTestFunc) (didPanic bool, message interface{}, stack string) { - didPanic = true - - defer func() { - message = recover() - if didPanic { - stack = string(debug.Stack()) - } - }() - - // call the target function - f() - didPanic = false - - return -} - -// Panics asserts that the code inside the specified PanicTestFunc panics. -// -// assert.Panics(t, func(){ GoCrazy() }) -func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if funcDidPanic, panicValue, _ := didPanic(f); !funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) - } - - return true -} - -// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that -// the recovered panic value equals the expected panic value. -// -// assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) -func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - funcDidPanic, panicValue, panickedStack := didPanic(f) - if !funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) - } - if panicValue != expected { - return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%#v\n\tPanic value:\t%#v\n\tPanic stack:\t%s", f, expected, panicValue, panickedStack), msgAndArgs...) - } - - return true -} - -// PanicsWithError asserts that the code inside the specified PanicTestFunc -// panics, and that the recovered panic value is an error that satisfies the -// EqualError comparison. -// -// assert.PanicsWithError(t, "crazy error", func(){ GoCrazy() }) -func PanicsWithError(t TestingT, errString string, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - funcDidPanic, panicValue, panickedStack := didPanic(f) - if !funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) - } - panicErr, ok := panicValue.(error) - if !ok || panicErr.Error() != errString { - return Fail(t, fmt.Sprintf("func %#v should panic with error message:\t%#v\n\tPanic value:\t%#v\n\tPanic stack:\t%s", f, errString, panicValue, panickedStack), msgAndArgs...) - } - - return true -} - -// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. -// -// assert.NotPanics(t, func(){ RemainCalm() }) -func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if funcDidPanic, panicValue, panickedStack := didPanic(f); funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should not panic\n\tPanic value:\t%v\n\tPanic stack:\t%s", f, panicValue, panickedStack), msgAndArgs...) - } - - return true -} - -// WithinDuration asserts that the two times are within duration delta of each other. -// -// assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) -func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - dt := expected.Sub(actual) - if dt < -delta || dt > delta { - return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) - } - - return true -} - -// WithinRange asserts that a time is within a time range (inclusive). -// -// assert.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) -func WithinRange(t TestingT, actual, start, end time.Time, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if end.Before(start) { - return Fail(t, "Start should be before end", msgAndArgs...) - } - - if actual.Before(start) { - return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is before the range", actual, start, end), msgAndArgs...) - } else if actual.After(end) { - return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is after the range", actual, start, end), msgAndArgs...) - } - - return true -} - -func toFloat(x interface{}) (float64, bool) { - var xf float64 - xok := true - - switch xn := x.(type) { - case uint: - xf = float64(xn) - case uint8: - xf = float64(xn) - case uint16: - xf = float64(xn) - case uint32: - xf = float64(xn) - case uint64: - xf = float64(xn) - case int: - xf = float64(xn) - case int8: - xf = float64(xn) - case int16: - xf = float64(xn) - case int32: - xf = float64(xn) - case int64: - xf = float64(xn) - case float32: - xf = float64(xn) - case float64: - xf = xn - case time.Duration: - xf = float64(xn) - default: - xok = false - } - - return xf, xok -} - -// InDelta asserts that the two numerals are within delta of each other. -// -// assert.InDelta(t, math.Pi, 22/7.0, 0.01) -func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - af, aok := toFloat(expected) - bf, bok := toFloat(actual) - - if !aok || !bok { - return Fail(t, "Parameters must be numerical", msgAndArgs...) - } - - if math.IsNaN(af) && math.IsNaN(bf) { - return true - } - - if math.IsNaN(af) { - return Fail(t, "Expected must not be NaN", msgAndArgs...) - } - - if math.IsNaN(bf) { - return Fail(t, fmt.Sprintf("Expected %v with delta %v, but was NaN", expected, delta), msgAndArgs...) - } - - dt := af - bf - if dt < -delta || dt > delta { - return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) - } - - return true -} - -// InDeltaSlice is the same as InDelta, except it compares two slices. -func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if expected == nil || actual == nil || - reflect.TypeOf(actual).Kind() != reflect.Slice || - reflect.TypeOf(expected).Kind() != reflect.Slice { - return Fail(t, "Parameters must be slice", msgAndArgs...) - } - - actualSlice := reflect.ValueOf(actual) - expectedSlice := reflect.ValueOf(expected) - - for i := 0; i < actualSlice.Len(); i++ { - result := InDelta(t, actualSlice.Index(i).Interface(), expectedSlice.Index(i).Interface(), delta, msgAndArgs...) - if !result { - return result - } - } - - return true -} - -// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. -func InDeltaMapValues(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if expected == nil || actual == nil || - reflect.TypeOf(actual).Kind() != reflect.Map || - reflect.TypeOf(expected).Kind() != reflect.Map { - return Fail(t, "Arguments must be maps", msgAndArgs...) - } - - expectedMap := reflect.ValueOf(expected) - actualMap := reflect.ValueOf(actual) - - if expectedMap.Len() != actualMap.Len() { - return Fail(t, "Arguments must have the same number of keys", msgAndArgs...) - } - - for _, k := range expectedMap.MapKeys() { - ev := expectedMap.MapIndex(k) - av := actualMap.MapIndex(k) - - if !ev.IsValid() { - return Fail(t, fmt.Sprintf("missing key %q in expected map", k), msgAndArgs...) - } - - if !av.IsValid() { - return Fail(t, fmt.Sprintf("missing key %q in actual map", k), msgAndArgs...) - } - - if !InDelta( - t, - ev.Interface(), - av.Interface(), - delta, - msgAndArgs..., - ) { - return false - } - } - - return true -} - -func calcRelativeError(expected, actual interface{}) (float64, error) { - af, aok := toFloat(expected) - bf, bok := toFloat(actual) - if !aok || !bok { - return 0, fmt.Errorf("Parameters must be numerical") - } - if math.IsNaN(af) && math.IsNaN(bf) { - return 0, nil - } - if math.IsNaN(af) { - return 0, errors.New("expected value must not be NaN") - } - if af == 0 { - return 0, fmt.Errorf("expected value must have a value other than zero to calculate the relative error") - } - if math.IsNaN(bf) { - return 0, errors.New("actual value must not be NaN") - } - - return math.Abs(af-bf) / math.Abs(af), nil -} - -// InEpsilon asserts that expected and actual have a relative error less than epsilon -func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if math.IsNaN(epsilon) { - return Fail(t, "epsilon must not be NaN", msgAndArgs...) - } - actualEpsilon, err := calcRelativeError(expected, actual) - if err != nil { - return Fail(t, err.Error(), msgAndArgs...) - } - if actualEpsilon > epsilon { - return Fail(t, fmt.Sprintf("Relative error is too high: %#v (expected)\n"+ - " < %#v (actual)", epsilon, actualEpsilon), msgAndArgs...) - } - - return true -} - -// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. -func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if expected == nil || actual == nil { - return Fail(t, "Parameters must be slice", msgAndArgs...) - } - - expectedSlice := reflect.ValueOf(expected) - actualSlice := reflect.ValueOf(actual) - - if expectedSlice.Type().Kind() != reflect.Slice { - return Fail(t, "Expected value must be slice", msgAndArgs...) - } - - expectedLen := expectedSlice.Len() - if !IsType(t, expected, actual) || !Len(t, actual, expectedLen) { - return false - } - - for i := 0; i < expectedLen; i++ { - if !InEpsilon(t, expectedSlice.Index(i).Interface(), actualSlice.Index(i).Interface(), epsilon, "at index %d", i) { - return false - } - } - - return true -} - -/* - Errors -*/ - -// NoError asserts that a function returned no error (i.e. `nil`). -// -// actualObj, err := SomeFunction() -// if assert.NoError(t, err) { -// assert.Equal(t, expectedObj, actualObj) -// } -func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { - if err != nil { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...) - } - - return true -} - -// Error asserts that a function returned an error (i.e. not `nil`). -// -// actualObj, err := SomeFunction() -// if assert.Error(t, err) { -// assert.Equal(t, expectedError, err) -// } -func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { - if err == nil { - if h, ok := t.(tHelper); ok { - h.Helper() - } - return Fail(t, "An error is expected but got nil.", msgAndArgs...) - } - - return true -} - -// EqualError asserts that a function returned an error (i.e. not `nil`) -// and that it is equal to the provided error. -// -// actualObj, err := SomeFunction() -// assert.EqualError(t, err, expectedErrorString) -func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if !Error(t, theError, msgAndArgs...) { - return false - } - expected := errString - actual := theError.Error() - // don't need to use deep equals here, we know they are both strings - if expected != actual { - return Fail(t, fmt.Sprintf("Error message not equal:\n"+ - "expected: %q\n"+ - "actual : %q", expected, actual), msgAndArgs...) - } - return true -} - -// ErrorContains asserts that a function returned an error (i.e. not `nil`) -// and that the error contains the specified substring. -// -// actualObj, err := SomeFunction() -// assert.ErrorContains(t, err, expectedErrorSubString) -func ErrorContains(t TestingT, theError error, contains string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if !Error(t, theError, msgAndArgs...) { - return false - } - - actual := theError.Error() - if !strings.Contains(actual, contains) { - return Fail(t, fmt.Sprintf("Error %#v does not contain %#v", actual, contains), msgAndArgs...) - } - - return true -} - -// matchRegexp return true if a specified regexp matches a string. -func matchRegexp(rx interface{}, str interface{}) bool { - - var r *regexp.Regexp - if rr, ok := rx.(*regexp.Regexp); ok { - r = rr - } else { - r = regexp.MustCompile(fmt.Sprint(rx)) - } - - return (r.FindStringIndex(fmt.Sprint(str)) != nil) - -} - -// Regexp asserts that a specified regexp matches a string. -// -// assert.Regexp(t, regexp.MustCompile("start"), "it's starting") -// assert.Regexp(t, "start...$", "it's not starting") -func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - match := matchRegexp(rx, str) - - if !match { - Fail(t, fmt.Sprintf("Expect \"%v\" to match \"%v\"", str, rx), msgAndArgs...) - } - - return match -} - -// NotRegexp asserts that a specified regexp does not match a string. -// -// assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") -// assert.NotRegexp(t, "^start", "it's not starting") -func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - match := matchRegexp(rx, str) - - if match { - Fail(t, fmt.Sprintf("Expect \"%v\" to NOT match \"%v\"", str, rx), msgAndArgs...) - } - - return !match - -} - -// Zero asserts that i is the zero value for its type. -func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if i != nil && !reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { - return Fail(t, fmt.Sprintf("Should be zero, but was %v", i), msgAndArgs...) - } - return true -} - -// NotZero asserts that i is not the zero value for its type. -func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if i == nil || reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { - return Fail(t, fmt.Sprintf("Should not be zero, but was %v", i), msgAndArgs...) - } - return true -} - -// FileExists checks whether a file exists in the given path. It also fails if -// the path points to a directory or there is an error when trying to check the file. -func FileExists(t TestingT, path string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - info, err := os.Lstat(path) - if err != nil { - if os.IsNotExist(err) { - return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) - } - return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) - } - if info.IsDir() { - return Fail(t, fmt.Sprintf("%q is a directory", path), msgAndArgs...) - } - return true -} - -// NoFileExists checks whether a file does not exist in a given path. It fails -// if the path points to an existing _file_ only. -func NoFileExists(t TestingT, path string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - info, err := os.Lstat(path) - if err != nil { - return true - } - if info.IsDir() { - return true - } - return Fail(t, fmt.Sprintf("file %q exists", path), msgAndArgs...) -} - -// DirExists checks whether a directory exists in the given path. It also fails -// if the path is a file rather a directory or there is an error checking whether it exists. -func DirExists(t TestingT, path string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - info, err := os.Lstat(path) - if err != nil { - if os.IsNotExist(err) { - return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) - } - return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) - } - if !info.IsDir() { - return Fail(t, fmt.Sprintf("%q is a file", path), msgAndArgs...) - } - return true -} - -// NoDirExists checks whether a directory does not exist in the given path. -// It fails if the path points to an existing _directory_ only. -func NoDirExists(t TestingT, path string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - info, err := os.Lstat(path) - if err != nil { - if os.IsNotExist(err) { - return true - } - return true - } - if !info.IsDir() { - return true - } - return Fail(t, fmt.Sprintf("directory %q exists", path), msgAndArgs...) -} - -// JSONEq asserts that two JSON strings are equivalent. -// -// assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - var expectedJSONAsInterface, actualJSONAsInterface interface{} - - if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil { - return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", expected, err.Error()), msgAndArgs...) - } - - if err := json.Unmarshal([]byte(actual), &actualJSONAsInterface); err != nil { - return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", actual, err.Error()), msgAndArgs...) - } - - return Equal(t, expectedJSONAsInterface, actualJSONAsInterface, msgAndArgs...) -} - -// YAMLEq asserts that two YAML strings are equivalent. -func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - var expectedYAMLAsInterface, actualYAMLAsInterface interface{} - - if err := yaml.Unmarshal([]byte(expected), &expectedYAMLAsInterface); err != nil { - return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid yaml.\nYAML parsing error: '%s'", expected, err.Error()), msgAndArgs...) - } - - if err := yaml.Unmarshal([]byte(actual), &actualYAMLAsInterface); err != nil { - return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid yaml.\nYAML error: '%s'", actual, err.Error()), msgAndArgs...) - } - - return Equal(t, expectedYAMLAsInterface, actualYAMLAsInterface, msgAndArgs...) -} - -func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { - t := reflect.TypeOf(v) - k := t.Kind() - - if k == reflect.Ptr { - t = t.Elem() - k = t.Kind() - } - return t, k -} - -// diff returns a diff of both values as long as both are of the same type and -// are a struct, map, slice, array or string. Otherwise it returns an empty string. -func diff(expected interface{}, actual interface{}) string { - if expected == nil || actual == nil { - return "" - } - - et, ek := typeAndKind(expected) - at, _ := typeAndKind(actual) - - if et != at { - return "" - } - - if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array && ek != reflect.String { - return "" - } - - var e, a string - - switch et { - case reflect.TypeOf(""): - e = reflect.ValueOf(expected).String() - a = reflect.ValueOf(actual).String() - case reflect.TypeOf(time.Time{}): - e = spewConfigStringerEnabled.Sdump(expected) - a = spewConfigStringerEnabled.Sdump(actual) - default: - e = spewConfig.Sdump(expected) - a = spewConfig.Sdump(actual) - } - - diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ - A: difflib.SplitLines(e), - B: difflib.SplitLines(a), - FromFile: "Expected", - FromDate: "", - ToFile: "Actual", - ToDate: "", - Context: 1, - }) - - return "\n\nDiff:\n" + diff -} - -func isFunction(arg interface{}) bool { - if arg == nil { - return false - } - return reflect.TypeOf(arg).Kind() == reflect.Func -} - -var spewConfig = spew.ConfigState{ - Indent: " ", - DisablePointerAddresses: true, - DisableCapacities: true, - SortKeys: true, - DisableMethods: true, - MaxDepth: 10, -} - -var spewConfigStringerEnabled = spew.ConfigState{ - Indent: " ", - DisablePointerAddresses: true, - DisableCapacities: true, - SortKeys: true, - MaxDepth: 10, -} - -type tHelper interface { - Helper() -} - -// Eventually asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. -// -// assert.Eventually(t, func() bool { return true; }, time.Second, 10*time.Millisecond) -func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - ch := make(chan bool, 1) - - timer := time.NewTimer(waitFor) - defer timer.Stop() - - ticker := time.NewTicker(tick) - defer ticker.Stop() - - for tick := ticker.C; ; { - select { - case <-timer.C: - return Fail(t, "Condition never satisfied", msgAndArgs...) - case <-tick: - tick = nil - go func() { ch <- condition() }() - case v := <-ch: - if v { - return true - } - tick = ticker.C - } - } -} - -// CollectT implements the TestingT interface and collects all errors. -type CollectT struct { - errors []error -} - -// Errorf collects the error. -func (c *CollectT) Errorf(format string, args ...interface{}) { - c.errors = append(c.errors, fmt.Errorf(format, args...)) -} - -// FailNow panics. -func (*CollectT) FailNow() { - panic("Assertion failed") -} - -// Deprecated: That was a method for internal usage that should not have been published. Now just panics. -func (*CollectT) Reset() { - panic("Reset() is deprecated") -} - -// Deprecated: That was a method for internal usage that should not have been published. Now just panics. -func (*CollectT) Copy(TestingT) { - panic("Copy() is deprecated") -} - -// EventuallyWithT asserts that given condition will be met in waitFor time, -// periodically checking target function each tick. In contrast to Eventually, -// it supplies a CollectT to the condition function, so that the condition -// function can use the CollectT to call other assertions. -// The condition is considered "met" if no errors are raised in a tick. -// The supplied CollectT collects all errors from one tick (if there are any). -// If the condition is not met before waitFor, the collected errors of -// the last tick are copied to t. -// -// externalValue := false -// go func() { -// time.Sleep(8*time.Second) -// externalValue = true -// }() -// assert.EventuallyWithT(t, func(c *assert.CollectT) { -// // add assertions as needed; any assertion failure will fail the current tick -// assert.True(c, externalValue, "expected 'externalValue' to be true") -// }, 1*time.Second, 10*time.Second, "external state has not changed to 'true'; still false") -func EventuallyWithT(t TestingT, condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - var lastFinishedTickErrs []error - ch := make(chan []error, 1) - - timer := time.NewTimer(waitFor) - defer timer.Stop() - - ticker := time.NewTicker(tick) - defer ticker.Stop() - - for tick := ticker.C; ; { - select { - case <-timer.C: - for _, err := range lastFinishedTickErrs { - t.Errorf("%v", err) - } - return Fail(t, "Condition never satisfied", msgAndArgs...) - case <-tick: - tick = nil - go func() { - collect := new(CollectT) - defer func() { - ch <- collect.errors - }() - condition(collect) - }() - case errs := <-ch: - if len(errs) == 0 { - return true - } - // Keep the errors from the last ended condition, so that they can be copied to t if timeout is reached. - lastFinishedTickErrs = errs - tick = ticker.C - } - } -} - -// Never asserts that the given condition doesn't satisfy in waitFor time, -// periodically checking the target function each tick. -// -// assert.Never(t, func() bool { return false; }, time.Second, 10*time.Millisecond) -func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - ch := make(chan bool, 1) - - timer := time.NewTimer(waitFor) - defer timer.Stop() - - ticker := time.NewTicker(tick) - defer ticker.Stop() - - for tick := ticker.C; ; { - select { - case <-timer.C: - return true - case <-tick: - tick = nil - go func() { ch <- condition() }() - case v := <-ch: - if v { - return Fail(t, "Condition satisfied", msgAndArgs...) - } - tick = ticker.C - } - } -} - -// ErrorIs asserts that at least one of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func ErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if errors.Is(err, target) { - return true - } - - var expectedText string - if target != nil { - expectedText = target.Error() - } - - chain := buildErrorChainString(err) - - return Fail(t, fmt.Sprintf("Target error should be in err chain:\n"+ - "expected: %q\n"+ - "in chain: %s", expectedText, chain, - ), msgAndArgs...) -} - -// NotErrorIs asserts that at none of the errors in err's chain matches target. -// This is a wrapper for errors.Is. -func NotErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if !errors.Is(err, target) { - return true - } - - var expectedText string - if target != nil { - expectedText = target.Error() - } - - chain := buildErrorChainString(err) - - return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+ - "found: %q\n"+ - "in chain: %s", expectedText, chain, - ), msgAndArgs...) -} - -// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. -// This is a wrapper for errors.As. -func ErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if errors.As(err, target) { - return true - } - - chain := buildErrorChainString(err) - - return Fail(t, fmt.Sprintf("Should be in error chain:\n"+ - "expected: %q\n"+ - "in chain: %s", target, chain, - ), msgAndArgs...) -} - -func buildErrorChainString(err error) string { - if err == nil { - return "" - } - - e := errors.Unwrap(err) - chain := fmt.Sprintf("%q", err.Error()) - for e != nil { - chain += fmt.Sprintf("\n\t%q", e.Error()) - e = errors.Unwrap(e) - } - return chain -} diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/doc.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/doc.go deleted file mode 100644 index 4953981d38..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/doc.go +++ /dev/null @@ -1,46 +0,0 @@ -// Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. -// -// # Example Usage -// -// The following is a complete example using assert in a standard test function: -// -// import ( -// "testing" -// "github.com/stretchr/testify/assert" -// ) -// -// func TestSomething(t *testing.T) { -// -// var a string = "Hello" -// var b string = "Hello" -// -// assert.Equal(t, a, b, "The two words should be the same.") -// -// } -// -// if you assert many times, use the format below: -// -// import ( -// "testing" -// "github.com/stretchr/testify/assert" -// ) -// -// func TestSomething(t *testing.T) { -// assert := assert.New(t) -// -// var a string = "Hello" -// var b string = "Hello" -// -// assert.Equal(a, b, "The two words should be the same.") -// } -// -// # Assertions -// -// Assertions allow you to easily write test code, and are global funcs in the `assert` package. -// All assertion functions take, as the first argument, the `*testing.T` object provided by the -// testing framework. This allows the assertion funcs to write the failings and other details to -// the correct place. -// -// Every assertion function also takes an optional string message as the final argument, -// allowing custom error messages to be appended to the message the assertion method outputs. -package assert diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/errors.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/errors.go deleted file mode 100644 index ac9dc9d1d6..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/errors.go +++ /dev/null @@ -1,10 +0,0 @@ -package assert - -import ( - "errors" -) - -// AnError is an error instance useful for testing. If the code does not care -// about error specifics, and only needs to return the error for example, this -// error should be used to make the test code more readable. -var AnError = errors.New("assert.AnError general error for testing") diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/forward_assertions.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/forward_assertions.go deleted file mode 100644 index df189d2348..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/forward_assertions.go +++ /dev/null @@ -1,16 +0,0 @@ -package assert - -// Assertions provides assertion methods around the -// TestingT interface. -type Assertions struct { - t TestingT -} - -// New makes a new Assertions object for the specified TestingT. -func New(t TestingT) *Assertions { - return &Assertions{ - t: t, - } -} - -//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_forward.go.tmpl -include-format-funcs" diff --git a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/http_assertions.go b/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/http_assertions.go deleted file mode 100644 index 861ed4b7ce..0000000000 --- a/cluster-provision/gocli/vendor/github.com/stretchr/testify/assert/http_assertions.go +++ /dev/null @@ -1,165 +0,0 @@ -package assert - -import ( - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "strings" -) - -// httpCode is a helper that returns HTTP code of the response. It returns -1 and -// an error if building a new request fails. -func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (int, error) { - w := httptest.NewRecorder() - req, err := http.NewRequest(method, url, http.NoBody) - if err != nil { - return -1, err - } - req.URL.RawQuery = values.Encode() - handler(w, req) - return w.Code, nil -} - -// HTTPSuccess asserts that a specified handler returns a success status code. -// -// assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - code, err := httpCode(handler, method, url, values) - if err != nil { - Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) - } - - isSuccessCode := code >= http.StatusOK && code <= http.StatusPartialContent - if !isSuccessCode { - Fail(t, fmt.Sprintf("Expected HTTP success status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) - } - - return isSuccessCode -} - -// HTTPRedirect asserts that a specified handler returns a redirect status code. -// -// assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - code, err := httpCode(handler, method, url, values) - if err != nil { - Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) - } - - isRedirectCode := code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect - if !isRedirectCode { - Fail(t, fmt.Sprintf("Expected HTTP redirect status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) - } - - return isRedirectCode -} - -// HTTPError asserts that a specified handler returns an error status code. -// -// assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - code, err := httpCode(handler, method, url, values) - if err != nil { - Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) - } - - isErrorCode := code >= http.StatusBadRequest - if !isErrorCode { - Fail(t, fmt.Sprintf("Expected HTTP error status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) - } - - return isErrorCode -} - -// HTTPStatusCode asserts that a specified handler returns a specified status code. -// -// assert.HTTPStatusCode(t, myHandler, "GET", "/notImplemented", nil, 501) -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPStatusCode(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - code, err := httpCode(handler, method, url, values) - if err != nil { - Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) - } - - successful := code == statuscode - if !successful { - Fail(t, fmt.Sprintf("Expected HTTP status code %d for %q but received %d", statuscode, url+"?"+values.Encode(), code), msgAndArgs...) - } - - return successful -} - -// HTTPBody is a helper that returns HTTP body of the response. It returns -// empty string if building a new request fails. -func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) string { - w := httptest.NewRecorder() - if len(values) > 0 { - url += "?" + values.Encode() - } - req, err := http.NewRequest(method, url, http.NoBody) - if err != nil { - return "" - } - handler(w, req) - return w.Body.String() -} - -// HTTPBodyContains asserts that a specified handler returns a -// body that contains a string. -// -// assert.HTTPBodyContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - body := HTTPBody(handler, method, url, values) - - contains := strings.Contains(body, fmt.Sprint(str)) - if !contains { - Fail(t, fmt.Sprintf("Expected response body for \"%s\" to contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body), msgAndArgs...) - } - - return contains -} - -// HTTPBodyNotContains asserts that a specified handler returns a -// body that does not contain a string. -// -// assert.HTTPBodyNotContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") -// -// Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - body := HTTPBody(handler, method, url, values) - - contains := strings.Contains(body, fmt.Sprint(str)) - if contains { - Fail(t, fmt.Sprintf("Expected response body for \"%s\" to NOT contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body), msgAndArgs...) - } - - return !contains -} From 9a86050eead475f434b2e27893166e4d3cbf1424 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:28:48 +0300 Subject: [PATCH 02/69] feat/refactor: Introduce the KubevirtProvider type The KubevirtProvider is a struct representing an arbitrary Kubevirtci running cluster. It holds all config flags and options that are in the run and provision commands. A Kubevirt provider can be created in two ways, by creating a cluster using the Start method, or from an already running cluster. For this to be possible then json representation of the struct is persisted on the dnsmasq container and later read to parse the deployed settings Or through the normal constructor which uses the option pattern to avoid a bloated function signature The logic that was previously in run.go has been split to several methods to facilitate readability and testing (runNFSGanesha, runRegistry, prepareQemuCmd, prepareDeviceMappings) and dnsmasq creation logic got moved to its own method instead of existing in its own package Floating methods such as waitForVMToBeUp, nodeNameFromIndex, nodeContainer.. etc were grouped to be methods of the struct Signed-off-by: aerosouund --- .../gocli/providers/base_provider.go | 396 ++++++++++++++++++ cluster-provision/gocli/providers/opts.go | 346 +++++++++++++++ .../gocli/providers/provision.go | 213 ++++++++++ cluster-provision/gocli/providers/start.go | 352 ++++++++++++++++ cluster-provision/gocli/providers/types.go | 344 +++++++++++++++ 5 files changed, 1651 insertions(+) create mode 100644 cluster-provision/gocli/providers/base_provider.go create mode 100644 cluster-provision/gocli/providers/opts.go create mode 100644 cluster-provision/gocli/providers/provision.go create mode 100644 cluster-provision/gocli/providers/start.go create mode 100644 cluster-provision/gocli/providers/types.go diff --git a/cluster-provision/gocli/providers/base_provider.go b/cluster-provision/gocli/providers/base_provider.go new file mode 100644 index 0000000000..185383f67c --- /dev/null +++ b/cluster-provision/gocli/providers/base_provider.go @@ -0,0 +1,396 @@ +package providers + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/alessio/shellescape" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" + "github.com/docker/go-connections/nat" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/resource" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" + "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" +) + +func NewKubevirtProvider(k8sversion string, image string, cli *client.Client, + options []KubevirtProviderOption) *KubevirtProvider { + kp := &KubevirtProvider{ + Image: image, + Version: k8sversion, + Docker: cli, + Nodes: 1, // start with nodes equal one and will be later modified by options that set a different value + } + + for _, option := range options { + option(kp) + } + + return kp +} + +func NewFromRunning(dnsmasqPrefix string) (*KubevirtProvider, error) { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return nil, err + } + + containers, err := docker.GetPrefixedContainers(cli, dnsmasqPrefix+"-dnsmasq") + if err != nil { + return nil, err + } + + if len(containers) == 0 { + return nil, fmt.Errorf("No running provider has the prefix %s", dnsmasqPrefix) + } + + var buf bytes.Buffer + _, err = docker.Exec(cli, containers[0].ID, []string{"cat", "provider.json"}, &buf) + if err != nil { + return nil, err + } + + kp := &KubevirtProvider{} + + err = json.Unmarshal(buf.Bytes(), kp) + if err != nil { + return nil, err + } + + kp.Docker = cli + return kp, nil +} + +func (kp *KubevirtProvider) runDNSMasq(ctx context.Context, portMap nat.PortMap) (string, error) { + dnsmasqMounts := []mount.Mount{} + _, err := os.Stat("/lib/modules") + if err == nil { + dnsmasqMounts = []mount.Mount{ + { + Type: mount.TypeBind, + Source: "/lib/modules", + Target: "/lib/modules", + }, + } + } + + dnsmasq, err := kp.Docker.ContainerCreate(ctx, &container.Config{ + Image: kp.Image, + Env: []string{ + fmt.Sprintf("NUM_NODES=%d", kp.Nodes), + fmt.Sprintf("NUM_SECONDARY_NICS=%d", kp.SecondaryNics), + }, + Cmd: []string{"/bin/bash", "-c", "/dnsmasq.sh"}, + ExposedPorts: nat.PortSet{ + utils.TCPPortOrDie(utils.PortSSH): {}, + utils.TCPPortOrDie(utils.PortRegistry): {}, + utils.TCPPortOrDie(utils.PortOCP): {}, + utils.TCPPortOrDie(utils.PortAPI): {}, + utils.TCPPortOrDie(utils.PortVNC): {}, + utils.TCPPortOrDie(utils.PortHTTP): {}, + utils.TCPPortOrDie(utils.PortHTTPS): {}, + utils.TCPPortOrDie(utils.PortPrometheus): {}, + utils.TCPPortOrDie(utils.PortGrafana): {}, + utils.TCPPortOrDie(utils.PortUploadProxy): {}, + utils.UDPPortOrDie(utils.PortDNS): {}, + }, + }, &container.HostConfig{ + Privileged: true, + PublishAllPorts: kp.RandomPorts, + PortBindings: portMap, + ExtraHosts: []string{ + "nfs:192.168.66.2", + "registry:192.168.66.2", + "ceph:192.168.66.2", + }, + Mounts: dnsmasqMounts, + }, nil, nil, kp.Version+"-dnsmasq") + + if err := kp.Docker.ContainerStart(ctx, dnsmasq.ID, container.StartOptions{}); err != nil { + return "", err + } + return dnsmasq.ID, nil +} + +func (kp *KubevirtProvider) runRegistry(ctx context.Context) (string, error) { + err := docker.ImagePull(kp.Docker, ctx, utils.DockerRegistryImage, types.ImagePullOptions{}) + if err != nil { + return "", err + } + registry, err := kp.Docker.ContainerCreate(ctx, &container.Config{ + Image: utils.DockerRegistryImage, + }, &container.HostConfig{ + Privileged: true, + NetworkMode: container.NetworkMode("container:" + kp.DNSMasq), + }, nil, nil, kp.Version+"-registry") + if err != nil { + return "", err + } + + if err := kp.Docker.ContainerStart(ctx, registry.ID, container.StartOptions{}); err != nil { + return "", err + } + + return registry.ID, nil +} + +func (kp *KubevirtProvider) runNFSGanesha(ctx context.Context) (string, error) { + nfsData, err := filepath.Abs(kp.NFSData) + if err != nil { + return "", err + } + + err = docker.ImagePull(kp.Docker, ctx, utils.NFSGaneshaImage, types.ImagePullOptions{}) + if err != nil { + panic(err) + } + + nfsGanesha, err := kp.Docker.ContainerCreate(ctx, &container.Config{ + Image: utils.NFSGaneshaImage, + }, &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: nfsData, + Target: "/data/nfs", + }, + }, + Privileged: true, + NetworkMode: container.NetworkMode("container:" + kp.DNSMasq), + }, nil, nil, kp.Version+"-nfs-ganesha") + if err != nil { + return "", err + } + + if err := kp.Docker.ContainerStart(ctx, nfsGanesha.ID, container.StartOptions{}); err != nil { + return "", err + } + return nfsGanesha.ID, nil +} + +func (kp *KubevirtProvider) prepareDeviceMappings() ([]container.DeviceMapping, error) { + iommuGroup, err := kp.getPCIDeviceIOMMUGroup(kp.GPU) + if err != nil { + return nil, err + } + vfioDevice := fmt.Sprintf("/dev/vfio/%s", iommuGroup) + return []container.DeviceMapping{ + { + PathOnHost: "/dev/vfio/vfio", + PathInContainer: "/dev/vfio/vfio", + CgroupPermissions: "mrw", + }, + { + PathOnHost: vfioDevice, + PathInContainer: vfioDevice, + CgroupPermissions: "mrw", + }, + }, nil +} + +func (kp *KubevirtProvider) prepareQemuCmd(x int) string { + nodeQemuArgs := kp.QemuArgs + kernelArgs := kp.KernelArgs + macSuffix := fmt.Sprintf("%02x", x) + + for i := 0; i < int(kp.SecondaryNics); i++ { + netSuffix := fmt.Sprintf("%d-%d", x, i) + nodeQemuArgs = fmt.Sprintf("%s -device virtio-net-pci,netdev=secondarynet%s,mac=52:55:00:d1:56:%s -netdev tap,id=secondarynet%s,ifname=stap%s,script=no,downscript=no", nodeQemuArgs, netSuffix, macSuffix, netSuffix, netSuffix) + } + + if kp.GPU != "" && x == int(kp.Nodes)-1 { + nodeQemuArgs = fmt.Sprintf("%s -device vfio-pci,host=%s", nodeQemuArgs, kp.GPU) + } + + var vmArgsNvmeDisks []string + if len(kp.NvmeDisks) > 0 { + for i, size := range kp.NvmeDisks { + resource.MustParse(size) + disk := fmt.Sprintf("%s-%d.img", "/nvme", i) + nodeQemuArgs = fmt.Sprintf("%s -drive file=%s,format=raw,id=NVME%d,if=none -device nvme,drive=NVME%d,serial=nvme-%d", nodeQemuArgs, disk, i, i, i) + vmArgsNvmeDisks = append(vmArgsNvmeDisks, fmt.Sprintf("--nvme-device-size %s", size)) + } + } + var vmArgsSCSIDisks []string + if len(kp.ScsiDisks) > 0 { + nodeQemuArgs = fmt.Sprintf("%s -device virtio-scsi-pci,id=scsi0", nodeQemuArgs) + for i, size := range kp.ScsiDisks { + resource.MustParse(size) + disk := fmt.Sprintf("%s-%d.img", "/scsi", i) + nodeQemuArgs = fmt.Sprintf("%s -drive file=%s,if=none,id=drive%d -device scsi-hd,drive=drive%d,bus=scsi0.0,channel=0,scsi-id=0,lun=%d", nodeQemuArgs, disk, i, i, i) + vmArgsSCSIDisks = append(vmArgsSCSIDisks, fmt.Sprintf("--scsi-device-size %s", size)) + } + } + + var vmArgsUSBDisks []string + const bus = " -device qemu-xhci,id=bus%d" + const drive = " -drive if=none,id=stick%d,format=raw,file=/usb-%d.img" + const dev = " -device usb-storage,bus=bus%d.0,drive=stick%d" + const usbSizefmt = " --usb-device-size %s" + if len(kp.USBDisks) > 0 { + for i, size := range kp.USBDisks { + resource.MustParse(size) + if i%2 == 0 { + nodeQemuArgs += fmt.Sprintf(bus, i/2) + } + nodeQemuArgs += fmt.Sprintf(drive, i, i) + nodeQemuArgs += fmt.Sprintf(dev, i/2, i) + vmArgsUSBDisks = append(vmArgsUSBDisks, fmt.Sprintf(usbSizefmt, size)) + } + } + + additionalArgs := []string{} + if len(nodeQemuArgs) > 0 { + additionalArgs = append(additionalArgs, "--qemu-args", shellescape.Quote(nodeQemuArgs)) + } + + if kp.Hugepages2M > 0 { + kernelArgs += fmt.Sprintf(" hugepagesz=2M hugepages=%d", kp.Hugepages2M) + } + + if kp.Hugepages1G > 0 { + kernelArgs += fmt.Sprintf(" hugepagesz=1G hugepages=%d", kp.Hugepages1G) + } + + if kp.EnableFIPS { + kernelArgs += " fips=1" + } + + blockDev := "" + if kp.EnableCeph { + blockDev = "--block-device /var/run/disk/blockdev.qcow2 --block-device-size 32212254720" + } + + kernelArgs = strings.TrimSpace(kernelArgs) + if kernelArgs != "" { + additionalArgs = append(additionalArgs, "--additional-kernel-args", shellescape.Quote(kernelArgs)) + } + + return fmt.Sprintf("/vm.sh -n /var/run/disk/disk.qcow2 --memory %s --cpu %s --numa %s %s %s %s %s %s", + kp.Memory, + strconv.Itoa(int(kp.CPU)), + strconv.Itoa(int(kp.Numa)), + blockDev, + strings.Join(vmArgsSCSIDisks, " "), + strings.Join(vmArgsNvmeDisks, " "), + strings.Join(vmArgsUSBDisks, " "), + strings.Join(additionalArgs, " "), + ) +} + +func (kp *KubevirtProvider) persistProvider() error { + providerJson, err := json.Marshal(kp) + if err != nil { + return err + } + escapedJson := strconv.Quote(string(providerJson)) + + _, err = docker.Exec(kp.Docker, kp.DNSMasq, []string{"/bin/bash", "-c", fmt.Sprintf("echo %s | tee /provider.json > /dev/null", string(escapedJson))}, os.Stdout) + if err != nil { + return err + } + return nil +} + +func (kp *KubevirtProvider) getDevicePCIID(pciAddress string) (string, error) { + file, err := os.Open(filepath.Join("/sys/bus/pci/devices", pciAddress, "uevent")) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "PCI_ID") { + equal := strings.Index(line, "=") + value := strings.TrimSpace(line[equal+1:]) + return strings.ToLower(value), nil + } + } + return "", fmt.Errorf("no pci_id is found") +} + +func (kp *KubevirtProvider) copyDirectory(ctx context.Context, cli *client.Client, containerID string, sourceDirectory string, targetDirectory string) error { + srcInfo, err := archive.CopyInfoSourcePath(sourceDirectory, false) + if err != nil { + return err + } + + srcArchive, err := archive.TarResource(srcInfo) + if err != nil { + return err + } + defer srcArchive.Close() + + dstInfo := archive.CopyInfo{Path: targetDirectory} + + dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) + if err != nil { + return err + } + defer preparedArchive.Close() + + err = cli.CopyToContainer(ctx, containerID, dstDir, preparedArchive, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false}) + if err != nil { + return err + } + return nil +} + +func (kp *KubevirtProvider) getPCIDeviceIOMMUGroup(address string) (string, error) { + iommuLink := filepath.Join("/sys/bus/pci/devices", address, "iommu_group") + iommuPath, err := os.Readlink(iommuLink) + if err != nil { + return "", fmt.Errorf("failed to read iommu_group link %s for device %s - %v", iommuLink, address, err) + } + _, iommuGroup := filepath.Split(iommuPath) + return iommuGroup, nil +} + +func (kp *KubevirtProvider) handleInterrupt(cancel context.CancelFunc, stop chan error) { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + <-interrupt + cancel() + stop <- fmt.Errorf("Interrupt received, clean up") +} + +func (kp *KubevirtProvider) nodeNameFromIndex(x int) string { + return fmt.Sprintf("node%02d", x) +} + +func (kp *KubevirtProvider) nodeContainer(version string, node string) string { + return version + "-" + node +} + +func (kp *KubevirtProvider) waitForVMToBeUp(prefix string, nodeName string) error { + var err error + for x := 0; x < 10; x++ { + _, err = docker.Exec(kp.Docker, kp.nodeContainer(prefix, nodeName), []string{"/bin/bash", "-c", "ssh.sh echo VM is up"}, os.Stdout) + if err == nil { + break + } + logrus.WithError(err).Warningf("Could not establish a ssh connection to the VM, retrying ...") + time.Sleep(1 * time.Second) + } + + if err != nil { + return fmt.Errorf("could not establish a connection to the node after a generous timeout: %v", err) + } + + return nil +} diff --git a/cluster-provision/gocli/providers/opts.go b/cluster-provision/gocli/providers/opts.go new file mode 100644 index 0000000000..69537c238c --- /dev/null +++ b/cluster-provision/gocli/providers/opts.go @@ -0,0 +1,346 @@ +package providers + +func WithNodes(nodes interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Nodes = nodes.(uint) + } +} + +func WithNuma(numa interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Numa = numa.(uint) + } +} + +func WithMemory(memory interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Memory = memory.(string) + } +} + +func WithCPU(cpu interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.CPU = cpu.(uint) + } +} + +func WithSwap(swap interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Swap = swap.(bool) + } +} + +func WithUnlimitedSwap(us interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.UnlimitedSwap = us.(bool) + } +} + +func WithSwapiness(s interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Swapiness = s.(uint) + } +} + +func WithSwapSize(s interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Swapsize = s.(uint) + } +} + +func WithSecondaryNics(secondaryNics interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.SecondaryNics = secondaryNics.(uint) + } +} + +func WithQemuArgs(qemuArgs interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.QemuArgs = qemuArgs.(string) + } +} + +func WithKernelArgs(kernelArgs interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.KernelArgs = kernelArgs.(string) + } +} + +func WithBackground(background interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Background = background.(bool) + } +} + +func WithRandomPorts(randomPorts interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.RandomPorts = randomPorts.(bool) + } +} + +func WithSlim(slim interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Slim = slim.(bool) + } +} + +func WithVNCPort(vncPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.VNCPort = vncPort.(uint16) + } +} + +func WithHTTPPort(httpPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.HTTPPort = httpPort.(uint16) + } +} + +func WithHTTPSPort(httpsPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.HTTPSPort = httpsPort.(uint16) + } +} + +func WithRegistryPort(registryPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.RegistryPort = registryPort.(uint16) + } +} + +func WithOCPort(ocpPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.OCPort = ocpPort.(uint16) + } +} + +func WithK8sPort(k8sPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.K8sPort = k8sPort.(uint16) + } +} + +func WithSSHPort(sshPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.SSHPort = sshPort.(uint16) + } +} + +func WithPrometheusPort(prometheusPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.PrometheusPort = prometheusPort.(uint16) + } +} + +func WithGrafanaPort(grafanaPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.GrafanaPort = grafanaPort.(uint16) + } +} + +func WithDNSPort(dnsPort interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.DNSPort = dnsPort.(uint16) + } +} + +func WithNFSData(nfsData interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.NFSData = nfsData.(string) + } +} + +func WithEnableCeph(enableCeph interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableCeph = enableCeph.(bool) + } +} + +func WithEnableIstio(enableIstio interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableIstio = enableIstio.(bool) + } +} + +func WithEnableCNAO(enableCNAO interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableCNAO = enableCNAO.(bool) + } +} + +func WithEnableNFSCSI(enableNFSCSI interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableNFSCSI = enableNFSCSI.(bool) + } +} + +func WithEnablePrometheus(enablePrometheus interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnablePrometheus = enablePrometheus.(bool) + } +} + +func WithEnablePrometheusAlertManager(enablePrometheusAlertManager interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnablePrometheusAlertManager = enablePrometheusAlertManager.(bool) + } +} + +func WithEnableGrafana(enableGrafana interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableGrafana = enableGrafana.(bool) + } +} +func WithMultus(multus interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableMultus = multus.(bool) + } +} +func WithAAQ(aaq interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.AAQ = aaq.(bool) + } +} +func WithCDI(cdi interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.CDI = cdi.(bool) + } +} + +func WithKSM(ksm interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.KSM = ksm.(bool) + } +} + +func WithKSMInterval(ki interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.KSMInterval = ki.(uint) + } +} + +func WithKSMPages(kp interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.KSMPages = kp.(uint) + } +} + +func WithDockerProxy(dockerProxy interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.DockerProxy = dockerProxy.(string) + } +} + +func WithGPU(gpu interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.GPU = gpu.(string) + } +} + +func WithCDIVersion(cdi interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.CDIVersion = cdi.(string) + } +} + +func WithAAQVersion(aaq interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.AAQVersion = aaq.(string) + } +} + +func WithNvmeDisks(nvmeDisks interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.NvmeDisks = nvmeDisks.([]string) + } +} + +func WithScsiDisks(scsiDisks interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.ScsiDisks = scsiDisks.([]string) + } +} + +func WithRunEtcdOnMemory(runEtcdOnMemory interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.RunEtcdOnMemory = runEtcdOnMemory.(bool) + } +} + +func WithEtcdCapacity(etcdCapacity interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EtcdCapacity = etcdCapacity.(string) + } +} + +func WithHugepages2M(hugepages2M interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Hugepages2M = hugepages2M.(uint) + } +} + +func WithHugepages1G(hugepages1G interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Hugepages1G = hugepages1G.(uint) + } +} + +func WithEnableRealtimeScheduler(enableRealtimeScheduler interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableRealtimeScheduler = enableRealtimeScheduler.(bool) + } +} + +func WithEnableFIPS(enableFIPS interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableFIPS = enableFIPS.(bool) + } +} + +func WithEnablePSA(enablePSA interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnablePSA = enablePSA.(bool) + } +} + +func WithSingleStack(singleStack interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.SingleStack = singleStack.(bool) + } +} + +func WithEnableAudit(enableAudit interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableAudit = enableAudit.(bool) + } +} + +func WithUSBDisks(usbDisks interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.USBDisks = usbDisks.([]string) + } +} + +func WithAdditionalKernelArgs(ka interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.AdditionalKernelArgs = ka.([]string) + } +} + +func WithPhases(p interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.Phases = p.(string) + } +} + +func WithSkipCnaoCR(skip interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.SkipCnaoCR = skip.(bool) + } +} + +func WithNoEtcdFsync(fsync interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.NoEtcdFsync = fsync.(bool) + } +} diff --git a/cluster-provision/gocli/providers/provision.go b/cluster-provision/gocli/providers/provision.go new file mode 100644 index 0000000000..5667eae69a --- /dev/null +++ b/cluster-provision/gocli/providers/provision.go @@ -0,0 +1,213 @@ +package providers + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/alessio/shellescape" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/volume" + "github.com/docker/go-connections/nat" + "github.com/sirupsen/logrus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" + "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/k8sprovision" + provisionopt "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/provision" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rootkey" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +func (kp *KubevirtProvider) Provision(ctx context.Context, cancel context.CancelFunc, portMap nat.PortMap, k8sVersion string) (retErr error) { + prefix := fmt.Sprintf("k8s-%s-provision", kp.Version) + target := fmt.Sprintf("quay.io/kubevirtci/k8s-%s", kp.Version) + if kp.Phases == "linux" { + target = kp.Image + "-base" + } + kp.Version = prefix + + stop := make(chan error, 10) + containers, volumes, done := docker.NewCleanupHandler(kp.Docker, stop, os.Stdout, true) + + defer func() { + stop <- retErr + <-done + }() + + go kp.handleInterrupt(cancel, stop) + + err := docker.ImagePull(kp.Docker, ctx, kp.Image, types.ImagePullOptions{}) + if err != nil { + return err + } + + dnsmasq, err := kp.runDNSMasq(ctx, portMap) + if err != nil { + return err + } + + kp.DNSMasq = dnsmasq + containers <- dnsmasq + + dm, err := kp.Docker.ContainerInspect(context.Background(), dnsmasq) + if err != nil { + return err + } + + sshPort, err := utils.GetPublicPort(utils.PortSSH, dm.NetworkSettings.Ports) + if err != nil { + return err + } + + nodeName := kp.nodeNameFromIndex(1) + nodeNum := fmt.Sprintf("%02d", 1) + + vol, err := kp.Docker.VolumeCreate(ctx, volume.CreateOptions{ + Name: fmt.Sprintf("%s-%s", prefix, nodeName), + }) + if err != nil { + return err + } + volumes <- vol.Name + registryVol, err := kp.Docker.VolumeCreate(ctx, volume.CreateOptions{ + Name: fmt.Sprintf("%s-%s", prefix, "registry"), + }) + if err != nil { + return err + } + + node, err := kp.Docker.ContainerCreate(ctx, &container.Config{ + Image: kp.Image, + Env: []string{ + fmt.Sprintf("NODE_NUM=%s", nodeNum), + }, + Volumes: map[string]struct{}{ + "/var/run/disk": {}, + "/var/lib/registry": {}, + }, + Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh --memory %s --cpu %s %s", kp.Memory, strconv.Itoa(int(kp.CPU)), kp.QemuArgs)}, + }, &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: "volume", + Source: vol.Name, + Target: "/var/run/disk", + }, + { + Type: "volume", + Source: registryVol.Name, + Target: "/var/lib/registry", + }, + }, + Privileged: true, + NetworkMode: container.NetworkMode("container:" + kp.DNSMasq), + }, nil, nil, kp.nodeContainer(kp.Version, nodeName)) + if err != nil { + return err + } + containers <- node.ID + if err := kp.Docker.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { + return err + } + + // Wait for ssh.sh script to exist + _, err = docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"bin/bash", "-c", "while [ ! -f /ssh_ready ] ; do sleep 1; done", "checking for ssh.sh script"}, os.Stdout) + if err != nil { + return err + } + + sshClient, err := libssh.NewSSHClient(sshPort, 1, false) + if err != nil { + return err + } + + rootkey := rootkey.NewRootKey(sshClient) + if err = rootkey.Exec(); err != nil { + fmt.Println(err) + } + + sshClient, err = libssh.NewSSHClient(sshPort, 1, true) + if err != nil { + return err + } + + if strings.Contains(kp.Phases, "linux") { + provisionOpt := provisionopt.NewLinuxProvisioner(sshClient) + if err = provisionOpt.Exec(); err != nil { + return err + } + } + + if strings.Contains(kp.Phases, "k8s") { + // copy provider scripts + if err = sshClient.Command("mkdir -p /tmp/ceph /tmp/cnao /tmp/nfs-csi /tmp/nodeports /tmp/prometheus /tmp/whereabouts /tmp/kwok"); err != nil { + return err + } + // Copy manifests to the VM + success, err := docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"/bin/bash", "-c", "scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/manifests/* root@192.168.66.101:/tmp"}, os.Stdout) + if err != nil { + return err + } + + if !success { + return fmt.Errorf("error copying manifests to node") + } + + provisionK8sOpt := k8sprovision.NewK8sProvisioner(sshClient, k8sVersion, kp.Slim) + if err = provisionK8sOpt.Exec(); err != nil { + return err + } + } + + _ = sshClient.Command("sudo shutdown now -h") + + _, err = docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"rm", "/ssh_ready"}, io.Discard) + if err != nil { + return err + } + + logrus.Info("waiting for the node to stop") + okChan, errChan := kp.Docker.ContainerWait(ctx, kp.nodeContainer(kp.Version, nodeName), container.WaitConditionNotRunning) + select { + case <-okChan: + case err := <-errChan: + if err != nil { + return fmt.Errorf("waiting for the node to stop failed: %v", err) + } + } + + if len(kp.AdditionalKernelArgs) > 0 { + dir, err := os.MkdirTemp("", "gocli") + if err != nil { + return fmt.Errorf("failed creating a temporary directory: %v", err) + } + defer os.RemoveAll(dir) + if err := os.WriteFile(filepath.Join(dir, "additional.kernel.args"), []byte(shellescape.QuoteCommand(kp.AdditionalKernelArgs)), 0666); err != nil { + return fmt.Errorf("failed creating additional.kernel.args file: %v", err) + } + if err := kp.copyDirectory(ctx, kp.Docker, node.ID, dir, "/"); err != nil { + return fmt.Errorf("failed copying additional kernel arguments into the container: %v", err) + } + } + + logrus.Infof("Commiting the node as %s", target) + _, err = kp.Docker.ContainerCommit(ctx, node.ID, container.CommitOptions{ + Reference: target, + Comment: "PROVISION SUCCEEDED", + Author: "gocli", + Changes: nil, + Pause: false, + Config: nil, + }) + if err != nil { + return fmt.Errorf("commiting the node failed: %v", err) + } + + return nil +} diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go new file mode 100644 index 0000000000..2ee3ceeeeb --- /dev/null +++ b/cluster-provision/gocli/providers/start.go @@ -0,0 +1,352 @@ +package providers + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/sirupsen/logrus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" + "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/aaq" + bindvfio "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/bind-vfio" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/cdi" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/cnao" + dockerproxy "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/docker-proxy" + etcd "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/labelnodes" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" + nodesprovision "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nodes" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/prometheus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/psa" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/realtime" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rookceph" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rootkey" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/swap" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc, portMap nat.PortMap) (retErr error) { + stop := make(chan error, 10) + containers, _, done := docker.NewCleanupHandler(kp.Docker, stop, os.Stdout, false) + + defer func() { + stop <- retErr + <-done + }() + + go kp.handleInterrupt(cancel, stop) + + dnsmasq, err := kp.runDNSMasq(ctx, portMap) + if err != nil { + return err + } + kp.DNSMasq = dnsmasq + containers <- dnsmasq + + dnsmasqJSON, err := kp.Docker.ContainerInspect(context.Background(), kp.DNSMasq) + if err != nil { + return err + } + + if kp.SSHPort == 0 { + port, err := utils.GetPublicPort(utils.PortSSH, dnsmasqJSON.NetworkSettings.Ports) + if err != nil { + return err + } + kp.SSHPort = port + } + + if kp.APIServerPort == 0 { + port, err := utils.GetPublicPort(utils.PortAPI, dnsmasqJSON.NetworkSettings.Ports) + if err != nil { + return err + } + kp.APIServerPort = port + } + + registry, err := kp.runRegistry(ctx) + if err != nil { + return err + } + containers <- registry + + if kp.NFSData != "" { + nfsGanesha, err := kp.runNFSGanesha(ctx) + if err != nil { + return nil + } + containers <- nfsGanesha + } + + wg := sync.WaitGroup{} + wg.Add(int(kp.Nodes)) + macCounter := 0 + + for x := 0; x < int(kp.Nodes); x++ { + nodeName := kp.nodeNameFromIndex(x + 1) + sshClient, err := libssh.NewSSHClient(kp.SSHPort, x+1, false) + if err != nil { + return err + } + + nodeNum := fmt.Sprintf("%02d", x+1) + qemuCMD := kp.prepareQemuCmd(x) + macCounter++ + + vmContainerConfig := &container.Config{ + Image: kp.Image, + Env: []string{ + fmt.Sprintf("NODE_NUM=%s", nodeNum), + }, + Cmd: []string{"/bin/bash", "-c", qemuCMD}, + } + var deviceMappings []container.DeviceMapping + + if kp.GPU != "" && x == int(kp.Nodes)-1 { + dm, err := kp.prepareDeviceMappings() + if err != nil { + return err + } + deviceMappings = dm + qemuCMD = fmt.Sprintf("%s -device vfio-pci,host=%s", qemuCMD, kp.GPU) + } + + if kp.EnableCeph { + vmContainerConfig.Volumes = map[string]struct{}{ + "/var/lib/rook": {}, + } + } + + node, err := kp.Docker.ContainerCreate(ctx, vmContainerConfig, &container.HostConfig{ + Privileged: true, + NetworkMode: container.NetworkMode("container:" + kp.DNSMasq), + Resources: container.Resources{ + Devices: deviceMappings, + }, + }, nil, nil, kp.Version+"-"+nodeName) + if err != nil { + return err + } + containers <- node.ID + + if err := kp.Docker.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { + return err + } + + success, err := docker.Exec(kp.Docker, kp.nodeContainer(kp.Version, nodeName), []string{"/bin/bash", "-c", "while [ ! -f /ssh_ready ] ; do sleep 1; done"}, os.Stdout) + if err != nil { + return err + } + + if !success { + return fmt.Errorf("checking for ssh.sh script for node %s failed", nodeName) + } + + err = kp.waitForVMToBeUp(kp.Version, nodeName) + if err != nil { + return err + } + + rootkey := rootkey.NewRootKey(sshClient) + if err = rootkey.Exec(); err != nil { + return err + } + sshClient, err = libssh.NewSSHClient(kp.SSHPort, x+1, true) + + if err = kp.provisionNode(sshClient, x+1); err != nil { + return err + } + + go func(id string) { + kp.Docker.ContainerWait(ctx, id, container.WaitConditionNotRunning) + wg.Done() + }(node.ID) + } + + sshClient, err := libssh.NewSSHClient(kp.SSHPort, 1, true) + if err != nil { + return err + } + + kubeConf, err := os.Create(".kubeconfig") + if err != nil { + return err + } + + err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", kubeConf) + if err != nil { + return err + } + + config, err := k8s.NewConfig(".kubeconfig", kp.APIServerPort) + if err != nil { + return err + } + + k8sClient, err := k8s.NewDynamicClient(config) + if err != nil { + return err + } + kp.Client = k8sClient + + if err = kp.provisionK8sOpts(sshClient); err != nil { + return err + } + + err = kp.persistProvider() + if err != nil { + return err + } + + return nil +} + +func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) error { + opts := []opts.Opt{} + nodeName := kp.nodeNameFromIndex(nodeIdx) + + if kp.EnableFIPS { + for _, cmd := range []string{"sudo fips-mode-setup --enable", "sudo reboot"} { + if err := sshClient.Command(cmd); err != nil { + return fmt.Errorf("Starting fips mode failed: %s", err) + } + } + err := kp.waitForVMToBeUp(kp.Version, nodeName) + if err != nil { + return err + } + } + + if kp.DockerProxy != "" { + //if dockerProxy has value, generate a shell script`/script/docker-proxy.sh` which can be applied to set proxy settings + dp := dockerproxy.NewDockerProxyOpt(sshClient, kp.DockerProxy) + opts = append(opts, dp) + } + + if kp.RunEtcdOnMemory { + logrus.Infof("Creating in-memory mount for etcd data on node %s", nodeName) + etcdinmem := etcd.NewEtcdInMemOpt(sshClient, kp.EtcdCapacity) + opts = append(opts, etcdinmem) + } + + if kp.EnableRealtimeScheduler { + realtimeOpt := realtime.NewRealtimeOpt(sshClient) + opts = append(opts, realtimeOpt) + } + + for _, s := range []string{"8086:2668", "8086:2415"} { + // move the VM sound cards to a vfio-pci driver to prepare for assignment + bvfio := bindvfio.NewBindVfioOpt(sshClient, s) + opts = append(opts, bvfio) + } + + if kp.EnableAudit { + if err := sshClient.Command("touch /home/vagrant/enable_audit"); err != nil { + return fmt.Errorf("provisioning node %d failed (setting enableAudit phase): %s", nodeIdx, err) + } + } + + if kp.EnablePSA { + psaOpt := psa.NewPsaOpt(sshClient) + opts = append(opts, psaOpt) + } + + if nodeIdx == 1 { + n := node01.NewNode01Provisioner(sshClient, kp.SingleStack, kp.NoEtcdFsync) + opts = append(opts, n) + + } else { + if kp.GPU != "" { + // move the assigned PCI device to a vfio-pci driver to prepare for assignment + gpuDeviceID, err := kp.getDevicePCIID(kp.GPU) + if err != nil { + return err + } + bindVfioOpt := bindvfio.NewBindVfioOpt(sshClient, gpuDeviceID) + opts = append(opts, bindVfioOpt) + } + n := nodesprovision.NewNodesProvisioner(sshClient, kp.SingleStack) + opts = append(opts, n) + } + + if kp.KSM { + ksmOpt := ksm.NewKsmOpt(sshClient, int(kp.KSMInterval), int(kp.KSMPages)) + opts = append(opts, ksmOpt) + } + + if kp.Swap { + swapOpt := swap.NewSwapOpt(sshClient, int(kp.Swapiness), kp.UnlimitedSwap, int(kp.Swapsize)) + opts = append(opts, swapOpt) + } + + for _, o := range opts { + if err := o.Exec(); err != nil { + return err + } + } + + return nil +} + +func (kp *KubevirtProvider) provisionK8sOpts(sshClient libssh.Client) error { + opts := []opts.Opt{} + labelSelector := "node-role.kubernetes.io/control-plane" + if kp.Nodes > 1 { + labelSelector = "!node-role.kubernetes.io/control-plane" + } + opts = append(opts, labelnodes.NewNodeLabler(sshClient, labelSelector)) + + if kp.CDI { + opts = append(opts, cdi.NewCdiOpt(kp.Client, sshClient, kp.CDIVersion)) + } + + if kp.AAQ { + if kp.Version == "k8s-1.30" { + opts = append(opts, aaq.NewAaqOpt(kp.Client, sshClient, kp.AAQVersion)) + } else { + logrus.Info("AAQ was requested but kubernetes version is less than 1.30, skipping") + } + } + + if kp.EnablePrometheus { + opts = append(opts, prometheus.NewPrometheusOpt(kp.Client, kp.EnableGrafana, kp.EnablePrometheusAlertManager)) + } + + if kp.EnableCeph { + opts = append(opts, rookceph.NewCephOpt(kp.Client, sshClient)) + } + + if kp.EnableNFSCSI { + opts = append(opts, nfscsi.NewNfsCsiOpt(kp.Client)) + } + + if kp.EnableMultus { + opts = append(opts, multus.NewMultusOpt(kp.Client, sshClient)) + } + + if kp.EnableCNAO { + opts = append(opts, cnao.NewCnaoOpt(kp.Client, sshClient, kp.EnableMultus, kp.SkipCnaoCR)) + } + + if kp.EnableIstio { + opts = append(opts, istio.NewIstioOpt(sshClient, kp.Client, kp.EnableCNAO)) + } + + for _, opt := range opts { + if err := opt.Exec(); err != nil { + return err + } + } + + return nil +} diff --git a/cluster-provision/gocli/providers/types.go b/cluster-provision/gocli/providers/types.go new file mode 100644 index 0000000000..fbb419e87d --- /dev/null +++ b/cluster-provision/gocli/providers/types.go @@ -0,0 +1,344 @@ +package providers + +import ( + "github.com/docker/docker/client" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +type KubevirtProvider struct { + Client k8s.K8sDynamicClient `json:"-"` + Docker *client.Client `json:"-"` + DNSMasq string `json:"dnsmasq"` + + Version string `json:"version"` + Image string `json:"image"` + Nodes uint `flag:"nodes" short:"n" json:"nodes"` + Numa uint `flag:"numa" short:"u" json:"numa"` + Memory string `flag:"memory" short:"m" json:"memory"` + CPU uint `flag:"cpu" short:"c" json:"cpu"` + + SecondaryNics uint `flag:"secondary-nics" json:"secondary_nics"` + QemuArgs string `flag:"qemu-args" json:"qemu_args"` + KernelArgs string `flag:"kernel-args" json:"kernel_args"` + Background bool `flag:"background" short:"b" json:"background"` + RandomPorts bool `flag:"random-ports" json:"random_ports"` + Slim bool `flag:"slim" json:"slim"` + VNCPort uint16 `flag:"vnc-port" json:"vnc_port"` + HTTPPort uint16 `flag:"http-port" json:"http_port"` + HTTPSPort uint16 `flag:"https-port" json:"https_port"` + RegistryPort uint16 `flag:"registry-port" json:"registry_port"` + OCPort uint16 `flag:"ocp-port" json:"ocp_port"` + K8sPort uint16 `flag:"k8s-port" json:"k8s_port"` + SSHPort uint16 `flag:"ssh-port" json:"ssh_port"` + PrometheusPort uint16 `flag:"prometheus-port" json:"prometheus_port"` + GrafanaPort uint16 `flag:"grafana-port" json:"grafana_port"` + DNSPort uint16 `flag:"dns-port" json:"dns_port"` + APIServerPort uint16 `json:"api_server_port"` + NFSData string `flag:"nfs-data" json:"nfs_data"` + EnableCeph bool `flag:"enable-ceph" json:"enable_ceph"` + EnableIstio bool `flag:"enable-istio" json:"enable_istio"` + EnableCNAO bool `flag:"enable-cnao" json:"enable_cnao"` + SkipCnaoCR bool `flag:"skip-cnao-cr" json:"skip_cnao_cr"` + EnableNFSCSI bool `flag:"enable-nfs-csi" json:"enable_nfs_csi"` + EnablePrometheus bool `flag:"enable-prometheus" json:"enable_prometheus"` + EnablePrometheusAlertManager bool `flag:"enable-prometheus-alertmanager" json:"enable_prometheus_alertmanager"` + EnableGrafana bool `flag:"enable-grafana" json:"enable_grafana"` + EnableMultus bool `flag:"deploy-multus" json:"deploy_multus"` + DockerProxy string `flag:"docker-proxy" json:"docker_proxy"` + AAQ bool `flag:"deploy-aaq" json:"deploy_aaq"` + AAQVersion string `flag:"aaq-version" json:"aaq_version"` + CDI bool `flag:"deploy-cdi" json:"deploy_cdi"` + CDIVersion string `flag:"cdi-version" json:"cdi_version"` + GPU string `flag:"gpu" json:"gpu"` + KSM bool `flag:"enable-ksm" json:"enable_ksm"` + KSMPages uint `flag:"ksm-page-count" json:"ksm_page_count"` + KSMInterval uint `flag:"ksm-scan-interval" json:"ksm_scan_interval"` + Swap bool `flag:"enable-swap" json:"enable_swap"` + Swapsize uint `flag:"swap-size" json:"swap_size"` + UnlimitedSwap bool `flag:"unlimited-swap" json:"unlimited_swap"` + Swapiness uint `flag:"swapiness" json:"swapiness"` + NvmeDisks []string `flag:"nvme" json:"nvme"` + ScsiDisks []string `flag:"scsi" json:"scsi"` + USBDisks []string `flag:"usb" json:"usb"` + AdditionalKernelArgs []string `flag:"additional-persistent-kernel-arguments" json:"additional-persistent-kernel-arguments"` + Phases string `flag:"phases" json:"phases"` + RunEtcdOnMemory bool `flag:"run-etcd-on-memory" json:"run_etcd_on_memory"` + EtcdCapacity string `flag:"etcd-capacity" json:"etcd_capacity"` + NoEtcdFsync bool `flag:"no-etcd-fsync" json:"no_etcd_fsync"` + Hugepages2M uint `flag:"hugepages-2m" json:"hugepages_2m"` + Hugepages1G uint `flag:"hugepages-1g" json:"hugepages_1g"` + EnableRealtimeScheduler bool `flag:"enable-realtime-scheduler" json:"enable_realtime_scheduler"` + EnableFIPS bool `flag:"enable-fips" json:"enable_fips"` + EnablePSA bool `flag:"enable-psa" json:"enable_psa"` + SingleStack bool `flag:"single-stack" json:"single_stack"` + EnableAudit bool `flag:"enable-audit" json:"enable_audit"` +} + +type KubevirtProviderOption func(c *KubevirtProvider) + +type FlagConfig struct { + FlagType string + ProviderOptFunc func(interface{}) KubevirtProviderOption +} + +var ProvisionFlagMap = map[string]FlagConfig{ + "memory": { + FlagType: "string", + ProviderOptFunc: WithMemory, + }, + "cpu": { + FlagType: "uint", + ProviderOptFunc: WithCPU, + }, + "slim": { + FlagType: "bool", + ProviderOptFunc: WithSlim, + }, + "random-ports": { + FlagType: "bool", + ProviderOptFunc: WithRandomPorts, + }, + "phases": { + FlagType: "string", + ProviderOptFunc: WithPhases, + }, + "additional-persistent-kernel-arguments": { + FlagType: "[]string", + ProviderOptFunc: WithAdditionalKernelArgs, + }, + "vnc-port": { + FlagType: "uint", + ProviderOptFunc: WithVNCPort, + }, + "ssh-port": { + FlagType: "uint", + ProviderOptFunc: WithSSHPort, + }, + "qemu-args": { + FlagType: "string", + ProviderOptFunc: WithQemuArgs, + }, +} + +var RunFlagMap = map[string]FlagConfig{ + "nodes": { + FlagType: "uint", + ProviderOptFunc: WithNodes, + }, + "numa": { + FlagType: "uint", + ProviderOptFunc: WithNuma, + }, + "memory": { + FlagType: "string", + ProviderOptFunc: WithMemory, + }, + "cpu": { + FlagType: "uint", + ProviderOptFunc: WithCPU, + }, + "secondary-nics": { + FlagType: "uint", + ProviderOptFunc: WithSecondaryNics, + }, + "qemu-args": { + FlagType: "string", + ProviderOptFunc: WithQemuArgs, + }, + "kernel-args": { + FlagType: "string", + ProviderOptFunc: WithKernelArgs, + }, + "background": { + FlagType: "bool", + ProviderOptFunc: WithBackground, + }, + "random-ports": { + FlagType: "bool", + ProviderOptFunc: WithRandomPorts, + }, + "slim": { + FlagType: "bool", + ProviderOptFunc: WithSlim, + }, + "vnc-port": { + FlagType: "uint", + ProviderOptFunc: WithVNCPort, + }, + "http-port": { + FlagType: "uint", + ProviderOptFunc: WithHTTPPort, + }, + "https-port": { + FlagType: "uint", + ProviderOptFunc: WithHTTPSPort, + }, + "registry-port": { + FlagType: "uint", + ProviderOptFunc: WithRegistryPort, + }, + "ocp-port": { + FlagType: "uint", + ProviderOptFunc: WithOCPort, + }, + "k8s-port": { + FlagType: "uint", + ProviderOptFunc: WithK8sPort, + }, + "ssh-port": { + FlagType: "uint", + ProviderOptFunc: WithSSHPort, + }, + "prometheus-port": { + FlagType: "uint", + ProviderOptFunc: WithPrometheusPort, + }, + "grafana-port": { + FlagType: "uint", + ProviderOptFunc: WithGrafanaPort, + }, + "dns-port": { + FlagType: "uint", + ProviderOptFunc: WithDNSPort, + }, + "nfs-data": { + FlagType: "string", + ProviderOptFunc: WithNFSData, + }, + "enable-ceph": { + FlagType: "bool", + ProviderOptFunc: WithEnableCeph, + }, + "enable-istio": { + FlagType: "bool", + ProviderOptFunc: WithEnableIstio, + }, + "enable-cnao": { + FlagType: "bool", + ProviderOptFunc: WithEnableCNAO, + }, + "enable-nfs-csi": { + FlagType: "bool", + ProviderOptFunc: WithEnableNFSCSI, + }, + "enable-prometheus": { + FlagType: "bool", + ProviderOptFunc: WithEnablePrometheus, + }, + "enable-prometheus-alertmanager": { + FlagType: "bool", + ProviderOptFunc: WithEnablePrometheusAlertManager, + }, + "enable-grafana": { + FlagType: "bool", + ProviderOptFunc: WithEnableGrafana, + }, + "docker-proxy": { + FlagType: "string", + ProviderOptFunc: WithDockerProxy, + }, + "gpu": { + FlagType: "string", + ProviderOptFunc: WithGPU, + }, + "nvme": { + FlagType: "[]string", + ProviderOptFunc: WithNvmeDisks, + }, + "scsi": { + FlagType: "[]string", + ProviderOptFunc: WithScsiDisks, + }, + "run-etcd-on-memory": { + FlagType: "bool", + ProviderOptFunc: WithRunEtcdOnMemory, + }, + "etcd-capacity": { + FlagType: "string", + ProviderOptFunc: WithEtcdCapacity, + }, + "hugepages-2m": { + FlagType: "uint", + ProviderOptFunc: WithHugepages2M, + }, + "enable-realtime-scheduler": { + FlagType: "bool", + ProviderOptFunc: WithEnableRealtimeScheduler, + }, + "enable-fips": { + FlagType: "bool", + ProviderOptFunc: WithEnableFIPS, + }, + "enable-psa": { + FlagType: "bool", + ProviderOptFunc: WithEnablePSA, + }, + "single-stack": { + FlagType: "bool", + ProviderOptFunc: WithSingleStack, + }, + "enable-audit": { + FlagType: "bool", + ProviderOptFunc: WithEnableAudit, + }, + "usb": { + FlagType: "[]string", + ProviderOptFunc: WithUSBDisks, + }, + "deploy-multus": { + FlagType: "bool", + ProviderOptFunc: WithMultus, + }, + "deploy-aaq": { + FlagType: "bool", + ProviderOptFunc: WithAAQ, + }, + "deploy-cdi": { + FlagType: "bool", + ProviderOptFunc: WithCDI, + }, + "enable-ksm": { + FlagType: "bool", + ProviderOptFunc: WithKSM, + }, + "ksm-page-count": { + FlagType: "uint", + ProviderOptFunc: WithKSMPages, + }, + "ksm-scan-interval": { + FlagType: "uint", + ProviderOptFunc: WithKSMInterval, + }, + "enable-swap": { + FlagType: "bool", + ProviderOptFunc: WithSwap, + }, + "unlimited-swap": { + FlagType: "bool", + ProviderOptFunc: WithUnlimitedSwap, + }, + "swap-size": { + FlagType: "uint", + ProviderOptFunc: WithSwapSize, + }, + "swapiness": { + FlagType: "uint", + ProviderOptFunc: WithSwapiness, + }, + "cdi-version": { + FlagType: "string", + ProviderOptFunc: WithCDIVersion, + }, + "aaq-version": { + FlagType: "string", + ProviderOptFunc: WithAAQVersion, + }, + "skip-cnao-cr": { + FlagType: "bool", + ProviderOptFunc: WithSkipCnaoCR, + }, + "no-etcd-fsync": { + FlagType: "bool", + ProviderOptFunc: WithNoEtcdFsync, + }, +} From 977d81e5eb649637e7aadf3079683a81c2e9b511 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:31:07 +0300 Subject: [PATCH 03/69] refactor: Rewrite run.go to use the KubevirtProvider To avoid having to read each flag and return an error if its unset leverage the FlagMap, a map of flag name to FlagConfig. a FlagConfig is the type of this flag (string, int, uint16, bool or array of string) and the option function that sets the value of this flag on the KubevirtProvider struct. During parsing of flags this map is being iterated on and each option gets appended to an array to later be used in the KubevirtProvider constructor. The run method's role is now to parse the flags and pass them to the provider and just call Start. All the floating methods in run.go are removed after being moved to the provider. Signed-off-by: aerosouund --- cluster-provision/gocli/cmd/run.go | 957 ++--------------------------- 1 file changed, 58 insertions(+), 899 deletions(-) diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index 7b512a7cc1..1392c03772 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -1,94 +1,22 @@ package cmd import ( - "bufio" - "bytes" + "context" + _ "embed" "fmt" - "os" - "os/signal" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "text/template" - "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/net/context" - "k8s.io/apimachinery/pkg/api/resource" - "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/nodesconfig" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" - containers2 "kubevirt.io/kubevirtci/cluster-provision/gocli/containers" "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" - "kubevirt.io/kubevirtci/cluster-provision/gocli/images" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/aaq" - bindvfio "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/bind-vfio" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/cdi" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/cnao" - dockerproxy "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/docker-proxy" - etcdinmemory "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" - nodesprovision "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nodes" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/prometheus" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/psa" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/realtime" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rookceph" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/rootkey" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/swap" - k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" - "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" - - "github.com/alessio/shellescape" + "kubevirt.io/kubevirtci/cluster-provision/gocli/providers" ) -const ( - proxySettings = ` -curl {{.Proxy}}/ca.crt > /etc/pki/ca-trust/source/anchors/docker_registry_proxy.crt -update-ca-trust - -mkdir -p /etc/systemd/system/crio.service.d -cat </etc/systemd/system/crio.service.d/override.conf -[Service] -Environment="HTTP_PROXY={{.Proxy}}" -Environment="HTTPS_PROXY={{.Proxy}}" -Environment="NO_PROXY=localhost,127.0.0.1,registry,10.96.0.0/12,10.244.0.0/16,192.168.0.0/16,fd00:10:96::/112,fd00:10:244::/112,fd00::/64" -EOT - -systemctl daemon-reload -systemctl restart crio.service -EOF -` - etcdDataDir = "/var/lib/etcd" - nvmeDiskImagePrefix = "/nvme" - scsiDiskImagePrefix = "/scsi" -) - -var soundcardPCIIDs = []string{"8086:2668", "8086:2415"} -var cli *client.Client -var nvmeDisks []string -var scsiDisks []string -var usbDisks []string -var sshClient libssh.Client - -type dockerSetting struct { - Proxy string -} +var nvmeDisks, scsiDisks, usbDisks []string -// NewRunCommand returns command that runs given cluster func NewRunCommand() *cobra.Command { - run := &cobra.Command{ Use: "run", Short: "run starts a given cluster", @@ -118,7 +46,6 @@ func NewRunCommand() *cobra.Command { run.Flags().String("nfs-data", "", "path to data which should be exposed via nfs to the nodes") run.Flags().Bool("enable-ceph", false, "enables dynamic storage provisioning using Ceph") run.Flags().Bool("enable-istio", false, "deploys Istio service mesh") - run.Flags().Bool("reverse", false, "reverse node setup order") run.Flags().Bool("enable-cnao", false, "enable network extensions with istio") run.Flags().Bool("skip-cnao-cr", false, "skip deploying cnao custom resource. if true, only cnao CRDS will be deployed") run.Flags().Bool("deploy-multus", false, "deploy multus") @@ -159,40 +86,45 @@ func NewRunCommand() *cobra.Command { } func run(cmd *cobra.Command, args []string) (retErr error) { + opts := []providers.KubevirtProviderOption{} + flags := cmd.Flags() + for flagName, flagConfig := range providers.RunFlagMap { + switch flagConfig.FlagType { + case "string": + flagVal, err := flags.GetString(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "bool": + flagVal, err := flags.GetBool(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) - prefix, err := cmd.Flags().GetString("prefix") - if err != nil { - return err - } - - nodes, err := cmd.Flags().GetUint("nodes") - if err != nil { - return err - } - - memory, err := cmd.Flags().GetString("memory") - if err != nil { - return err - } - resource.MustParse(memory) - - reverse, err := cmd.Flags().GetBool("reverse") - if err != nil { - return err - } - - randomPorts, err := cmd.Flags().GetBool("random-ports") - if err != nil { - return err - } - - slim, err := cmd.Flags().GetBool("slim") - if err != nil { - return err + case "uint": + flagVal, err := flags.GetUint(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "uint16": + flagVal, err := flags.GetUint16(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "[]string": + flagVal, err := flags.GetStringArray(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + } } portMap := nat.PortMap{} - utils.AppendTCPIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port") utils.AppendTCPIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port") utils.AppendTCPIfExplicit(portMap, utils.PortHTTP, cmd.Flags(), "http-port") @@ -204,85 +136,22 @@ func run(cmd *cobra.Command, args []string) (retErr error) { utils.AppendTCPIfExplicit(portMap, utils.PortGrafana, cmd.Flags(), "grafana-port") utils.AppendUDPIfExplicit(portMap, utils.PortDNS, cmd.Flags(), "dns-port") - qemuArgs, err := cmd.Flags().GetString("qemu-args") - if err != nil { - return err - } - kernelArgs, err := cmd.Flags().GetString("kernel-args") - if err != nil { - return err - } - - cpu, err := cmd.Flags().GetUint("cpu") - if err != nil { - return err - } - - numa, err := cmd.Flags().GetUint("numa") - if err != nil { - return err - } - - secondaryNics, err := cmd.Flags().GetUint("secondary-nics") - if err != nil { - return err - } - - nfsData, err := cmd.Flags().GetString("nfs-data") - if err != nil { - return err - } - - dockerProxy, err := cmd.Flags().GetString("docker-proxy") - if err != nil { - return err - } - - cephEnabled, err := cmd.Flags().GetBool("enable-ceph") - if err != nil { - return err - } - - nfsCsiEnabled, err := cmd.Flags().GetBool("enable-nfs-csi") - if err != nil { - return err - } - - istioEnabled, err := cmd.Flags().GetBool("enable-istio") - if err != nil { - return err - } - - prometheusEnabled, err := cmd.Flags().GetBool("enable-prometheus") - if err != nil { - return err - } - - prometheusAlertmanagerEnabled, err := cmd.Flags().GetBool("enable-prometheus-alertmanager") - if err != nil { - return err - } - - grafanaEnabled, err := cmd.Flags().GetBool("enable-grafana") - if err != nil { - return err + k8sVersion := args[0] + allowedK8sVersions := []string{"k8s-1.28", "k8s-1.29", "k8s-1.30", "k8s-1.31"} + var validVersion bool + for _, v := range allowedK8sVersions { + if k8sVersion == v { + validVersion = true + } } - - cluster := args[0] - - background, err := cmd.Flags().GetBool("background") - if err != nil { - return err + if !validVersion { + return fmt.Errorf("Invalid k8s version passed, please use one of k8s-1.28, k8s-1.29, k8s-1.30 or k8s-1.31") } containerRegistry, err := cmd.Flags().GetString("container-registry") if err != nil { return err } - gpuAddress, err := cmd.Flags().GetString("gpu") - if err != nil { - return err - } containerOrg, err := cmd.Flags().GetString("container-org") if err != nil { @@ -294,744 +163,34 @@ func run(cmd *cobra.Command, args []string) (retErr error) { return err } - runEtcdOnMemory, err := cmd.Flags().GetBool("run-etcd-on-memory") - if err != nil { - return err - } - - etcdDataMountSize, err := cmd.Flags().GetString("etcd-capacity") - if err != nil { - return err - } - resource.MustParse(etcdDataMountSize) - - hugepages2Mcount, err := cmd.Flags().GetUint("hugepages-2m") - if err != nil { - return err - } - hugepages1Gcount, err := cmd.Flags().GetUint("hugepages-1g") - if err != nil { - return err - } - - realtimeSchedulingEnabled, err := cmd.Flags().GetBool("enable-realtime-scheduler") - if err != nil { - return err - } - psaEnabled, err := cmd.Flags().GetBool("enable-psa") - if err != nil { - return err - } - singleStack, err := cmd.Flags().GetBool("single-stack") - if err != nil { - return err - } - noEtcdFsync, err := cmd.Flags().GetBool("no-etcd-fsync") - if err != nil { - return err - } - enableAudit, err := cmd.Flags().GetBool("enable-audit") - if err != nil { - return err - } - fipsEnabled, err := cmd.Flags().GetBool("enable-fips") - if err != nil { - return err - } - cnaoEnabled, err := cmd.Flags().GetBool("enable-cnao") - if err != nil { - return err - } - cnaoSkipCR, err := cmd.Flags().GetBool("skip-cnao-cr") - if err != nil { - return err - } - - deployCdi, err := cmd.Flags().GetBool("deploy-cdi") - if err != nil { - return err - } - - cdiVersion, err := cmd.Flags().GetString("cdi-version") - if err != nil { - return err - } - - deployAaq, err := cmd.Flags().GetBool("deploy-aaq") - if err != nil { - return err - } - - aaqVersion, err := cmd.Flags().GetString("aaq-version") - if err != nil { - return err - } - - deployMultus, err := cmd.Flags().GetBool("deploy-multus") - if err != nil { - return err - } - - enableSwap, err := cmd.Flags().GetBool("enable-swap") - if err != nil { - return err - } - - unlimitedSwap, err := cmd.Flags().GetBool("unlimited-swap") - if err != nil { - return err - } - - swapiness, err := cmd.Flags().GetUint("swapiness") - if err != nil { - return err - } - - swapSize, err := cmd.Flags().GetUint("swap-size") - if err != nil { - return err - } - - enableKsm, err := cmd.Flags().GetBool("enable-ksm") - if err != nil { - return err - } - - ksmPageCount, err := cmd.Flags().GetUint("ksm-page-count") + slim, err := cmd.Flags().GetBool("slim") if err != nil { return err } - ksmScanInterval, err := cmd.Flags().GetUint("ksm-scan-interval") + cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return err } - cli, err = client.NewClientWithOpts(client.FromEnv) - if err != nil { - return err - } - - b := context.Background() - ctx, cancel := context.WithCancel(b) - - stop := make(chan error, 10) - containers, _, done := docker.NewCleanupHandler(cli, stop, cmd.OutOrStderr(), false) - - defer func() { - stop <- retErr - <-done - }() - - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - cancel() - stop <- fmt.Errorf("Interrupt received, clean up") - }() - - clusterImage := cluster - - // Check if cluster container suffix has not being override - // in that case use the default prefix stored at the binary - if containerSuffix == "" { - containerSuffix = images.SUFFIX - } - if containerSuffix != "" { - clusterImage = fmt.Sprintf("%s/%s%s", containerOrg, cluster, containerSuffix) - } else { - clusterImage = path.Join(containerOrg, cluster) - } + clusterImage := fmt.Sprintf("%s/%s/%s%s", containerRegistry, containerOrg, k8sVersion, containerSuffix) if slim { clusterImage += "-slim" } - if len(containerRegistry) > 0 { - clusterImage = path.Join(containerRegistry, clusterImage) - fmt.Printf("Download the image %s\n", clusterImage) - err = docker.ImagePull(cli, ctx, clusterImage, types.ImagePullOptions{}) - if err != nil { - panic(fmt.Sprintf("Failed to download cluster image %s, %s", clusterImage, err)) - } - } - - var dnsmasq *container.CreateResponse - for i := 0; i <= 3; i++ { - if i == 3 { - fmt.Printf("dnsmasq container failed to start 3 times") - return err - } - dnsmasq, err = containers2.DNSMasq(cli, ctx, &containers2.DNSMasqOptions{ - ClusterImage: clusterImage, - SecondaryNicsCount: secondaryNics, - RandomPorts: randomPorts, - PortMap: portMap, - Prefix: prefix, - NodeCount: nodes, - }) - if err != nil { - return err - } - - if err := cli.ContainerStart(ctx, dnsmasq.ID, container.StartOptions{}); err != nil { - fmt.Printf("Failed to start dnsmasq container: %s\n", err) - fmt.Printf("Retry creating and starting dnsmasq container\n") - if err := cli.ContainerRemove(ctx, dnsmasq.ID, container.RemoveOptions{}); err != nil { - return err - } - time.Sleep(2 * time.Second) - - } else { - containers <- dnsmasq.ID - break - } - } - - dm, err := cli.ContainerInspect(context.Background(), dnsmasq.ID) - if err != nil { - return err - } - - sshPort, err := utils.GetPublicPort(utils.PortSSH, dm.NetworkSettings.Ports) - apiServerPort, err := utils.GetPublicPort(utils.PortAPI, dm.NetworkSettings.Ports) - - // Pull the registry image - err = docker.ImagePull(cli, ctx, utils.DockerRegistryImage, types.ImagePullOptions{}) - if err != nil { - panic(err) - } - - // Start registry - registry, err := cli.ContainerCreate(ctx, &container.Config{ - Image: utils.DockerRegistryImage, - }, &container.HostConfig{ - Privileged: true, // fixme we just need proper selinux volume labeling - NetworkMode: container.NetworkMode("container:" + dnsmasq.ID), - }, nil, nil, prefix+"-registry") - if err != nil { - return err - } - containers <- registry.ID - if err := cli.ContainerStart(ctx, registry.ID, container.StartOptions{}); err != nil { - return err - } - - if nfsData != "" { - nfsData, err := filepath.Abs(nfsData) - if err != nil { - return err - } - // Pull the ganesha image - err = docker.ImagePull(cli, ctx, utils.NFSGaneshaImage, types.ImagePullOptions{}) - if err != nil { - panic(err) - } - - // Start the ganesha image - nfsServer, err := cli.ContainerCreate(ctx, &container.Config{ - Image: utils.NFSGaneshaImage, - }, &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: nfsData, - Target: "/data/nfs", - }, - }, - Privileged: true, - NetworkMode: container.NetworkMode("container:" + dnsmasq.ID), - }, nil, nil, prefix+"-nfs-ganesha") - if err != nil { - return err - } - containers <- nfsServer.ID - if err := cli.ContainerStart(ctx, nfsServer.ID, container.StartOptions{}); err != nil { - return err - } - } - - // Add serial pty so we can do stuff like 'screen /dev/pts0' to access - // the VM console from the container without ssh - qemuArgs += " -serial pty" - - wg := sync.WaitGroup{} - wg.Add(int(nodes)) - // start one vm after each other - macCounter := 0 - for x := 0; x < int(nodes); x++ { - - nodeQemuArgs := qemuArgs - - for i := 0; i < int(secondaryNics); i++ { - netSuffix := fmt.Sprintf("%d-%d", x, i) - macSuffix := fmt.Sprintf("%02x", macCounter) - macCounter++ - nodeQemuArgs = fmt.Sprintf("%s -device virtio-net-pci,netdev=secondarynet%s,mac=52:55:00:d1:56:%s -netdev tap,id=secondarynet%s,ifname=stap%s,script=no,downscript=no", nodeQemuArgs, netSuffix, macSuffix, netSuffix, netSuffix) - } - - nodeName := nodeNameFromIndex(x + 1) - nodeNum := fmt.Sprintf("%02d", x+1) - sshClient, err = libssh.NewSSHClient(sshPort, x+1, false) - if err != nil { - return err - } - if reverse { - nodeName = nodeNameFromIndex((int(nodes) - x)) - nodeNum = fmt.Sprintf("%02d", (int(nodes) - x)) - sshClient, err = libssh.NewSSHClient(sshPort, (int(nodes) - x), false) - - if err != nil { - return err - } - } - - // assign a GPU to one node - var deviceMappings []container.DeviceMapping - if gpuAddress != "" && x == int(nodes)-1 { - iommu_group, err := getPCIDeviceIOMMUGroup(gpuAddress) - if err != nil { - return err - } - vfioDevice := fmt.Sprintf("/dev/vfio/%s", iommu_group) - deviceMappings = []container.DeviceMapping{ - { - PathOnHost: "/dev/vfio/vfio", - PathInContainer: "/dev/vfio/vfio", - CgroupPermissions: "mrw", - }, - { - PathOnHost: vfioDevice, - PathInContainer: vfioDevice, - CgroupPermissions: "mrw", - }, - } - nodeQemuArgs = fmt.Sprintf("%s -device vfio-pci,host=%s", nodeQemuArgs, gpuAddress) - } - - var vmArgsNvmeDisks []string - if len(nvmeDisks) > 0 { - for i, size := range nvmeDisks { - resource.MustParse(size) - disk := fmt.Sprintf("%s-%d.img", nvmeDiskImagePrefix, i) - nodeQemuArgs = fmt.Sprintf("%s -drive file=%s,format=raw,id=NVME%d,if=none -device nvme,drive=NVME%d,serial=nvme-%d", nodeQemuArgs, disk, i, i, i) - vmArgsNvmeDisks = append(vmArgsNvmeDisks, fmt.Sprintf("--nvme-device-size %s", size)) - } - } - var vmArgsSCSIDisks []string - if len(scsiDisks) > 0 { - nodeQemuArgs = fmt.Sprintf("%s -device virtio-scsi-pci,id=scsi0", nodeQemuArgs) - for i, size := range scsiDisks { - resource.MustParse(size) - disk := fmt.Sprintf("%s-%d.img", scsiDiskImagePrefix, i) - nodeQemuArgs = fmt.Sprintf("%s -drive file=%s,if=none,id=drive%d -device scsi-hd,drive=drive%d,bus=scsi0.0,channel=0,scsi-id=0,lun=%d", nodeQemuArgs, disk, i, i, i) - vmArgsSCSIDisks = append(vmArgsSCSIDisks, fmt.Sprintf("--scsi-device-size %s", size)) - } - } - - var vmArgsUSBDisks []string - const bus = " -device qemu-xhci,id=bus%d" - const drive = " -drive if=none,id=stick%d,format=raw,file=/usb-%d.img" - const dev = " -device usb-storage,bus=bus%d.0,drive=stick%d" - const usbSizefmt = " --usb-device-size %s" - if len(usbDisks) > 0 { - for i, size := range usbDisks { - resource.MustParse(size) - if i%2 == 0 { - nodeQemuArgs += fmt.Sprintf(bus, i/2) - } - nodeQemuArgs += fmt.Sprintf(drive, i, i) - nodeQemuArgs += fmt.Sprintf(dev, i/2, i) - vmArgsUSBDisks = append(vmArgsUSBDisks, fmt.Sprintf(usbSizefmt, size)) - } - } - - additionalArgs := []string{} - if len(nodeQemuArgs) > 0 { - additionalArgs = append(additionalArgs, "--qemu-args", shellescape.Quote(nodeQemuArgs)) - } - - if hugepages2Mcount > 0 { - kernelArgs += fmt.Sprintf(" hugepagesz=2M hugepages=%d", hugepages2Mcount) - } - - if hugepages1Gcount > 0 { - kernelArgs += fmt.Sprintf(" hugepagesz=1G hugepages=%d", hugepages1Gcount) - } - - if fipsEnabled { - kernelArgs += " fips=1" - } - - blockDev := "" - if cephEnabled { - blockDev = "--block-device /var/run/disk/blockdev.qcow2 --block-device-size 32212254720" - } - - kernelArgs = strings.TrimSpace(kernelArgs) - if kernelArgs != "" { - additionalArgs = append(additionalArgs, "--additional-kernel-args", shellescape.Quote(kernelArgs)) - } - - vmContainerConfig := &container.Config{ - Image: clusterImage, - Env: []string{ - fmt.Sprintf("NODE_NUM=%s", nodeNum), - }, - Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh -n /var/run/disk/disk.qcow2 --memory %s --cpu %s --numa %s %s %s %s %s %s", - memory, - strconv.Itoa(int(cpu)), - strconv.Itoa(int(numa)), - blockDev, - strings.Join(vmArgsSCSIDisks, " "), - strings.Join(vmArgsNvmeDisks, " "), - strings.Join(vmArgsUSBDisks, " "), - strings.Join(additionalArgs, " "), - )}, - } - - if cephEnabled { - vmContainerConfig.Volumes = map[string]struct{}{ - "/var/lib/rook": {}, - } - } - - node, err := cli.ContainerCreate(ctx, vmContainerConfig, &container.HostConfig{ - Privileged: true, - NetworkMode: container.NetworkMode("container:" + dnsmasq.ID), - Resources: container.Resources{ - Devices: deviceMappings, - }, - }, nil, nil, prefix+"-"+nodeName) - if err != nil { - return err - } - containers <- node.ID - if err := cli.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { - return err - } - - // Wait for vm start - success, err := docker.Exec(cli, nodeContainer(prefix, nodeName), []string{"/bin/bash", "-c", "while [ ! -f /ssh_ready ] ; do sleep 1; done"}, os.Stdout) - if err != nil { - return err - } - - if !success { - return fmt.Errorf("checking for ssh.sh script for node %s failed", nodeName) - } - - err = waitForVMToBeUp(prefix, nodeName) - if err != nil { - return err - } - - rootkey := rootkey.NewRootKey(sshClient) - if err = rootkey.Exec(); err != nil { - return err - } - sshClient, err = libssh.NewSSHClient(sshPort, x+1, true) - - linuxConfigFuncs := []nodesconfig.LinuxConfigFunc{ - nodesconfig.WithFipsEnabled(fipsEnabled), - nodesconfig.WithDockerProxy(dockerProxy), - nodesconfig.WithEtcdInMemory(runEtcdOnMemory), - nodesconfig.WithEtcdSize(etcdDataMountSize), - nodesconfig.WithSingleStack(singleStack), - nodesconfig.WithNoEtcdFsync(noEtcdFsync), - nodesconfig.WithEnableAudit(enableAudit), - nodesconfig.WithGpuAddress(gpuAddress), - nodesconfig.WithRealtime(realtimeSchedulingEnabled), - nodesconfig.WithPSA(psaEnabled), - nodesconfig.WithKsm(enableKsm), - nodesconfig.WithKsmPageCount(int(ksmPageCount)), - nodesconfig.WithKsmScanInterval(int(ksmScanInterval)), - nodesconfig.WithSwap(enableSwap), - nodesconfig.WithSwapiness(int(swapiness)), - nodesconfig.WithSwapSize(int(swapSize)), - nodesconfig.WithUnlimitedSwap(unlimitedSwap), - } - - n := nodesconfig.NewNodeLinuxConfig(x+1, prefix, linuxConfigFuncs) - - if err = provisionNode(sshClient, n); err != nil { - return err - } - - go func(id string) { - cli.ContainerWait(ctx, id, container.WaitConditionNotRunning) - wg.Done() - }(node.ID) - } - - sshClient, err := libssh.NewSSHClient(sshPort, 1, true) - if err != nil { - return err - } - - k8sConfs := []nodesconfig.K8sConfigFunc{ - nodesconfig.WithCeph(cephEnabled), - nodesconfig.WithPrometheus(prometheusEnabled), - nodesconfig.WithAlertmanager(prometheusAlertmanagerEnabled), - nodesconfig.WithGrafana(grafanaEnabled), - nodesconfig.WithIstio(istioEnabled), - nodesconfig.WithNfsCsi(nfsCsiEnabled), - nodesconfig.WithCnao(cnaoEnabled), - nodesconfig.WithCNAOSkipCR(cnaoSkipCR), - nodesconfig.WithMultus(deployMultus), - nodesconfig.WithCdi(deployCdi), - nodesconfig.WithCdiVersion(cdiVersion), - nodesconfig.WithAAQ(deployAaq), - nodesconfig.WithAAQVersion(aaqVersion), - } - n := nodesconfig.NewNodeK8sConfig(k8sConfs) - - kubeConfFile, err := os.Create(".kubeconfig") - if err != nil { - return err - } - - err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", kubeConfFile) - if err != nil { - return err - } - - config, err := k8s.NewConfig(".kubeconfig", apiServerPort) + b := context.Background() + ctx, cancel := context.WithCancel(b) + err = docker.ImagePull(cli, ctx, clusterImage, image.PullOptions{}) if err != nil { - return err + return fmt.Errorf("Failed to download cluster image %s, %s", clusterImage, err) } + kp := providers.NewKubevirtProvider(k8sVersion, clusterImage, cli, opts) - k8sClient, err := k8s.NewDynamicClient(config) + err = kp.Start(ctx, cancel, portMap) if err != nil { return err } - if err = provisionK8sOptions(sshClient, k8sClient, n, prefix); err != nil { - return err - } - - // If background flag was specified, we don't want to clean up if we reach that state - if !background { - wg.Wait() - stop <- fmt.Errorf("Done. please clean up") - } - return nil } - -func provisionK8sOptions(sshClient libssh.Client, k8sClient k8s.K8sDynamicClient, n *nodesconfig.NodeK8sConfig, k8sVersion string) error { - opts := []opts.Opt{} - - if n.Ceph { - cephOpt := rookceph.NewCephOpt(k8sClient, sshClient) - opts = append(opts, cephOpt) - } - - if n.NfsCsi { - nfsCsiOpt := nfscsi.NewNfsCsiOpt(k8sClient) - opts = append(opts, nfsCsiOpt) - } - - if n.Multus { - multusOpt := multus.NewMultusOpt(k8sClient, sshClient) - opts = append(opts, multusOpt) - } - - if n.CNAO { - cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient, n.Multus, n.CNAOSkipCR) - opts = append(opts, cnaoOpt) - } - - if n.Istio { - istioOpt := istio.NewIstioOpt(sshClient, k8sClient, n.CNAO) - opts = append(opts, istioOpt) - } - - if n.Prometheus { - prometheusOpt := prometheus.NewPrometheusOpt(k8sClient, n.Grafana, n.Alertmanager) - opts = append(opts, prometheusOpt) - } - - if n.CDI { - cdi := cdi.NewCdiOpt(k8sClient, sshClient, n.CDIVersion) - opts = append(opts, cdi) - } - - if n.AAQ { - if k8sVersion == "k8s-1.30" { - aaq := aaq.NewAaqOpt(k8sClient, sshClient, n.CDIVersion) - opts = append(opts, aaq) - } else { - logrus.Info("AAQ was requested but k8s version is not k8s-1.30, skipping") - } - } - - for _, opt := range opts { - if err := opt.Exec(); err != nil { - return err - } - } - - return nil -} - -func provisionNode(sshClient libssh.Client, n *nodesconfig.NodeLinuxConfig) error { - opts := []opts.Opt{} - nodeName := nodeNameFromIndex(n.NodeIdx) - - if n.FipsEnabled { - for _, cmd := range []string{"sudo fips-mode-setup --enable", "sudo reboot"} { - if err := sshClient.Command(cmd); err != nil { - return fmt.Errorf("Starting fips mode failed: %s", err) - } - } - err := waitForVMToBeUp(n.K8sVersion, nodeName) - if err != nil { - return err - } - } - - if n.DockerProxy != "" { - //if dockerProxy has value, generate a shell script`/script/docker-proxy.sh` which can be applied to set proxy settings - dp := dockerproxy.NewDockerProxyOpt(sshClient, n.DockerProxy) - opts = append(opts, dp) - } - - if n.EtcdInMemory { - logrus.Infof("Creating in-memory mount for etcd data on node %s", nodeName) - etcdinmem := etcdinmemory.NewEtcdInMemOpt(sshClient, n.EtcdSize) - opts = append(opts, etcdinmem) - } - - if n.Realtime { - realtimeOpt := realtime.NewRealtimeOpt(sshClient) - opts = append(opts, realtimeOpt) - } - - for _, s := range soundcardPCIIDs { - // move the VM sound cards to a vfio-pci driver to prepare for assignment - bvfio := bindvfio.NewBindVfioOpt(sshClient, s) - opts = append(opts, bvfio) - } - - if n.EnableAudit { - if err := sshClient.Command("touch /home/vagrant/enable_audit"); err != nil { - return fmt.Errorf("provisioning node %d failed (setting enableAudit phase): %s", n.NodeIdx, err) - } - } - - if n.PSA { - psaOpt := psa.NewPsaOpt(sshClient) - opts = append(opts, psaOpt) - } - - if n.NodeIdx == 1 { - n := node01.NewNode01Provisioner(sshClient, n.SingleStack, n.NoEtcdFsync) - opts = append(opts, n) - - } else { - if n.GpuAddress != "" { - // move the assigned PCI device to a vfio-pci driver to prepare for assignment - gpuDeviceID, err := getDevicePCIID(n.GpuAddress) - if err != nil { - return err - } - bindVfioOpt := bindvfio.NewBindVfioOpt(sshClient, gpuDeviceID) - opts = append(opts, bindVfioOpt) - } - n := nodesprovision.NewNodesProvisioner(sshClient, n.SingleStack) - opts = append(opts, n) - } - - if n.KsmEnabled { - ksmOpt := ksm.NewKsmOpt(sshClient, n.KsmScanInterval, n.KsmPageCount) - opts = append(opts, ksmOpt) - } - - if n.SwapEnabled { - swapOpt := swap.NewSwapOpt(sshClient, n.Swappiness, n.UnlimitedSwap, n.SwapSize) - opts = append(opts, swapOpt) - } - - for _, o := range opts { - if err := o.Exec(); err != nil { - return err - } - } - - return nil -} - -func waitForVMToBeUp(prefix string, nodeName string) error { - var err error - // Wait for the VM to be up - for x := 0; x < 10; x++ { - err = _cmd(cli, nodeContainer(prefix, nodeName), "ssh.sh echo VM is up", "waiting for node to come up") - if err == nil { - break - } - logrus.WithError(err).Warningf("Could not establish a ssh connection to the VM, retrying ...") - time.Sleep(1 * time.Second) - } - - if err != nil { - return fmt.Errorf("could not establish a connection to the node after a generous timeout: %v", err) - } - - return nil -} - -func nodeNameFromIndex(x int) string { - return fmt.Sprintf("node%02d", x) -} - -func nodeContainer(prefix string, node string) string { - return prefix + "-" + node -} - -func getDockerProxyConfig(proxy string) (string, error) { - p := dockerSetting{Proxy: proxy} - buf := new(bytes.Buffer) - - t, err := template.New("docker-proxy").Parse(proxySettings) - if err != nil { - return "", err - } - err = t.Execute(buf, p) - if err != nil { - return "", err - } - return buf.String(), nil -} - -// getDeviceIOMMUGroup gets devices iommu_group -// e.g. /sys/bus/pci/devices/0000\:65\:00.0/iommu_group -> ../../../../../kernel/iommu_groups/45 -func getPCIDeviceIOMMUGroup(pciAddress string) (string, error) { - iommuLink := filepath.Join("/sys/bus/pci/devices", pciAddress, "iommu_group") - iommuPath, err := os.Readlink(iommuLink) - if err != nil { - return "", fmt.Errorf("failed to read iommu_group link %s for device %s - %v", iommuLink, pciAddress, err) - } - _, iommuGroup := filepath.Split(iommuPath) - return iommuGroup, nil -} - -func getDevicePCIID(pciAddress string) (string, error) { - file, err := os.Open(filepath.Join("/sys/bus/pci/devices", pciAddress, "uevent")) - if err != nil { - return "", err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "PCI_ID") { - equal := strings.Index(line, "=") - value := strings.TrimSpace(line[equal+1:]) - return strings.ToLower(value), nil - } - } - return "", fmt.Errorf("no pci_id is found") -} From 79b1d7a341dca1f9a8d1858bd9a93ec8339131a0 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 4 Oct 2024 20:14:58 +0300 Subject: [PATCH 04/69] refactor: Rewrite provision.go to use the KubevirtProvider Signed-off-by: aerosouund --- cluster-provision/gocli/cmd/provision.go | 371 ++++------------------- 1 file changed, 64 insertions(+), 307 deletions(-) diff --git a/cluster-provision/gocli/cmd/provision.go b/cluster-provision/gocli/cmd/provision.go index e86858eca8..47e43ae1c6 100644 --- a/cluster-provision/gocli/cmd/provision.go +++ b/cluster-provision/gocli/cmd/provision.go @@ -2,33 +2,32 @@ package cmd import ( "fmt" - "io/ioutil" - "os" - "os/signal" "path/filepath" - "strconv" "strings" - "github.com/alessio/shellescape" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/docker/go-connections/nat" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/net/context" - containers2 "kubevirt.io/kubevirtci/cluster-provision/gocli/containers" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" - "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" + "kubevirt.io/kubevirtci/cluster-provision/gocli/providers" ) +const ( + baseLinuxPhase = "quay.io/kubevirtci/centos9" + baseK8sPhase = "quay.io/kubevirtci/centos9:2408130400-bb670376" +) + +var versionMap = map[string]string{ + "1.30": "1.30.2", + "1.29": "1.29.6", + "1.28": "1.28.11", + "1.31": "1.31.0", +} + // NewProvisionCommand provision given cluster func NewProvisionCommand() *cobra.Command { - provision := &cobra.Command{ Use: "provision", Short: "provision starts a given cluster", @@ -38,7 +37,7 @@ func NewProvisionCommand() *cobra.Command { provision.Flags().StringP("memory", "m", "3096M", "amount of ram per node") provision.Flags().UintP("cpu", "c", 2, "number of cpu cores per node") provision.Flags().String("qemu-args", "", "additional qemu args to pass through to the nodes") - provision.Flags().Bool("random-ports", false, "expose all ports on random localhost ports") + provision.Flags().Bool("random-ports", true, "expose all ports on random localhost ports") provision.Flags().Bool("slim", false, "create slim provider (uncached images)") provision.Flags().Uint("vnc-port", 0, "port on localhost for vnc") provision.Flags().Uint("ssh-port", 0, "port on localhost for ssh server") @@ -51,330 +50,88 @@ func NewProvisionCommand() *cobra.Command { func provisionCluster(cmd *cobra.Command, args []string) (retErr error) { var base string - packagePath := args[0] - versionBytes, err := os.ReadFile(filepath.Join(packagePath, "version")) - if err != nil { - return err - } - version := strings.TrimSpace(string(versionBytes)) - baseBytes, err := os.ReadFile(filepath.Join(packagePath, "base")) - if err != nil { - return err + versionNoMinor := args[0] + + v, ok := versionMap[versionNoMinor] + if !ok { + return fmt.Errorf("Invalid version passed, exiting!") + } + + opts := []providers.KubevirtProviderOption{} + flags := cmd.Flags() + for flagName, flagConfig := range providers.ProvisionFlagMap { + switch flagConfig.FlagType { + case "string": + flagVal, err := flags.GetString(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "bool": + flagVal, err := flags.GetBool(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + + case "uint": + flagVal, err := flags.GetUint(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "uint16": + flagVal, err := flags.GetUint16(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + case "[]string": + flagVal, err := flags.GetStringArray(flagName) + if err != nil { + return err + } + opts = append(opts, flagConfig.ProviderOptFunc(flagVal)) + } } + phases, err := cmd.Flags().GetString("phases") if err != nil { return err } if strings.Contains(phases, "linux") { - base = fmt.Sprintf("quay.io/kubevirtci/%s", strings.TrimSpace(string(baseBytes))) + base = baseLinuxPhase } else { - k8sPath := fmt.Sprintf("%s/../", packagePath) - baseImageBytes, err := os.ReadFile(filepath.Join(k8sPath, "base-image")) - if err != nil { - return err - } - base = strings.TrimSpace(string(baseImageBytes)) + base = baseK8sPhase } containerSuffix, err := cmd.Flags().GetString("container-suffix") if err != nil { return err } - name := filepath.Base(packagePath) + name := filepath.Base(versionNoMinor) if len(containerSuffix) > 0 { name = fmt.Sprintf("%s-%s", name, containerSuffix) } - prefix := fmt.Sprintf("k8s-%s-provision", name) - target := fmt.Sprintf("quay.io/kubevirtci/k8s-%s", name) - scripts := filepath.Join(packagePath) - - if phases == "linux" { - target = base + "-base" - } - - memory, err := cmd.Flags().GetString("memory") - if err != nil { - return err - } - - randomPorts, err := cmd.Flags().GetBool("random-ports") - if err != nil { - return err - } - - slim, err := cmd.Flags().GetBool("slim") - if err != nil { - return err - } portMap := nat.PortMap{} utils.AppendTCPIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port") utils.AppendTCPIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port") - qemuArgs, err := cmd.Flags().GetString("qemu-args") - if err != nil { - return err - } - - cpu, err := cmd.Flags().GetUint("cpu") - if err != nil { - return err - } - cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return err } - ctx := context.Background() - - stop := make(chan error, 10) - containers, volumes, done := docker.NewCleanupHandler(cli, stop, cmd.OutOrStderr(), true) - - defer func() { - stop <- retErr - <-done - }() - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - stop <- fmt.Errorf("Interrupt received, clean up") - }() + ctx, cancel := context.WithCancel(context.Background()) - // Pull the base image - err = docker.ImagePull(cli, ctx, base, types.ImagePullOptions{}) + kp := providers.NewKubevirtProvider(versionNoMinor, base, cli, opts) + err = kp.Provision(ctx, cancel, portMap, v) if err != nil { - panic(err) - } - - // Start dnsmasq - dnsmasq, err := containers2.DNSMasq(cli, ctx, &containers2.DNSMasqOptions{ - ClusterImage: base, - SecondaryNicsCount: 0, - RandomPorts: randomPorts, - PortMap: portMap, - Prefix: prefix, - NodeCount: 1, - }) - if err != nil { - return err - } - containers <- dnsmasq.ID - if err := cli.ContainerStart(ctx, dnsmasq.ID, container.StartOptions{}); err != nil { return err } - nodeName := nodeNameFromIndex(1) - nodeNum := fmt.Sprintf("%02d", 1) - - vol, err := cli.VolumeCreate(ctx, volume.CreateOptions{ - Name: fmt.Sprintf("%s-%s", prefix, nodeName), - }) - if err != nil { - return err - } - volumes <- vol.Name - registryVol, err := cli.VolumeCreate(ctx, volume.CreateOptions{ - Name: fmt.Sprintf("%s-%s", prefix, "registry"), - }) - if err != nil { - return err - } - - if len(qemuArgs) > 0 { - qemuArgs = "--qemu-args " + qemuArgs - } - node, err := cli.ContainerCreate(ctx, &container.Config{ - Image: base, - Env: []string{ - fmt.Sprintf("NODE_NUM=%s", nodeNum), - }, - Volumes: map[string]struct{}{ - "/var/run/disk": {}, - "/var/lib/registry": {}, - }, - Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh --memory %s --cpu %s %s", memory, strconv.Itoa(int(cpu)), qemuArgs)}, - }, &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: "volume", - Source: vol.Name, - Target: "/var/run/disk", - }, - { - Type: "volume", - Source: registryVol.Name, - Target: "/var/lib/registry", - }, - }, - Privileged: true, - NetworkMode: container.NetworkMode("container:" + dnsmasq.ID), - }, nil, nil, nodeContainer(prefix, nodeName)) - if err != nil { - return err - } - containers <- node.ID - if err := cli.ContainerStart(ctx, node.ID, container.StartOptions{}); err != nil { - return err - } - - // copy provider scripts - err = copyDirectory(ctx, cli, node.ID, scripts, "/scripts") - if err != nil { - return err - } - - // Wait for ssh.sh script to exist - err = _cmd(cli, nodeContainer(prefix, nodeName), "while [ ! -f /ssh_ready ] ; do sleep 1; done", "checking for ssh.sh script") - if err != nil { - return err - } - - // Wait for the VM to be up - err = _cmd(cli, nodeContainer(prefix, nodeName), "ssh.sh echo VM is up", "waiting for node to come up") - if err != nil { - return err - } - - envVars := fmt.Sprintf("version=%s slim=%t", version, slim) - if strings.Contains(phases, "linux") { - err = performPhase(cli, nodeContainer(prefix, nodeName), "/scripts/provision.sh", envVars) - if err != nil { - return err - } - } - if strings.Contains(phases, "k8s") { - // copy provider scripts - err = copyDirectory(ctx, cli, node.ID, scripts, "/scripts") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "if [ -f /scripts/extra-pre-pull-images ]; then scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/extra-pre-pull-images vagrant@192.168.66.101:/tmp/extra-pre-pull-images; fi", "copying /scripts/extra-pre-pull-images if existing") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "if [ -f /scripts/fetch-images.sh ]; then scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/fetch-images.sh vagrant@192.168.66.101:/tmp/fetch-images.sh; fi", "copying /scripts/fetch-images.sh if existing") - if err != nil { - return err - } - - err = _cmd(cli, nodeContainer(prefix, nodeName), "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key vagrant@192.168.66.101 'mkdir -p /tmp/ceph /tmp/cnao /tmp/nfs-csi /tmp/nodeports /tmp/prometheus /tmp/whereabouts /tmp/kwok'", "Create required manifest directories before copy") - if err != nil { - return err - } - // Copy manifests to the VM - err = _cmd(cli, nodeContainer(prefix, nodeName), "scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/manifests/* vagrant@192.168.66.101:/tmp", "copying manifests to the VM") - if err != nil { - return err - } - - err = performPhase(cli, nodeContainer(prefix, nodeName), "/scripts/k8s_provision.sh", envVars) - if err != nil { - return err - } - } - - _cmd(cli, nodeContainer(prefix, nodeName), "ssh.sh sudo shutdown now -h", "shutting down the node") - err = _cmd(cli, nodeContainer(prefix, nodeName), "rm /usr/local/bin/ssh.sh", "removing the ssh.sh script") - if err != nil { - return err - } - err = _cmd(cli, nodeContainer(prefix, nodeName), "rm /ssh_ready", "removing the ssh_ready mark") - if err != nil { - return err - } - logrus.Info("waiting for the node to stop") - okChan, errChan := cli.ContainerWait(ctx, nodeContainer(prefix, nodeName), container.WaitConditionNotRunning) - select { - case <-okChan: - case err := <-errChan: - if err != nil { - return fmt.Errorf("waiting for the node to stop failed: %v", err) - } - } - - logrus.Info("preparing additional persistent kernel arguments after initial provision") - additionalKernelArguments, err := cmd.Flags().GetStringArray("additional-persistent-kernel-arguments") - if err != nil { - return err - } - - dir, err := ioutil.TempDir("", "gocli") - if err != nil { - return fmt.Errorf("failed creating a temporary directory: %v", err) - } - defer os.RemoveAll(dir) - if err := ioutil.WriteFile(filepath.Join(dir, "additional.kernel.args"), []byte(shellescape.QuoteCommand(additionalKernelArguments)), 0666); err != nil { - return fmt.Errorf("failed creating additional.kernel.args file: %v", err) - } - if err := copyDirectory(ctx, cli, node.ID, dir, "/"); err != nil { - return fmt.Errorf("failed copying additional kernel arguments into the container: %v", err) - } - - logrus.Infof("Commiting the node as %s", target) - _, err = cli.ContainerCommit(ctx, node.ID, container.CommitOptions{ - Reference: target, - Comment: "PROVISION SUCCEEDED", - Author: "gocli", - Changes: nil, - Pause: false, - Config: nil, - }) - if err != nil { - return fmt.Errorf("commiting the node failed: %v", err) - } - return nil } - -func copyDirectory(ctx context.Context, cli *client.Client, containerID string, sourceDirectory string, targetDirectory string) error { - srcInfo, err := archive.CopyInfoSourcePath(sourceDirectory, false) - if err != nil { - return err - } - - srcArchive, err := archive.TarResource(srcInfo) - if err != nil { - return err - } - defer srcArchive.Close() - - dstInfo := archive.CopyInfo{Path: targetDirectory} - - dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) - if err != nil { - return err - } - defer preparedArchive.Close() - - err = cli.CopyToContainer(ctx, containerID, dstDir, preparedArchive, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false}) - if err != nil { - return err - } - return nil -} - -func _cmd(cli *client.Client, container string, cmd string, description string) error { - logrus.Info(description) - success, err := docker.Exec(cli, container, []string{"/bin/bash", "-c", cmd}, os.Stdout) - if err != nil { - return fmt.Errorf("%s failed: %v", description, err) - } else if !success { - return fmt.Errorf("%s failed", cmd) - } - return nil -} - -func performPhase(cli *client.Client, container string, script string, envVars string) error { - err := _cmd(cli, container, fmt.Sprintf("test -f %s", script), "checking provision scripts") - if err != nil { - return err - } - - return _cmd(cli, container, - fmt.Sprintf("ssh.sh sudo %s /bin/bash < %s", envVars, script), - fmt.Sprintf("provisioning the node (%s)", script)) -} From 29a2bc4b6f5c80fa1e88e0f301eddf53a3385da7 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:31:39 +0300 Subject: [PATCH 05/69] refactor: Delete the nodesconfig package This functionality now exists in the KubevirtProvider type and doesn't need a package of its own Signed-off-by: aerosouund --- .../gocli/cmd/nodesconfig/nodeconfig.go | 64 ------ .../gocli/cmd/nodesconfig/opts.go | 209 ------------------ 2 files changed, 273 deletions(-) delete mode 100644 cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go delete mode 100644 cluster-provision/gocli/cmd/nodesconfig/opts.go diff --git a/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go b/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go deleted file mode 100644 index 187cbb0774..0000000000 --- a/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go +++ /dev/null @@ -1,64 +0,0 @@ -package nodesconfig - -// NodeLinuxConfig type holds the config params that a node can have for its linux system -type NodeLinuxConfig struct { - NodeIdx int - K8sVersion string - FipsEnabled bool - DockerProxy string - EtcdInMemory bool - EtcdSize string - SingleStack bool - NoEtcdFsync bool - EnableAudit bool - GpuAddress string - Realtime bool - PSA bool - KsmEnabled bool - SwapEnabled bool - KsmPageCount int - KsmScanInterval int - Swappiness int - UnlimitedSwap bool - SwapSize int -} - -// NodeK8sConfig type holds the config k8s options for kubevirt cluster -type NodeK8sConfig struct { - Ceph bool - Prometheus bool - Alertmanager bool - Grafana bool - Istio bool - NfsCsi bool - CNAO bool - CNAOSkipCR bool - Multus bool - CDI bool - CDIVersion string - AAQ bool - AAQVersion string -} - -func NewNodeK8sConfig(confs []K8sConfigFunc) *NodeK8sConfig { - n := &NodeK8sConfig{} - - for _, conf := range confs { - conf(n) - } - - return n -} - -func NewNodeLinuxConfig(nodeIdx int, k8sVersion string, confs []LinuxConfigFunc) *NodeLinuxConfig { - n := &NodeLinuxConfig{ - NodeIdx: nodeIdx, - K8sVersion: k8sVersion, - } - - for _, conf := range confs { - conf(n) - } - - return n -} diff --git a/cluster-provision/gocli/cmd/nodesconfig/opts.go b/cluster-provision/gocli/cmd/nodesconfig/opts.go deleted file mode 100644 index 1994660dd8..0000000000 --- a/cluster-provision/gocli/cmd/nodesconfig/opts.go +++ /dev/null @@ -1,209 +0,0 @@ -package nodesconfig - -type LinuxConfigFunc func(n *NodeLinuxConfig) - -type K8sConfigFunc func(n *NodeK8sConfig) - -func WithNodeIdx(nodeIdx int) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.NodeIdx = nodeIdx - } -} - -func WithK8sVersion(k8sVersion string) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.K8sVersion = k8sVersion - } -} - -func WithFipsEnabled(fipsEnabled bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.FipsEnabled = fipsEnabled - } -} - -func WithDockerProxy(dockerProxy string) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.DockerProxy = dockerProxy - } -} - -func WithEtcdInMemory(etcdInMemory bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.EtcdInMemory = etcdInMemory - } -} - -func WithEtcdSize(etcdSize string) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.EtcdSize = etcdSize - } -} - -func WithSingleStack(singleStack bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.SingleStack = singleStack - } -} - -func WithNoEtcdFsync(noEtcdFsync bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.NoEtcdFsync = noEtcdFsync - } -} - -func WithEnableAudit(enableAudit bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.EnableAudit = enableAudit - } -} - -func WithGpuAddress(gpuAddress string) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.GpuAddress = gpuAddress - } -} - -func WithRealtime(realtime bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.Realtime = realtime - } -} - -func WithPSA(psa bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.PSA = psa - } -} - -func WithKsm(ksm bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.KsmEnabled = ksm - } -} - -func WithSwap(swap bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.SwapEnabled = swap - } -} - -func WithKsmEnabled(ksmEnabled bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.KsmEnabled = ksmEnabled - } -} - -func WithSwapEnabled(swapEnabled bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.SwapEnabled = swapEnabled - } -} - -func WithKsmPageCount(ksmPageCount int) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.KsmPageCount = ksmPageCount - } -} - -func WithKsmScanInterval(ksmScanInterval int) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.KsmScanInterval = ksmScanInterval - } -} - -func WithSwapiness(swapiness int) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.Swappiness = swapiness - } -} - -func WithUnlimitedSwap(unlimitedSwap bool) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.UnlimitedSwap = unlimitedSwap - } -} - -func WithSwapSize(swapSize int) LinuxConfigFunc { - return func(n *NodeLinuxConfig) { - n.SwapSize = swapSize - } -} - -func WithCeph(ceph bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Ceph = ceph - } -} - -func WithPrometheus(prometheus bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Prometheus = prometheus - } -} - -func WithAlertmanager(alertmanager bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Alertmanager = alertmanager - } -} - -func WithGrafana(grafana bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Grafana = grafana - } -} - -func WithIstio(istio bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Istio = istio - } -} - -func WithNfsCsi(nfsCsi bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.NfsCsi = nfsCsi - } -} - -func WithCnao(cnao bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.CNAO = cnao - } -} - -func WithCNAOSkipCR(skip bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.CNAOSkipCR = skip - } -} - -func WithMultus(multus bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.Multus = multus - } -} - -func WithCdi(cdi bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.CDI = cdi - } -} - -func WithCdiVersion(cdiVersion string) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.CDIVersion = cdiVersion - } -} - -func WithAAQ(aaq bool) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.AAQ = aaq - } -} - -func WithAAQVersion(aaqVersion string) K8sConfigFunc { - return func(n *NodeK8sConfig) { - n.AAQVersion = aaqVersion - } -} From 9b07a179add36e6b26bbdea987c446a635b9a2d4 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:32:05 +0300 Subject: [PATCH 06/69] testing: Move testing logic to the providers package The KubevirtProvider type is what provides the methods that run a node or run the k8s options. Testing logic has been moved to a Base Provider Suite Signed-off-by: aerosouund --- .../providers/base_provider_suite_test.go | 13 ++++++ .../base_provider_test.go} | 44 +++++++++---------- 2 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 cluster-provision/gocli/providers/base_provider_suite_test.go rename cluster-provision/gocli/{cmd/run_test.go => providers/base_provider_test.go} (72%) diff --git a/cluster-provision/gocli/providers/base_provider_suite_test.go b/cluster-provision/gocli/providers/base_provider_suite_test.go new file mode 100644 index 0000000000..775f262b85 --- /dev/null +++ b/cluster-provision/gocli/providers/base_provider_suite_test.go @@ -0,0 +1,13 @@ +package providers + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCmd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Base Provider Suite") +} diff --git a/cluster-provision/gocli/cmd/run_test.go b/cluster-provision/gocli/providers/base_provider_test.go similarity index 72% rename from cluster-provision/gocli/cmd/run_test.go rename to cluster-provision/gocli/providers/base_provider_test.go index 985db922c1..3153bfea93 100644 --- a/cluster-provision/gocli/cmd/run_test.go +++ b/cluster-provision/gocli/providers/base_provider_test.go @@ -1,14 +1,15 @@ -package cmd +package providers import ( + "github.com/docker/docker/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" - "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/nodesconfig" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/aaq" bindvfio "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/bind-vfio" etcdinmemory "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/labelnodes" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/psa" @@ -23,6 +24,7 @@ var _ = Describe("Node Provisioning", func() { sshClient *kubevirtcimocks.MockSSHClient reactors []k8s.ReactorConfig k8sClient k8s.K8sDynamicClient + kp *KubevirtProvider ) BeforeEach(func() { @@ -35,6 +37,19 @@ var _ = Describe("Node Provisioning", func() { } k8sClient = k8s.NewTestClient(reactors...) + kp = NewKubevirtProvider("k8s-1.30", "", &client.Client{}, []KubevirtProviderOption{ + WithNodes(uint(1)), + WithEnablePSA(true), + WithEtcdCapacity("512M"), + WithRunEtcdOnMemory(true), + WithEnableCeph(true), + WithEnablePrometheus(true), + WithEnablePrometheusAlertManager(true), + WithEnableIstio(true), + WithAAQ(true), + WithEnableNFSCSI(true), + WithEnableGrafana(true), + }) }) AfterEach(func() { @@ -45,43 +60,26 @@ var _ = Describe("Node Provisioning", func() { Describe("ProvisionNode", func() { It("should execute the correct commands", func() { - linuxConfigFuncs := []nodesconfig.LinuxConfigFunc{ - nodesconfig.WithEtcdInMemory(true), - nodesconfig.WithEtcdSize("512M"), - nodesconfig.WithPSA(true), - } - - n := nodesconfig.NewNodeLinuxConfig(1, "k8s-1.30", linuxConfigFuncs) - etcdinmemory.AddExpectCalls(sshClient, "512M") bindvfio.AddExpectCalls(sshClient, "8086:2668") bindvfio.AddExpectCalls(sshClient, "8086:2415") psa.AddExpectCalls(sshClient) node01.AddExpectCalls(sshClient) - err := provisionNode(sshClient, n) + err := kp.provisionNode(sshClient, 1) Expect(err).NotTo(HaveOccurred()) }) }) Describe("ProvisionNodeK8sOpts", func() { It("should execute the correct K8s option commands", func() { - k8sConfs := []nodesconfig.K8sConfigFunc{ - nodesconfig.WithCeph(true), - nodesconfig.WithPrometheus(true), - nodesconfig.WithAlertmanager(true), - nodesconfig.WithGrafana(true), - nodesconfig.WithIstio(true), - nodesconfig.WithNfsCsi(true), - nodesconfig.WithAAQ(true), - } - n := nodesconfig.NewNodeK8sConfig(k8sConfs) + kp.Client = k8sClient - rookceph.AddExpectCalls(sshClient) + labelnodes.AddExpectCalls(sshClient, "node-role.kubernetes.io/control-plane") istio.AddExpectCalls(sshClient) aaq.AddExpectCalls(sshClient) - err := provisionK8sOptions(sshClient, k8sClient, n, "k8s-1.30") + err := kp.provisionK8sOpts(sshClient) Expect(err).NotTo(HaveOccurred()) }) }) From d0f4bca1bc54d545ebacfe377f9147a92a0f7e21 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 17 Aug 2024 15:42:15 +0300 Subject: [PATCH 07/69] feat: Introduce retries in ssh connection in the ssh client Signed-off-by: aerosouund --- cluster-provision/gocli/pkg/libssh/ssh.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cluster-provision/gocli/pkg/libssh/ssh.go b/cluster-provision/gocli/pkg/libssh/ssh.go index e6f8f3d10d..06a57285aa 100644 --- a/cluster-provision/gocli/pkg/libssh/ssh.go +++ b/cluster-provision/gocli/pkg/libssh/ssh.go @@ -12,7 +12,10 @@ import ( "strings" "sync" + "time" + "github.com/bramvdbogaerde/go-scp" + "github.com/cenkalti/backoff/v4" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) @@ -224,9 +227,24 @@ func (s *SSHClientImpl) executeCommand(cmd string, outWriter, errWriter io.Write func (s *SSHClientImpl) initClient() error { s.initMutex.Lock() defer s.initMutex.Unlock() - client, err := ssh.Dial("tcp", net.JoinHostPort("127.0.0.1", fmt.Sprint(s.sshPort)), s.config) + + var ( + client *ssh.Client + err error + ) + + operation := func() error { + client, err = ssh.Dial("tcp", net.JoinHostPort("127.0.0.1", fmt.Sprint(s.sshPort)), s.config) + return err + } + + backoffStrategy := backoff.NewExponentialBackOff() + backoffStrategy.InitialInterval = 3 * time.Second + backoffStrategy.MaxElapsedTime = 1 * time.Minute + + err = backoff.Retry(operation, backoffStrategy) if err != nil { - return fmt.Errorf("Failed to connect to SSH server: %v", err) + return err } conn, err := client.Dial("tcp", fmt.Sprintf("192.168.66.10%d:22", s.nodeIdx)) From 7cfb6508cabc318c42d8ad9ba26a9212691359d9 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 2 Oct 2024 14:26:45 +0300 Subject: [PATCH 08/69] refactor: Change the return type of GetPublicPort to uint instead of uint16 All references to ports in the codebase use uint not uint16. There is no reason to keep the ports as they are Signed-off-by: aerosouund --- cluster-provision/gocli/cmd/utils/ports.go | 4 ++-- cluster-provision/gocli/providers/opts.go | 20 ++++++++++---------- cluster-provision/gocli/providers/start.go | 12 ++++++------ cluster-provision/gocli/providers/types.go | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cluster-provision/gocli/cmd/utils/ports.go b/cluster-provision/gocli/cmd/utils/ports.go index 7753bba2c0..34d6e3ba70 100644 --- a/cluster-provision/gocli/cmd/utils/ports.go +++ b/cluster-provision/gocli/cmd/utils/ports.go @@ -65,7 +65,7 @@ const ( ) // GetPublicPort returns public port by private port -func GetPublicPort(port uint16, ports nat.PortMap) (uint16, error) { +func GetPublicPort(port uint, ports nat.PortMap) (uint16, error) { portStr := strconv.Itoa(int(port)) for k, p := range ports { if k == nat.Port(portStr+"/tcp") || k == nat.Port(portStr+"/udp") { @@ -84,7 +84,7 @@ func GetPublicPort(port uint16, ports nat.PortMap) (uint16, error) { } // PrintPublicPort prints public port -func PrintPublicPort(port uint16, ports nat.PortMap) error { +func PrintPublicPort(port uint, ports nat.PortMap) error { p, err := GetPublicPort(port, ports) if err != nil { return err diff --git a/cluster-provision/gocli/providers/opts.go b/cluster-provision/gocli/providers/opts.go index 69537c238c..bd2851ff03 100644 --- a/cluster-provision/gocli/providers/opts.go +++ b/cluster-provision/gocli/providers/opts.go @@ -86,61 +86,61 @@ func WithSlim(slim interface{}) KubevirtProviderOption { func WithVNCPort(vncPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.VNCPort = vncPort.(uint16) + c.VNCPort = vncPort.(uint) } } func WithHTTPPort(httpPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.HTTPPort = httpPort.(uint16) + c.HTTPPort = httpPort.(uint) } } func WithHTTPSPort(httpsPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.HTTPSPort = httpsPort.(uint16) + c.HTTPSPort = httpsPort.(uint) } } func WithRegistryPort(registryPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.RegistryPort = registryPort.(uint16) + c.RegistryPort = registryPort.(uint) } } func WithOCPort(ocpPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.OCPort = ocpPort.(uint16) + c.OCPort = ocpPort.(uint) } } func WithK8sPort(k8sPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.K8sPort = k8sPort.(uint16) + c.K8sPort = k8sPort.(uint) } } func WithSSHPort(sshPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.SSHPort = sshPort.(uint16) + c.SSHPort = sshPort.(uint) } } func WithPrometheusPort(prometheusPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.PrometheusPort = prometheusPort.(uint16) + c.PrometheusPort = prometheusPort.(uint) } } func WithGrafanaPort(grafanaPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.GrafanaPort = grafanaPort.(uint16) + c.GrafanaPort = grafanaPort.(uint) } } func WithDNSPort(dnsPort interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { - c.DNSPort = dnsPort.(uint16) + c.DNSPort = dnsPort.(uint) } } diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 2ee3ceeeeb..ff73c461e0 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -63,7 +63,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc if err != nil { return err } - kp.SSHPort = port + kp.SSHPort = uint(port) } if kp.APIServerPort == 0 { @@ -71,7 +71,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc if err != nil { return err } - kp.APIServerPort = port + kp.APIServerPort = uint(port) } registry, err := kp.runRegistry(ctx) @@ -94,7 +94,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc for x := 0; x < int(kp.Nodes); x++ { nodeName := kp.nodeNameFromIndex(x + 1) - sshClient, err := libssh.NewSSHClient(kp.SSHPort, x+1, false) + sshClient, err := libssh.NewSSHClient(uint16(kp.SSHPort), x+1, false) if err != nil { return err } @@ -161,7 +161,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc if err = rootkey.Exec(); err != nil { return err } - sshClient, err = libssh.NewSSHClient(kp.SSHPort, x+1, true) + sshClient, err = libssh.NewSSHClient(uint16(kp.SSHPort), x+1, true) if err = kp.provisionNode(sshClient, x+1); err != nil { return err @@ -173,7 +173,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc }(node.ID) } - sshClient, err := libssh.NewSSHClient(kp.SSHPort, 1, true) + sshClient, err := libssh.NewSSHClient(uint16(kp.SSHPort), 1, true) if err != nil { return err } @@ -188,7 +188,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc return err } - config, err := k8s.NewConfig(".kubeconfig", kp.APIServerPort) + config, err := k8s.NewConfig(".kubeconfig", uint16(kp.APIServerPort)) if err != nil { return err } diff --git a/cluster-provision/gocli/providers/types.go b/cluster-provision/gocli/providers/types.go index fbb419e87d..8da12d58d5 100644 --- a/cluster-provision/gocli/providers/types.go +++ b/cluster-provision/gocli/providers/types.go @@ -23,17 +23,17 @@ type KubevirtProvider struct { Background bool `flag:"background" short:"b" json:"background"` RandomPorts bool `flag:"random-ports" json:"random_ports"` Slim bool `flag:"slim" json:"slim"` - VNCPort uint16 `flag:"vnc-port" json:"vnc_port"` - HTTPPort uint16 `flag:"http-port" json:"http_port"` - HTTPSPort uint16 `flag:"https-port" json:"https_port"` - RegistryPort uint16 `flag:"registry-port" json:"registry_port"` - OCPort uint16 `flag:"ocp-port" json:"ocp_port"` - K8sPort uint16 `flag:"k8s-port" json:"k8s_port"` - SSHPort uint16 `flag:"ssh-port" json:"ssh_port"` - PrometheusPort uint16 `flag:"prometheus-port" json:"prometheus_port"` - GrafanaPort uint16 `flag:"grafana-port" json:"grafana_port"` - DNSPort uint16 `flag:"dns-port" json:"dns_port"` - APIServerPort uint16 `json:"api_server_port"` + VNCPort uint `flag:"vnc-port" json:"vnc_port"` + HTTPPort uint `flag:"http-port" json:"http_port"` + HTTPSPort uint `flag:"https-port" json:"https_port"` + RegistryPort uint `flag:"registry-port" json:"registry_port"` + OCPort uint `flag:"ocp-port" json:"ocp_port"` + K8sPort uint `flag:"k8s-port" json:"k8s_port"` + SSHPort uint `flag:"ssh-port" json:"ssh_port"` + PrometheusPort uint `flag:"prometheus-port" json:"prometheus_port"` + GrafanaPort uint `flag:"grafana-port" json:"grafana_port"` + DNSPort uint `flag:"dns-port" json:"dns_port"` + APIServerPort uint `json:"api_server_port"` NFSData string `flag:"nfs-data" json:"nfs_data"` EnableCeph bool `flag:"enable-ceph" json:"enable_ceph"` EnableIstio bool `flag:"enable-istio" json:"enable_istio"` From c68122c096efe2fc2b8f4d973b61467b86d990e8 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Mon, 30 Sep 2024 18:47:25 +0300 Subject: [PATCH 09/69] feat: CRI package Signed-off-by: aerosouund --- cluster-provision/gocli/cri/docker/docker.go | 135 ++++++++++++ cluster-provision/gocli/cri/main.go | 23 ++ cluster-provision/gocli/cri/podman/podman.go | 212 +++++++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 cluster-provision/gocli/cri/docker/docker.go create mode 100644 cluster-provision/gocli/cri/main.go create mode 100644 cluster-provision/gocli/cri/podman/podman.go diff --git a/cluster-provision/gocli/cri/docker/docker.go b/cluster-provision/gocli/cri/docker/docker.go new file mode 100644 index 0000000000..c45a1b0fc1 --- /dev/null +++ b/cluster-provision/gocli/cri/docker/docker.go @@ -0,0 +1,135 @@ +package docker + +import ( + "os" + "os/exec" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +) + +type DockerClient struct{} + +func NewDockerClient() *DockerClient { + return &DockerClient{} +} + +func IsAvailable() bool { + cmd := exec.Command("docker", "-v") + out, err := cmd.Output() + if err != nil { + return false + } + return strings.HasPrefix(string(out), "Docker version") +} + +func (dc *DockerClient) ImagePull(image string) error { + cmd := exec.Command("docker", "pull", image) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +func (dc *DockerClient) Inspect(containerID, format string) ([]byte, error) { + cmd := exec.Command("docker", "inspect", containerID, "--format", format) + out, err := cmd.Output() + if err != nil { + return nil, err + } + return out, nil +} + +func (dc *DockerClient) Start(containerID string) error { + cmd := exec.Command("docker", + "start", + containerID) + + if _, err := cmd.CombinedOutput(); err != nil { + return err + } + return nil +} + +func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string, error) { + ports := "" + for containerPort, hostPort := range createOpts.Ports { + ports += "-p " + containerPort + ":" + hostPort + } + + args := []string{ + "--name=" + createOpts.Name, + "--privileged=" + strconv.FormatBool(createOpts.Privileged), + "--rm=" + strconv.FormatBool(createOpts.Remove), + "--restart=" + createOpts.RestartPolicy, + "--network=" + createOpts.Network, + } + + for containerPort, hostPort := range createOpts.Ports { + args = append(args, "-p", containerPort+":"+hostPort) + } + + if len(createOpts.Capabilities) > 0 { + args = append(args, "--cap-add="+strings.Join(createOpts.Capabilities, ",")) + } + + fullArgs := append([]string{"create"}, args...) + fullArgs = append(fullArgs, image) + fullArgs = append(fullArgs, createOpts.Command...) + + cmd := exec.Command("docker", + fullArgs..., + ) + + containerID, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + logrus.Info("created registry container with id: ", string(containerID)) + return strings.TrimSuffix(string(containerID), "\n"), nil +} + +func (dc *DockerClient) Remove(containerID string) error { + cmd := exec.Command("docker", "rm", "-f", containerID) + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (dc *DockerClient) Build(tag, containerFile string, buildArgs map[string]string) error { + args := []string{"build", "--pull=false", "-t", tag} + + for k, v := range buildArgs { + args = append(args, "--build-arg "+k+"="+v) + } + + args = append(args, []string{"-f", containerFile, "."}...) + cmd := exec.Command("docker", args...) + logrus.Info("Running: ", cmd.String()) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (dc *DockerClient) Run(runArgs []string) error { + runArgs = append([]string{"run"}, runArgs...) + cmd := exec.Command("docker", runArgs...) + logrus.Info("Running: ", cmd.String()) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return err + } + return nil +} diff --git a/cluster-provision/gocli/cri/main.go b/cluster-provision/gocli/cri/main.go new file mode 100644 index 0000000000..95cb4d84f3 --- /dev/null +++ b/cluster-provision/gocli/cri/main.go @@ -0,0 +1,23 @@ +package cri + +// maybe just create wrappers around bash after all +type ContainerClient interface { + ImagePull(image string) error + Create(image string, co *CreateOpts) (string, error) + Start(containerID string) error + Remove(containerID string) error + Inspect(containerID, format string) ([]byte, error) + Build(tag, containerFile string, buildArgs map[string]string) error + Run(runArgs []string) error +} + +type CreateOpts struct { + Privileged bool + Name string + Ports map[string]string + RestartPolicy string + Network string + Command []string + Remove bool + Capabilities []string +} diff --git a/cluster-provision/gocli/cri/podman/podman.go b/cluster-provision/gocli/cri/podman/podman.go new file mode 100644 index 0000000000..5ac20dc0da --- /dev/null +++ b/cluster-provision/gocli/cri/podman/podman.go @@ -0,0 +1,212 @@ +package podman + +import ( + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +) + +type Podman struct{} + +func NewPodman() *Podman { + return &Podman{} +} + +type PodmanSSHClient struct { + containerName string +} + +func NewPodmanSSHClient(containerName string) *PodmanSSHClient { + return &PodmanSSHClient{ + containerName: containerName, + } +} + +func IsAvailable() bool { + cmd := exec.Command("podman", "-v") + out, err := cmd.Output() + if err != nil { + return false + } + return strings.HasPrefix(string(out), "podman version") +} + +func (p *PodmanSSHClient) Command(cmd string, stdOut bool) (string, error) { + logrus.Infof("[node %s]: %s\n", p.containerName, cmd) + command := exec.Command("podman", "exec", p.containerName, "/bin/sh", "-c", cmd) + if !stdOut { + out, err := command.CombinedOutput() + if err != nil { + return "", err + } + return string(out), nil + } + command.Stdout = os.Stdout + command.Stderr = os.Stderr + + if err := command.Run(); err != nil { + return "", err + } + return "", nil +} + +func (p *PodmanSSHClient) CopyRemoteFile(remotePath, localPath string) error { + cmd := exec.Command("podman", "cp", fmt.Sprintf("%s:%s", p.containerName, remotePath), localPath) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy file from container: %w, output: %s", err, output) + } + + return nil +} + +func (p *PodmanSSHClient) SCP(destPath string, contents fs.File) error { + tempFile, err := os.CreateTemp("", "podman_cp_temp") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + defer os.Remove(tempFile.Name()) + + fileContents, err := io.ReadAll(contents) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + + _, err = tempFile.Write(fileContents) + if err != nil { + return fmt.Errorf("failed to write to temp file: %w", err) + } + + err = tempFile.Close() + if err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + + cmd := exec.Command("podman", "cp", tempFile.Name(), p.containerName+":"+destPath) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("podman cp command failed: %w. Output: %s", err, string(output)) + } + + return nil +} + +func (p *Podman) ImagePull(image string) error { + cmd := exec.Command("podman", "pull", image) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +func (p *Podman) Create(image string, createOpts *cri.CreateOpts) (string, error) { + ports := "" + for containerPort, hostPort := range createOpts.Ports { + ports += "-p " + containerPort + ":" + hostPort + } + + args := []string{ + "--name=" + createOpts.Name, + "--privileged=" + strconv.FormatBool(createOpts.Privileged), + "--rm=" + strconv.FormatBool(createOpts.Remove), + "--restart=" + createOpts.RestartPolicy, + "--network=" + createOpts.Network, + } + + for containerPort, hostPort := range createOpts.Ports { + args = append(args, "-p", containerPort+":"+hostPort) + } + + if len(createOpts.Capabilities) > 0 { + args = append(args, "--cap-add="+strings.Join(createOpts.Capabilities, ",")) + } + + fullArgs := append([]string{"create"}, args...) + fullArgs = append(fullArgs, image) + fullArgs = append(fullArgs, createOpts.Command...) + + cmd := exec.Command("podman", + fullArgs..., + ) + fmt.Println(cmd.String()) + + containerID, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + logrus.Info("created registry container with id: ", string(containerID)) + return strings.TrimSuffix(string(containerID), "\n"), nil +} + +func (p *Podman) Start(containerID string) error { + cmd := exec.Command("podman", + "start", + containerID) + + if _, err := cmd.CombinedOutput(); err != nil { + return err + } + return nil +} + +func (p *Podman) Inspect(containerID, format string) ([]byte, error) { + cmd := exec.Command("podman", "inspect", containerID, "--format", format) + out, err := cmd.Output() + if err != nil { + return nil, err + } + return out, nil +} + +func (p *Podman) Remove(containerID string) error { + cmd := exec.Command("podman", "rm", "-f", containerID) + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (dc *Podman) Build(tag, containerFile string, buildArgs map[string]string) error { + args := []string{"build", "-t", tag} + + for k, v := range buildArgs { + args = append(args, "--build-arg", k+"="+v) + } + + args = append(args, []string{"-f", containerFile, "."}...) + + cmd := exec.Command("podman", args...) + logrus.Info("Running: ", cmd.String()) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (dc *Podman) Run(runArgs []string) error { + runArgs = append([]string{"run"}, runArgs...) + cmd := exec.Command("podman", runArgs...) + logrus.Info("Running: ", cmd.String()) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return err + } + return nil +} From 0bd7601f6e717012ac7f2f6d24ac853735fbb6e7 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 1 Oct 2024 15:17:10 +0300 Subject: [PATCH 10/69] control plane stuff --- cluster-provision/gocli/control-plane/cmd.go | 68 ++++++ .../control-plane/config/cluster-info.yaml | 17 ++ .../control-plane/config/kube-proxy.yaml | 196 ++++++++++++++++++ .../control-plane/config/kubelet-kubeadm.yaml | 84 ++++++++ .../gocli/control-plane/config/rbac.yaml | 183 ++++++++++++++++ .../gocli/control-plane/config/token.yaml | 12 ++ .../control-plane/control-plane-components.go | 127 ++++++++++++ cluster-provision/gocli/control-plane/etcd.go | 51 +++++ cluster-provision/gocli/control-plane/main.go | 42 ++++ cluster-provision/gocli/cri/docker/docker.go | 4 + cluster-provision/gocli/cri/main.go | 1 + 11 files changed, 785 insertions(+) create mode 100644 cluster-provision/gocli/control-plane/cmd.go create mode 100644 cluster-provision/gocli/control-plane/config/cluster-info.yaml create mode 100644 cluster-provision/gocli/control-plane/config/kube-proxy.yaml create mode 100644 cluster-provision/gocli/control-plane/config/kubelet-kubeadm.yaml create mode 100644 cluster-provision/gocli/control-plane/config/rbac.yaml create mode 100644 cluster-provision/gocli/control-plane/config/token.yaml create mode 100644 cluster-provision/gocli/control-plane/control-plane-components.go create mode 100644 cluster-provision/gocli/control-plane/etcd.go create mode 100644 cluster-provision/gocli/control-plane/main.go diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go new file mode 100644 index 0000000000..3ed49ded89 --- /dev/null +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -0,0 +1,68 @@ +package controlplane + +func buildEtcdCmdArgs() map[string]string { + return map[string]string{ + "--advertise-client-urls": "https://0.0.0.0:2379", + "--cert-file": "/etc/kubernetes/pki/etcd/kube-api-server.crt", + "--client-cert-auth": "true", + "--data-dir": "/var/lib/etcd", + "--experimental-initial-corrupt-check": "true", + "--experimental-watch-progress-notify-interval": "5s", + "--initial-advertise-peer-urls": "https://0.0.0.0:2380", + "--initial-cluster": "node01=https://0.0.0.0:2380", + "--key-file": "/etc/kubernetes/pki/etcd/kube-api-server.key", + "--listen-client-urls": "https://0.0.0.0:2379", + "--listen-metrics-urls": "http://0.0.0.0:2381", + "--name": "node01", + "--snapshot-count": "10000", + "--trusted-ca-file": "/etc/kubernetes/pki/etcd/ca.crt", + } +} + +func buildControllerMgrCmdArgs() map[string]string { + return map[string]string{ + "--allocate-node-cidrs": "true", + "--authorization-kubeconfig": "/etc/kubernetes/controller-manager.conf", + "--authentication-kubeconfig": "/etc/kubernetes/controller-manager.conf", + "--bind-address": "127.0.0.1", + "--cluster-cidr": "10.244.0.0/16,fd10:244::/112", + "--cluster-name": "kubernetes", + "--cluster-signing-cert-file": "/etc/kubernetes/pki/ca.crt", + "--cluster-signing-key-file": "/etc/kubernetes/pki/ca.key", + "--controllers": "*,csrapproving,csrsigning,bootstrapsigner,tokencleaner", + "--kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", + "--node-cidr-mask-size-ipv6": "116", + "--leader-elect": "true", + "-v": "5", + "--root-ca-file": "/etc/kubernetes/pki/ca.crt", + "--service-account-private-key-file": "/etc/kubernetes/pki/service-accounts.key", + "--service-cluster-ip-range": "10.96.0.0/12,fd10:96::/108", + "--use-service-account-credentials": "true", + } +} + +func buildApiServerCmdArgs() map[string]string { + return map[string]string{ + "--advertise-address": "192.168.66.110", + "--allow-privileged": "true", + "--audit-log-format": "json", + "--audit-log-path": "/var/log/k8s-audit/k8s-audit.log", + "--authorization-mode": "Node,RBAC", + "--client-ca-file": "/etc/kubernetes/pki/ca.crt", + "--enable-admission-plugins": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--enable-bootstrap-token-auth": "true", + "--etcd-cafile": "/etc/kubernetes/pki/ca.crt", + "--etcd-certfile": "/etc/kubernetes/pki/kube-api-server.crt", + "--etcd-keyfile": "/etc/kubernetes/pki/kube-api-server.key", + "--etcd-servers": "https://127.0.0.1:2379", + "--kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + "--secure-port": "6443", + "--v": "3", + "--service-account-issuer": "https://kubernetes.default.svc.cluster.local", + "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.key", + "--service-account-signing-key-file": " /etc/kubernetes/pki/service-accounts.key", + "--service-cluster-ip-range": "10.96.0.0/24", + "--tls-cert-file": "/etc/kubernetes/pki/kube-api-server.crt", + "--tls-private-key-file": "/etc/kubernetes/pki/kube-api-server.key", + } +} diff --git a/cluster-provision/gocli/control-plane/config/cluster-info.yaml b/cluster-provision/gocli/control-plane/config/cluster-info.yaml new file mode 100644 index 0000000000..7aa5226e61 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/cluster-info.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-info + namespace: kube-public +data: + kubeconfig: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZURENDQXpTZ0F3SUJBZ0lVVUxKYkdXeHI5cGxiZEIwMnh4U3N6UGVMNXlvd0RRWUpLb1pJaHZjTkFRRU4KQlFBd1FURUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2xkaGMyaHBibWQwYjI0eEVEQU9CZ05WQkFjTQpCMU5sWVhSMGJHVXhDekFKQmdOVkJBTU1Ba05CTUI0WERUSTBNRGt3TnpFeU16WTFObG9YRFRNME1Ea3dPREV5Ck16WTFObG93UVRFTE1Ba0dBMVVFQmhNQ1ZWTXhFekFSQmdOVkJBZ01DbGRoYzJocGJtZDBiMjR4RURBT0JnTlYKQkFjTUIxTmxZWFIwYkdVeEN6QUpCZ05WQkFNTUFrTkJNSUlDSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQWc4QQpNSUlDQ2dLQ0FnRUFzVDQzSFNqWDY1WFlIVVdqVkdhU0RpUlBBOU9maWM1NTNxaGhQa2lsR2dsblNsR21WYUcrCittWFBnS2E0WWUyUDRuTkRydzFGM1VlUEZ5Z0dxZE4rbnZhRmFDZmlXR1dCUUxzNFV1Nm1nZXJYdnJRaVFzZDMKVnlrc21MWUxNZlF6M0cyK3Jvbk9uQ056UTd1ZEdkTS94ckYzaEN1MWF3Q2xzMUlrMDQ5YUIzWUZQSnc5WHdFNQpqYy9YcGlqbXJGVVFZOGh1UHI2aWpWVGJPVXpzcmNJNmJqeDhtcDdGU0dQQ2ZCUGhPaUFJekRLcmlXOEFjaCtXCkJER0RsRGMxV01qTEdLVVorVUxjUFlzcVVTKzlrWUNxQmJtSUQvYkdRdWdXZm8vSUpkOFVCVmkwaStXVkhJeUsKNVY0QUJzakdYT3hRbHk1Y3BucDM0bUp6blAzRzF4bWIwMlRtcm1uZjdqVDJnQm1sS2F5TWZQMjJCeFFqL3pWZQpUeVJTOWs5dDk5dDBRTDhwNytla1BkekYwTFpnMXVhTU1iZEZaWWVSRjlLd3NNTzRiaU1sSWFMb21wQ2ZndFlZClZ1YUVqSnpLY0pERW84SXAvZTJJazRzaUE2SEdJRE0vWFpVTGVFaDR6NCt0V3pxQUE2MUlJNXM2U0c1OG5iN08KN1VXWXJiK3NRaFZkaTR4cVE5b3ZFRFF5L1pHTUVsbkNPcDF1NUFLTFZDQ2w2U1RIdWNXNnR5UEN4VWdqWm9GQwpOb096NUwvdXQ2YmZOdDJmZVB2a3JEejcyUVVWWG1tWmJmODYrLyszOFZPOCttdTNHanBXK1NoVFUwaW91NUpzCjlHd0dsZEJTNlptelN5blQ0ZXNSa0xWWkdVTkxZMGRRaG9BZ0w0VDg2TElRZ2N3WU1ZNnhVdXNDQXdFQUFhTTgKTURvd0RBWURWUjBUQkFVd0F3RUIvekFMQmdOVkhROEVCQU1DQVFZd0hRWURWUjBPQkJZRUZKeEl3cHFoQmQwNApBclV1MzIvLzJHV3M2ZWFjTUEwR0NTcUdTSWIzRFFFQkRRVUFBNElDQVFBa3pRR1ZWcXFWT3Q4NjRxblU5MFZLCnlGREM0eG4zcTdUeDVZYVV4SU1qQTF5Q2hlSjAzQ3c1QnNMT0VHdXc0Rmw5NWJHVU9ldzVieGp0L29lNGV6WEoKY09hWC9xd0FCRGNIbjBFSlNPczc0K3h0NkZhWnphYjNVaFphVXlSZ2FnQUVqL2RPaTZKYzZVSHZJbXVWbHVYTQpGWnp2dk1oSWJuak1tSmJ1N2xlenRXTE5BMWEybnM2d3M1LzdCdzdtQm1DUFhycDdaUzNqaWxSLzA1UjNRaTNUCkNzVGFMeGZxdU14WEJuZVdUSmRmWTBxZ1VBZGNhMlp6UTBDa2dyaXdienlCRldPU25TRGdNQzBNZjhsd3FGaGsKY3FybUk3bGlSZHd6RVQ5ekQ5aGFBVkpkRDdMVWRRMzFvbWdoVWp6cUFpeWZWU2t3dE5mczlkeExhWC9JTURxaQowRVVXR2ZEZHFyMm9xNExDTW4zNVQ3c1pLaDBvZ3MrdXA5UVVPYzY3UmNpcXdoQkwrOCs4Q0NHNlE3Q0E2TkNaCkk0cGQ1L1AwK0h6N0c2RlVoRWl4azJUbEZvd3hEdUFtd0V6T0ZUcG5GcmpjUVFyNlEvWDBYcTdYZFFQSzhhTUUKTmFNY1dTdCsyS1ZSbHdVTzRJOHRxYW05SkpqOGRPRHh6cVJNcnBNaUM0SkkvSGtvb0xkaDNxMkhSdUd1NUFQQgpXMGxheW5CejcyYUVzVkQycVhNQVZHaHNhdlVMNDlYdm8vY2hWcXpFY2t1QzRFVDcvQ2ZCakNTNTcxUUROL1Q0CkZGMThhZkh0eWNQbkF4TUNoTkpIZEVJZTBvbFg4eDJxcHdlRUg1eUpiYXk2U21od2NVU2NUUUZVUStISTBsQzUKY3Zjb2NpZ1g2elRhU2dKOVBVQWV1QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://192.168.66.110:6443 + contexts: null + current-context: "" + kind: Config + preferences: {} + users: null \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml new file mode 100644 index 0000000000..7db931bda0 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml @@ -0,0 +1,196 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-proxy + namespace: default + labels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy +automountServiceAccountToken: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-proxy + namespace: default + labels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy +data: + config: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + bindAddress: 0.0.0.0 + clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig + qps: 5 + clusterCIDR: "" + configSyncPeriod: 15m0s + conntrack: + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s + enableProfiling: false + healthzBindAddress: 0.0.0.0:10256 + hostnameOverride: "" + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0:10205 + mode: iptables + nodePortAddresses: null + oomScoreAdj: -999 + portRange: "" + kubeconfig: |- + kind: Config + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://192.168.66.110:6443 + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-proxy + labels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier +subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: default +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-proxy + namespace: default + labels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy +spec: + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 10% + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy + template: + metadata: + labels: + app.kubernetes.io/name: kube-proxy + app.kubernetes.io/instance: kube-proxy + annotations: + checksum/config: 56973c65a78d5adeeaa448c828c1e8aeb3b5663add4ba782e87c6605f33b3e47 + spec: + serviceAccountName: kube-proxy + securityContext: + runAsNonRoot: false + priorityClassName: "system-node-critical" + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirst + containers: + - name: kube-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - SYS_RESOURCE + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: false + image: registry.k8s.io/kube-proxy:v1.28.0 + imagePullPolicy: IfNotPresent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + command: + - kube-proxy + - --v=2 + - --config=/var/lib/kube-proxy/config + - --hostname-override=$(NODE_NAME) + ports: + - name: http-health + containerPort: 10256 + protocol: TCP + - name: http-metrics + containerPort: 10249 + protocol: TCP + livenessProbe: + httpGet: + path: /livez + port: http-health + readinessProbe: + httpGet: + path: /healthz + port: http-health + volumeMounts: + - mountPath: /var/lib/kube-proxy/ + name: config + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: false + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: config + configMap: + name: kube-proxy + - name: varlog + hostPath: + path: /var/log + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: lib-modules + hostPath: + path: /lib/modules + nodeSelector: + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/config/kubelet-kubeadm.yaml b/cluster-provision/gocli/control-plane/config/kubelet-kubeadm.yaml new file mode 100644 index 0000000000..e190307d81 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/kubelet-kubeadm.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +data: + ClusterConfiguration: | + apiServer: + extraArgs: + allow-privileged: "true" + audit-log-format: json + authorization-mode: Node,RBAC + enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota + timeoutForControlPlane: 4m0s + apiVersion: kubeadm.k8s.io/v1beta3 + certificatesDir: /etc/kubernetes/pki + clusterName: kubernetes + controllerManager: + extraArgs: + node-cidr-mask-size-ipv6: "116" + dns: {} + imageRepository: registry.k8s.io + kind: ClusterConfiguration + kubernetesVersion: v1.30.0-alpha.2 + networking: + dnsDomain: cluster.local + podSubnet: 10.244.0.0/16 + serviceSubnet: 10.96.0.0/12 + scheduler: {} +kind: ConfigMap +metadata: + name: kubeadm-config + namespace: kube-system +--- +apiVersion: v1 +data: + kubelet: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: true + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.96.0.10 + clusterDomain: cluster.local + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionPressureTransitionPeriod: 0s + fileCheckFrequency: 0s + healthzBindAddress: 127.0.0.1 + healthzPort: 10248 + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + rotateCertificates: true + runtimeRequestTimeout: 0s + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 0s +kind: ConfigMap +metadata: + annotations: + kubeadm.kubernetes.io/component-config.hash: sha256:70279caa991cfe4947cff54a2d72b50e2e8cdeb8479a5a413eca528ccc04e376 + name: kubelet-config + namespace: kube-system \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/config/rbac.yaml b/cluster-provision/gocli/control-plane/config/rbac.yaml new file mode 100644 index 0000000000..23fd5f20b5 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/rbac.yaml @@ -0,0 +1,183 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:nodes-kubeadm-config + namespace: kube-system +rules: +- apiGroups: + - "" + resourceNames: + - kubeadm-config + resources: + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:kubelet-config + namespace: kube-system +rules: +- apiGroups: + - "" + resourceNames: + - kubelet-config + resources: + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:bootstrap-signer-clusterinfo + namespace: kube-public +rules: +- apiGroups: + - "" + resourceNames: + - cluster-info + resources: + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeadm:get-nodes +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:cluster-admins +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: kubeadm:cluster-admins +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:get-nodes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeadm:get-nodes +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:kubelet-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:node-autoapprove-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:nodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:node-autoapprove-certificate-rotation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:node-proxier +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier +subjects: +- kind: ServiceAccount + name: kube-proxy + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeadm:bootstrap-signer-clusterinfo + namespace: kube-public +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:bootstrap-signer-clusterinfo +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeadm:kubelet-config + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:kubelet-config +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeadm:nodes-kubeadm-config + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:nodes-kubeadm-config +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/config/token.yaml b/cluster-provision/gocli/control-plane/config/token.yaml new file mode 100644 index 0000000000..4e3c806c28 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/token.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +data: + auth-extra-groups: c3lzdGVtOmJvb3RzdHJhcHBlcnM6a3ViZWFkbTpkZWZhdWx0LW5vZGUtdG9rZW4= + token-id: YWJjZGVm + token-secret: MTIzNDU2Nzg5MDEyMzQ1Ng== + usage-bootstrap-authentication: dHJ1ZQ== + usage-bootstrap-signing: dHJ1ZQ== +kind: Secret +metadata: + name: bootstrap-token-abcdef + namespace: kube-system +type: bootstrap.kubernetes.io/token \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go new file mode 100644 index 0000000000..0aa73f3ff5 --- /dev/null +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -0,0 +1,127 @@ +package controlplane + +import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" + +type RunControlPlaneComponentsPhase struct { + dnsmasqID string + k8sVersion string + containerRuntime cri.ContainerClient +} + +func NewRunControlPlaneComponentsPhase(dnsmasqID string, containerRuntime cri.ContainerClient) *RunControlPlaneComponentsPhase { + return &RunControlPlaneComponentsPhase{ + dnsmasqID: dnsmasqID, + containerRuntime: containerRuntime, + } +} + +func (p *RunControlPlaneComponentsPhase) Run() error { + componentFuncs := []func() error{p.runApiServer, p.runControllerMgr, p.runScheduler} + for _, componentFunc := range componentFuncs { + err := componentFunc() + if err != nil { + return err + } + } + return nil +} + +func (p *RunControlPlaneComponentsPhase) runApiServer() error { + apiServerImage := registry + "/" + apiServer + ":" + p.k8sVersion + err := p.containerRuntime.ImagePull(apiServerImage) + if err != nil { + return err + } + + args := buildApiServerCmdArgs() + + cmd := []string{"kube-apiserver"} + for flag, values := range args { + cmd = append(cmd, flag+"="+values) + } + + createOpts := &cri.CreateOpts{ + Name: "api-server", + Mounts: map[string]string{ + "": "", // TODO + }, + Network: "container:" + p.dnsmasqID, + Command: cmd, + } + + apiserverContainer, err := p.containerRuntime.Create(apiServerImage, createOpts) + if err != nil { + return err + } + + err = p.containerRuntime.Start(apiserverContainer) + if err != nil { + return err + } + return nil +} + +func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { + ctrlMgrImage := registry + "/" + controllerManager + ":" + p.k8sVersion + err := p.containerRuntime.ImagePull(ctrlMgrImage) + if err != nil { + return err + } + + args := buildControllerMgrCmdArgs() + + cmd := []string{"kube-controller-manager"} + for flag, values := range args { + cmd = append(cmd, flag+"="+values) + } + + createOpts := &cri.CreateOpts{ + Name: "kube-controller-manager", + Mounts: map[string]string{ + "": "", // TODO + }, + Network: "container:" + p.dnsmasqID, + Command: cmd, + } + + apiserverContainer, err := p.containerRuntime.Create(ctrlMgrImage, createOpts) + if err != nil { + return err + } + + err = p.containerRuntime.Start(apiserverContainer) + if err != nil { + return err + } + return nil +} + +func (p *RunControlPlaneComponentsPhase) runScheduler() error { + schedulerImage := registry + "/" + scheduler + ":" + p.k8sVersion + err := p.containerRuntime.ImagePull(schedulerImage) + if err != nil { + return err + } + + cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/scheduler.conf"} + + createOpts := &cri.CreateOpts{ + Name: "kube-scheduler", + Mounts: map[string]string{ + "": "", // TODO + }, + Network: "container:" + p.dnsmasqID, + Command: cmd, + } + + apiserverContainer, err := p.containerRuntime.Create(schedulerImage, createOpts) + if err != nil { + return err + } + + err = p.containerRuntime.Start(apiserverContainer) + if err != nil { + return err + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/etcd.go b/cluster-provision/gocli/control-plane/etcd.go new file mode 100644 index 0000000000..970d3cd7b8 --- /dev/null +++ b/cluster-provision/gocli/control-plane/etcd.go @@ -0,0 +1,51 @@ +package controlplane + +import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" + +type RunETCDPhase struct { + dnsmasqID string + containerRuntime cri.ContainerClient +} + +func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient) *RunETCDPhase { + return &RunETCDPhase{ + dnsmasqID: dnsmasqID, + containerRuntime: containerRuntime, + } +} + +func (p *RunETCDPhase) Run() error { + etcdImageRepo := registry + "/" + etcdImage + err := p.containerRuntime.ImagePull(etcdImageRepo) + if err != nil { + return err + } + + etcdCmd := []string{"etcd"} + args := buildEtcdCmdArgs() + + for flag, value := range args { + etcdCmd = append(etcdCmd, flag+"="+value) + } + + createOpts := &cri.CreateOpts{ + Name: "etcd", + Mounts: map[string]string{ + "/etcdcert": "/etc/kubernetes/pki/etcd", //TODO + }, + Network: "container:" + p.dnsmasqID, + Command: etcdCmd, + } + + etcdContainer, err := p.containerRuntime.Create(etcdImageRepo, createOpts) + if err != nil { + return err + } + + err = p.containerRuntime.Start(etcdContainer) + if err != nil { + return err + } + + return nil +} diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go new file mode 100644 index 0000000000..bfd58da56f --- /dev/null +++ b/cluster-provision/gocli/control-plane/main.go @@ -0,0 +1,42 @@ +package controlplane + +import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" + +const ( + etcdImage = "etcd:3.5.10-0" + apiServer = "kube-apiserver" + controllerManager = "kube-controller-manager" + scheduler = "kube-scheduler" + registry = "registry.k8s.io" +) + +type ControlPlane interface{} + +type ControlPlaneRunner struct { + Phases []Phase +} + +type Phase interface { + Run() error +} + +func NewControlPlaneRunner(containerRuntime cri.ContainerClient) *ControlPlaneRunner { + phases := []Phase{} + + phases = append(phases, NewRunETCDPhase("", containerRuntime)) + phases = append(phases, NewRunControlPlaneComponentsPhase("", containerRuntime)) + + return &ControlPlaneRunner{ + Phases: phases, + } +} + +func (cp *ControlPlaneRunner) Start() error { + for _, phase := range cp.Phases { + err := phase.Run() + if err != nil { + return err + } + } + return nil +} diff --git a/cluster-provision/gocli/cri/docker/docker.go b/cluster-provision/gocli/cri/docker/docker.go index c45a1b0fc1..43cd814e87 100644 --- a/cluster-provision/gocli/cri/docker/docker.go +++ b/cluster-provision/gocli/cri/docker/docker.go @@ -60,6 +60,10 @@ func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string ports += "-p " + containerPort + ":" + hostPort } + for hostFile, containerFile := range createOpts.Mounts { + ports += "-v " + hostFile + ":" + containerFile + } + args := []string{ "--name=" + createOpts.Name, "--privileged=" + strconv.FormatBool(createOpts.Privileged), diff --git a/cluster-provision/gocli/cri/main.go b/cluster-provision/gocli/cri/main.go index 95cb4d84f3..5d8982dec1 100644 --- a/cluster-provision/gocli/cri/main.go +++ b/cluster-provision/gocli/cri/main.go @@ -13,6 +13,7 @@ type ContainerClient interface { type CreateOpts struct { Privileged bool + Mounts map[string]string Name string Ports map[string]string RestartPolicy string From c22baab9590a882f2b693196dedce086ce4a308a Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 1 Oct 2024 15:57:39 +0300 Subject: [PATCH 11/69] control plane stuff --- .../gocli/control-plane/bootstrap.go | 44 +++++++++++++++++++ cluster-provision/gocli/control-plane/etcd.go | 2 +- .../gocli/control-plane/kube-proxy.go | 36 +++++++++++++++ cluster-provision/gocli/control-plane/main.go | 26 ++++------- cluster-provision/gocli/control-plane/rbac.go | 42 ++++++++++++++++++ 5 files changed, 132 insertions(+), 18 deletions(-) create mode 100644 cluster-provision/gocli/control-plane/bootstrap.go create mode 100644 cluster-provision/gocli/control-plane/kube-proxy.go create mode 100644 cluster-provision/gocli/control-plane/rbac.go diff --git a/cluster-provision/gocli/control-plane/bootstrap.go b/cluster-provision/gocli/control-plane/bootstrap.go new file mode 100644 index 0000000000..1bf51bc032 --- /dev/null +++ b/cluster-provision/gocli/control-plane/bootstrap.go @@ -0,0 +1,44 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/token.yaml +var token []byte + +//go:embed config/kubelet-kubeadm.yaml +var kk []byte + +//go:embed config/cluster-info.yaml +var clusterInfo []byte + +type BootstrapAuthResourcesPhase struct { + client k8s.K8sDynamicClient +} + +func (p *BootstrapAuthResourcesPhase) Run() error { + for _, manifest := range [][]byte{token, kk, clusterInfo} { + yamlDocs := bytes.Split(manifest, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := p.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/etcd.go b/cluster-provision/gocli/control-plane/etcd.go index 970d3cd7b8..ae4885d973 100644 --- a/cluster-provision/gocli/control-plane/etcd.go +++ b/cluster-provision/gocli/control-plane/etcd.go @@ -7,7 +7,7 @@ type RunETCDPhase struct { containerRuntime cri.ContainerClient } -func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient) *RunETCDPhase { +func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient) Phase { return &RunETCDPhase{ dnsmasqID: dnsmasqID, containerRuntime: containerRuntime, diff --git a/cluster-provision/gocli/control-plane/kube-proxy.go b/cluster-provision/gocli/control-plane/kube-proxy.go new file mode 100644 index 0000000000..e7906c7454 --- /dev/null +++ b/cluster-provision/gocli/control-plane/kube-proxy.go @@ -0,0 +1,36 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/kube-proxy.yaml +var kp []byte + +type KubeProxyPhase struct { + client k8s.K8sDynamicClient +} + +func (p *KubeProxyPhase) Run() error { + yamlDocs := bytes.Split(kp, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := p.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index bfd58da56f..88d0fd0c7f 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -10,33 +10,25 @@ const ( registry = "registry.k8s.io" ) -type ControlPlane interface{} - type ControlPlaneRunner struct { - Phases []Phase + dnsmasqID string + containerRuntime cri.ContainerClient } type Phase interface { Run() error } -func NewControlPlaneRunner(containerRuntime cri.ContainerClient) *ControlPlaneRunner { - phases := []Phase{} - - phases = append(phases, NewRunETCDPhase("", containerRuntime)) - phases = append(phases, NewRunControlPlaneComponentsPhase("", containerRuntime)) +func NewControlPlaneRunner() {} - return &ControlPlaneRunner{ - Phases: phases, +func (cp *ControlPlaneRunner) Start() error { + if err := NewRunETCDPhase(cp.dnsmasqID, cp.containerRuntime).Run(); err != nil { + return err } -} -func (cp *ControlPlaneRunner) Start() error { - for _, phase := range cp.Phases { - err := phase.Run() - if err != nil { - return err - } + if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime).Run(); err != nil { + return err } + return nil } diff --git a/cluster-provision/gocli/control-plane/rbac.go b/cluster-provision/gocli/control-plane/rbac.go new file mode 100644 index 0000000000..5a764c1a86 --- /dev/null +++ b/cluster-provision/gocli/control-plane/rbac.go @@ -0,0 +1,42 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/rbac.yaml +var rbac []byte + +type CreateBootstrappersRBACPhase struct { + client k8s.K8sDynamicClient +} + +func NewCreateBootstrappersRBACPhase(client k8s.K8sDynamicClient) *CreateBootstrappersRBACPhase { + return &CreateBootstrappersRBACPhase{ + client: client, + } +} + +func (p *CreateBootstrappersRBACPhase) Run() error { + yamlDocs := bytes.Split(rbac, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := p.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + return nil +} From d48370861de09d46cecd5cc808ce5d8d1cbc5f86 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sun, 6 Oct 2024 19:56:37 +0300 Subject: [PATCH 12/69] more control plane stuff Signed-off-by: aerosouund --- .../gocli/control-plane/certs.go | 43 ++++++ .../gocli/control-plane/certs/certlist.go | 123 ++++++++++++++++++ .../gocli/control-plane/certs/constants.go | 9 ++ cluster-provision/gocli/control-plane/cmd.go | 4 +- .../control-plane/control-plane-components.go | 12 +- .../gocli/control-plane/crypto/ca.go | 83 ++++++++++++ cluster-provision/gocli/control-plane/etcd.go | 6 +- .../gocli/control-plane/kube-proxy.go | 6 + .../gocli/control-plane/kubeconfig.go | 78 +++++++++++ cluster-provision/gocli/control-plane/main.go | 92 +++++++++++-- cluster-provision/gocli/go.mod | 4 +- cluster-provision/gocli/providers/start.go | 66 +++++++--- 12 files changed, 492 insertions(+), 34 deletions(-) create mode 100644 cluster-provision/gocli/control-plane/certs.go create mode 100644 cluster-provision/gocli/control-plane/certs/certlist.go create mode 100644 cluster-provision/gocli/control-plane/certs/constants.go create mode 100644 cluster-provision/gocli/control-plane/crypto/ca.go create mode 100644 cluster-provision/gocli/control-plane/kubeconfig.go diff --git a/cluster-provision/gocli/control-plane/certs.go b/cluster-provision/gocli/control-plane/certs.go new file mode 100644 index 0000000000..744053ec25 --- /dev/null +++ b/cluster-provision/gocli/control-plane/certs.go @@ -0,0 +1,43 @@ +package controlplane + +import ( + "path" + + "kubevirt.io/kubevirtci/cluster-provision/gocli/control-plane/certs" + "kubevirt.io/kubevirtci/cluster-provision/gocli/control-plane/crypto" +) + +type CertsPhase struct { + pkiPath string +} + +func NewCertsPhase(pkiPath string) *CertsPhase { + return &CertsPhase{ + pkiPath: pkiPath, + } +} + +func (p *CertsPhase) Run() error { + componentCerts := certs.GetAllCertificates() + ca, caKey, err := crypto.GenerateKubernetesCAKeyPair() + if err != nil { + return err + } + + err = crypto.WriteKeyAndCertToFile(ca, caKey, path.Join(p.pkiPath, "ca.crt"), path.Join(p.pkiPath, "key.pem")) + if err != nil { + return err + } + + for _, componentCert := range componentCerts { + cert, key, err := crypto.GenerateCertKeyPairWithCA(componentCert.Config, ca, caKey) + if err != nil { + return err + } + err = crypto.WriteKeyAndCertToFile(cert, key, path.Join(p.pkiPath, componentCert.Name+".crt"), path.Join(p.pkiPath, componentCert.Name+".pem")) + if err != nil { + return err + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/certs/certlist.go b/cluster-provision/gocli/control-plane/certs/certlist.go new file mode 100644 index 0000000000..c595b14c49 --- /dev/null +++ b/cluster-provision/gocli/control-plane/certs/certlist.go @@ -0,0 +1,123 @@ +package certs + +import ( + "crypto/x509" + "net" + "time" + + "k8s.io/client-go/util/cert" +) + +type Cert struct { + Name string + LongName string + BaseName string + CAName string + Config cert.Config + NotAfter *time.Time +} + +type Certificates []*Cert + +func GetAllCertificates() Certificates { + return Certificates{ + ApiServerCert(), + SchedulerCert(), + ControllerMgrCert(), + KubeletClientCert(), + ServiceAccountsCert(), + AdminCert(), + } +} + +func ApiServerCert() *Cert { + return &Cert{ + Name: "apiserver", + LongName: "certificate for serving the Kubernetes API", + BaseName: "kube-apiserver", + CAName: "ca", + Config: cert.Config{ + CommonName: "kube-apiserver", + AltNames: cert.AltNames{ + DNSNames: ApiServerDnsNames, + IPs: ApiServerIPs, + }, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + }, + } +} + +func SchedulerCert() *Cert { + return &Cert{ + Name: "kube-scheduler", + LongName: "certificate for the kubernetes scheduler", + BaseName: "kube-scheduler", + CAName: "ca", + Config: cert.Config{ + CommonName: "system:kube-scheduler", + Organization: []string{"system:system:kube-scheduler"}, + AltNames: cert.AltNames{ + DNSNames: []string{"kube-scheduler"}, + IPs: []net.IP{[]byte("127.0.0.1")}, + }, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + }, + } +} + +func ControllerMgrCert() *Cert { + return &Cert{ + Name: "kube-controller-manager", + LongName: "certificate for the kubernetes controller manager", + BaseName: "kube-controller-manager", + CAName: "ca", + Config: cert.Config{ + CommonName: "system:kube-controller-manager", + Organization: []string{"system:kube-controller-manager"}, + AltNames: cert.AltNames{ + DNSNames: []string{"kube-scheduler"}, + IPs: []net.IP{[]byte("127.0.0.1")}, + }, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + }, + } +} + +func ServiceAccountsCert() *Cert { + return &Cert{ + Name: "service-accounts", + LongName: "certificate for kubernetes service account token signature", + BaseName: "service-accounts", + CAName: "ca", + Config: cert.Config{ + CommonName: "service-accounts", + }, + } +} + +func AdminCert() *Cert { + return &Cert{ + Name: "admin", + LongName: "certificate for kubernetes adminstrator", + BaseName: "admin", + CAName: "ca", + Config: cert.Config{ + CommonName: "admin", + Organization: []string{"system:masters"}, + }, + } +} + +func KubeletClientCert() *Cert { + return &Cert{ + Name: "apiserver-kubelet-client", + LongName: "certificate for the API server to connect to kubelet", + BaseName: "kube-apiserver-kubelet-client", + CAName: "ca", + Config: cert.Config{ + CommonName: "kube-apiserver-kubelet-client", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } +} diff --git a/cluster-provision/gocli/control-plane/certs/constants.go b/cluster-provision/gocli/control-plane/certs/constants.go new file mode 100644 index 0000000000..4aac39fab7 --- /dev/null +++ b/cluster-provision/gocli/control-plane/certs/constants.go @@ -0,0 +1,9 @@ +package certs + +import "net" + +var ApiServerDnsNames = []string{"kubernetes", "kubernetes.default", + "kubernetes.default.svc", "kubernetes.default.svc.cluster", + "kubernetes.svc.cluster.local", "api-server.kubernetes.local"} + +var ApiServerIPs = []net.IP{[]byte("127.0.0.1"), []byte("10.96.0.1"), []byte("192.168.66.110")} diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 3ed49ded89..58e4f59295 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -22,8 +22,8 @@ func buildEtcdCmdArgs() map[string]string { func buildControllerMgrCmdArgs() map[string]string { return map[string]string{ "--allocate-node-cidrs": "true", - "--authorization-kubeconfig": "/etc/kubernetes/controller-manager.conf", - "--authentication-kubeconfig": "/etc/kubernetes/controller-manager.conf", + "--authorization-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", + "--authentication-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", "--bind-address": "127.0.0.1", "--cluster-cidr": "10.244.0.0/16,fd10:244::/112", "--cluster-name": "kubernetes", diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 0aa73f3ff5..7751f9e954 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -5,12 +5,14 @@ import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" type RunControlPlaneComponentsPhase struct { dnsmasqID string k8sVersion string + pkiPath string containerRuntime cri.ContainerClient } -func NewRunControlPlaneComponentsPhase(dnsmasqID string, containerRuntime cri.ContainerClient) *RunControlPlaneComponentsPhase { +func NewRunControlPlaneComponentsPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath string) *RunControlPlaneComponentsPhase { return &RunControlPlaneComponentsPhase{ dnsmasqID: dnsmasqID, + pkiPath: pkiPath, containerRuntime: containerRuntime, } } @@ -43,7 +45,7 @@ func (p *RunControlPlaneComponentsPhase) runApiServer() error { createOpts := &cri.CreateOpts{ Name: "api-server", Mounts: map[string]string{ - "": "", // TODO + p.pkiPath: "/etc/kubernetes/pki/", }, Network: "container:" + p.dnsmasqID, Command: cmd, @@ -78,7 +80,7 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { createOpts := &cri.CreateOpts{ Name: "kube-controller-manager", Mounts: map[string]string{ - "": "", // TODO + p.pkiPath: "/etc/kubernetes/pki/", }, Network: "container:" + p.dnsmasqID, Command: cmd, @@ -103,12 +105,12 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { return err } - cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/scheduler.conf"} + cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig"} createOpts := &cri.CreateOpts{ Name: "kube-scheduler", Mounts: map[string]string{ - "": "", // TODO + p.pkiPath: "/etc/kubernetes/pki/", }, Network: "container:" + p.dnsmasqID, Command: cmd, diff --git a/cluster-provision/gocli/control-plane/crypto/ca.go b/cluster-provision/gocli/control-plane/crypto/ca.go new file mode 100644 index 0000000000..767881b804 --- /dev/null +++ b/cluster-provision/gocli/control-plane/crypto/ca.go @@ -0,0 +1,83 @@ +package crypto + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math" + "math/big" + "time" + + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" +) + +func WriteKeyAndCertToFile(certificate *x509.Certificate, key crypto.Signer, certPath, keyPath string) error { + encoded, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + return err + } + if err := keyutil.WriteKey(keyPath, encoded); err != nil { + return err + } + + pemBlock := pem.Block{ + Type: "CERTIFICATE", + Bytes: certificate.Raw, + } + certEncoded := pem.EncodeToMemory(&pemBlock) + + return cert.WriteCert(certPath, certEncoded) +} + +func GenerateKubernetesCAKeyPair() (*x509.Certificate, crypto.Signer, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + cert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "ca", Organization: []string{""}}, key) + if err != nil { + return nil, nil, err + } + + return cert, key, nil +} + +func GenerateCertKeyPairWithCA(cfg cert.Config, ca *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, crypto.Signer, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + + serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) + if len(cfg.CommonName) == 0 { + return nil, nil, err + } + + keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature + + notAfter := time.Now().Add(time.Hour * 24 * 365).UTC() + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: ca.NotBefore, + NotAfter: notAfter, + KeyUsage: keyUsage, + ExtKeyUsage: cfg.Usages, + BasicConstraintsValid: true, + IsCA: false, + } + certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, ca, key.Public(), caKey) + if err != nil { + return nil, nil, err + } + finalCert, err := x509.ParseCertificate(certDERBytes) + return finalCert, key, err +} diff --git a/cluster-provision/gocli/control-plane/etcd.go b/cluster-provision/gocli/control-plane/etcd.go index ae4885d973..2b80867e47 100644 --- a/cluster-provision/gocli/control-plane/etcd.go +++ b/cluster-provision/gocli/control-plane/etcd.go @@ -5,12 +5,14 @@ import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" type RunETCDPhase struct { dnsmasqID string containerRuntime cri.ContainerClient + pkiPath string } -func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient) Phase { +func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath string) Phase { return &RunETCDPhase{ dnsmasqID: dnsmasqID, containerRuntime: containerRuntime, + pkiPath: pkiPath, } } @@ -31,7 +33,7 @@ func (p *RunETCDPhase) Run() error { createOpts := &cri.CreateOpts{ Name: "etcd", Mounts: map[string]string{ - "/etcdcert": "/etc/kubernetes/pki/etcd", //TODO + p.pkiPath: "/etc/kubernetes/pki/etcd", }, Network: "container:" + p.dnsmasqID, Command: etcdCmd, diff --git a/cluster-provision/gocli/control-plane/kube-proxy.go b/cluster-provision/gocli/control-plane/kube-proxy.go index e7906c7454..9d244add13 100644 --- a/cluster-provision/gocli/control-plane/kube-proxy.go +++ b/cluster-provision/gocli/control-plane/kube-proxy.go @@ -16,6 +16,12 @@ type KubeProxyPhase struct { client k8s.K8sDynamicClient } +func NewKubeProxyPhase(client k8s.K8sDynamicClient) *KubeProxyPhase { + return &KubeProxyPhase{ + client: client, + } +} + func (p *KubeProxyPhase) Run() error { yamlDocs := bytes.Split(kp, []byte("---\n")) for _, yamlDoc := range yamlDocs { diff --git a/cluster-provision/gocli/control-plane/kubeconfig.go b/cluster-provision/gocli/control-plane/kubeconfig.go new file mode 100644 index 0000000000..a05cb6ad71 --- /dev/null +++ b/cluster-provision/gocli/control-plane/kubeconfig.go @@ -0,0 +1,78 @@ +package controlplane + +import ( + "os" + "path" + + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +type KubeConfigPhase struct { + pkiPath string +} + +var defaultComponents = map[string]string{ + "admin": "admin", + "kube-scheduler": "system:kube-scheduler", + "kube-controller-manager": "system:kube-controller-manager"} + +func NewKubeConfigPhase(pkiPath string) *KubeConfigPhase { + return &KubeConfigPhase{ + pkiPath: pkiPath, + } +} + +func (p *KubeConfigPhase) Run() error { + ca, err := os.ReadFile(path.Join(p.pkiPath, "ca.crt")) + if err != nil { + return err + } + + for component, userName := range defaultComponents { + clientCert, err := os.ReadFile(path.Join(p.pkiPath, component+".crt")) + if err != nil { + return err + } + + key, err := os.ReadFile(path.Join(p.pkiPath, component+".pem")) + if err != nil { + return err + } + + kubeconfig := buildKubeConfigFromCerts(ca, clientCert, key, "https://127.0.0.1:6443", userName) // todo: handle this better + err = clientcmd.WriteToFile(kubeconfig, path.Join(p.pkiPath, component, ".kubeconfig")) + if err != nil { + return err + } + return nil + } + return nil +} + +func buildKubeConfigFromCerts(ca, clientCert, clientKey []byte, server, user string) clientcmdapi.Config { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["kubernetes"] = &clientcmdapi.Cluster{ + Server: server, + CertificateAuthorityData: ca, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default"] = &clientcmdapi.Context{ + Cluster: "kubernetes", + AuthInfo: user, + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos[user] = &clientcmdapi.AuthInfo{ + ClientCertificateData: clientCert, + ClientKeyData: clientKey, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default", + AuthInfos: authinfos, + } + return clientConfig +} diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 88d0fd0c7f..b4c95d82ca 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -1,6 +1,15 @@ package controlplane -import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +import ( + "path" + + "k8s.io/client-go/rest" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri/docker" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri/podman" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) const ( etcdImage = "etcd:3.5.10-0" @@ -8,27 +17,94 @@ const ( controllerManager = "kube-controller-manager" scheduler = "kube-scheduler" registry = "registry.k8s.io" + defaultPkiPath = "/etc/kubevirtci/pki" ) type ControlPlaneRunner struct { dnsmasqID string containerRuntime cri.ContainerClient + Client k8s.K8sDynamicClient } type Phase interface { Run() error } -func NewControlPlaneRunner() {} +func NewControlPlaneRunner(dnsmasqID string) *ControlPlaneRunner { + var containerRuntime cri.ContainerClient + if podman.IsAvailable() { + containerRuntime = podman.NewPodman() + + } + if docker.IsAvailable() { + containerRuntime = docker.NewDockerClient() + } + + return &ControlPlaneRunner{ + containerRuntime: containerRuntime, + dnsmasqID: dnsmasqID, + } +} + +func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { + if err := NewCertsPhase(defaultPkiPath).Run(); err != nil { + return nil, err + } + + if err := NewRunETCDPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath).Run(); err != nil { + return nil, err + } + + if err := NewKubeConfigPhase(defaultPkiPath).Run(); err != nil { + return nil, err + } + + if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath).Run(); err != nil { + return nil, err + } + + config, err := k8s.NewConfig(path.Join(defaultPkiPath, "admin.kubeconfig"), 6443) + if err != nil { + return nil, err + } + + k8sClient, err := k8s.NewDynamicClient(config) + if err != nil { + return nil, err + } + + if err := NewCreateBootstrappersRBACPhase(k8sClient).Run(); err != nil { + return nil, err + } + + if err := NewKubeProxyPhase(k8sClient).Run(); err != nil { + return nil, err + } + + return config, nil +} + +func RestConfigToKubeconfig(restConfig *rest.Config) *clientcmdapi.Config { + kubeconfig := clientcmdapi.NewConfig() + kubeconfig.Clusters["default"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + InsecureSkipTLSVerify: restConfig.Insecure, + } -func (cp *ControlPlaneRunner) Start() error { - if err := NewRunETCDPhase(cp.dnsmasqID, cp.containerRuntime).Run(); err != nil { - return err + kubeconfig.AuthInfos["default"] = &clientcmdapi.AuthInfo{ + Token: restConfig.BearerToken, + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + Username: restConfig.Username, + Password: restConfig.Password, } - if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime).Run(); err != nil { - return err + kubeconfig.Contexts["default"] = &clientcmdapi.Context{ + Cluster: "default", + AuthInfo: "default", } - return nil + kubeconfig.CurrentContext = "default" + return kubeconfig } diff --git a/cluster-provision/gocli/go.mod b/cluster-provision/gocli/go.mod index 8c735b81e6..68a4e03870 100644 --- a/cluster-provision/gocli/go.mod +++ b/cluster-provision/gocli/go.mod @@ -1,6 +1,8 @@ module kubevirt.io/kubevirtci/cluster-provision/gocli -go 1.21 +go 1.22.3 + +toolchain go1.23.1 require ( github.com/alessio/shellescape v1.4.2 diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index ff73c461e0..c85fc90c56 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -1,6 +1,7 @@ package providers import ( + "bytes" "context" "fmt" "os" @@ -9,7 +10,10 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" + controlplane "kubevirt.io/kubevirtci/cluster-provision/gocli/control-plane" "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/aaq" @@ -88,6 +92,21 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc containers <- nfsGanesha } + var c *rest.Config + + if kp.Nodes > 1 { + runner := controlplane.NewControlPlaneRunner(dnsmasq) + c, err = runner.Start() + if err != nil { + return err + } + k8sClient, err := k8s.NewDynamicClient(c) + if err != nil { + return err + } + kp.Client = k8sClient + } + wg := sync.WaitGroup{} wg.Add(int(kp.Nodes)) macCounter := 0 @@ -178,26 +197,41 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc return err } - kubeConf, err := os.Create(".kubeconfig") - if err != nil { - return err - } + switch kp.Nodes { + case 1: + kubeConf, err := os.Create(".kubeconfig") + if err != nil { + return err + } - err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", kubeConf) - if err != nil { - return err - } + err = sshClient.CopyRemoteFile("/etc/kubernetes/admin.conf", kubeConf) + if err != nil { + return err + } - config, err := k8s.NewConfig(".kubeconfig", uint16(kp.APIServerPort)) - if err != nil { - return err - } + config, err := k8s.NewConfig(".kubeconfig", uint16(kp.APIServerPort)) + if err != nil { + return err + } - k8sClient, err := k8s.NewDynamicClient(config) - if err != nil { - return err + k8sClient, err := k8s.NewDynamicClient(config) + if err != nil { + return err + } + kp.Client = k8sClient + default: + c.Host = "https://192.168.66.110:6443" + kubeconfig := controlplane.RestConfigToKubeconfig(c) + configBytes, err := clientcmd.Write(*kubeconfig) + if err != nil { + return err + } + + err = sshClient.SCP("/etc/kubernetes/admin.conf", bytes.NewReader(configBytes)) + if err != nil { + return err + } } - kp.Client = k8sClient if err = kp.provisionK8sOpts(sshClient); err != nil { return err From 8f01693741c8d0678b63f4b4691eca701b9482eb Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 23 Oct 2024 13:38:22 +0300 Subject: [PATCH 13/69] networking Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 12 ++++++++ cluster-provision/gocli/providers/start.go | 29 +++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index 1ba305f6c7..e4ab69dc5e 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -4,6 +4,7 @@ set -ex NUM_NODES=${NUM_NODES-1} NUM_SECONDARY_NICS=${NUM_SECONDARY_NICS:-0} +SEQ_START=1 ip link add br0 type bridge echo 0 > /proc/sys/net/ipv6/conf/br0/disable_ipv6 @@ -18,7 +19,18 @@ for snet in $(seq 1 ${NUM_SECONDARY_NICS}); do ip link set dev br${snet} up done +# if the number is one then do the normal thing, if its higher than one then do the manual ip thingy + for i in $(seq 1 ${NUM_NODES}); do + if [ $NUM_NODES > 1 ]; then + ip tuntap add dev tap01 mode tap user $(whoami) + ip link set tap01 master br0 + ip link set dev tap01 up + ip addr add 192.168.66.101/24 dev tap01 + ip -6 addr add fd00::101 dev tap01 + i++ + fi; + n="$(printf "%02d" ${i})" ip tuntap add dev tap${n} mode tap user $(whoami) ip link set tap${n} master br0 diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index c85fc90c56..3b6bae6bc0 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -220,7 +220,8 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc } kp.Client = k8sClient default: - c.Host = "https://192.168.66.110:6443" + // we need to be able to reach the control plane from the vm network + c.Host = "https://192.168.66.101:6443" kubeconfig := controlplane.RestConfigToKubeconfig(c) configBytes, err := clientcmd.Write(*kubeconfig) if err != nil { @@ -231,6 +232,32 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc if err != nil { return err } + + // and also from the host network + kubeConf, err := os.Create(".kubeconfig") + if err != nil { + return err + } + kubeconfig = controlplane.RestConfigToKubeconfig(c) + configBytes, err = clientcmd.Write(*kubeconfig) + if err != nil { + return err + } + + _, err = kubeConf.Write(configBytes) + if err != nil { + return err + } + + config, err := k8s.NewConfig(".kubeconfig", uint16(kp.APIServerPort)) + if err != nil { + return err + } + k8sClient, err := k8s.NewDynamicClient(config) + if err != nil { + return err + } + kp.Client = k8sClient } if err = kp.provisionK8sOpts(sshClient); err != nil { From 71edf3b606d1d28ff4224931480cb262546e90e5 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 2 Nov 2024 14:10:50 +0200 Subject: [PATCH 14/69] use net parse ip --- cluster-provision/gocli/control-plane/certs/certlist.go | 4 ++-- cluster-provision/gocli/control-plane/certs/constants.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-provision/gocli/control-plane/certs/certlist.go b/cluster-provision/gocli/control-plane/certs/certlist.go index c595b14c49..304e5839ec 100644 --- a/cluster-provision/gocli/control-plane/certs/certlist.go +++ b/cluster-provision/gocli/control-plane/certs/certlist.go @@ -58,7 +58,7 @@ func SchedulerCert() *Cert { Organization: []string{"system:system:kube-scheduler"}, AltNames: cert.AltNames{ DNSNames: []string{"kube-scheduler"}, - IPs: []net.IP{[]byte("127.0.0.1")}, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, }, Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, }, @@ -76,7 +76,7 @@ func ControllerMgrCert() *Cert { Organization: []string{"system:kube-controller-manager"}, AltNames: cert.AltNames{ DNSNames: []string{"kube-scheduler"}, - IPs: []net.IP{[]byte("127.0.0.1")}, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, }, Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, }, diff --git a/cluster-provision/gocli/control-plane/certs/constants.go b/cluster-provision/gocli/control-plane/certs/constants.go index 4aac39fab7..1897eb570a 100644 --- a/cluster-provision/gocli/control-plane/certs/constants.go +++ b/cluster-provision/gocli/control-plane/certs/constants.go @@ -6,4 +6,4 @@ var ApiServerDnsNames = []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.svc.cluster.local", "api-server.kubernetes.local"} -var ApiServerIPs = []net.IP{[]byte("127.0.0.1"), []byte("10.96.0.1"), []byte("192.168.66.110")} +var ApiServerIPs = []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("10.96.0.1"), net.ParseIP("192.168.66.101")} From 9ebcf45427a6a5c7015cf9a4225d888c7fade9d9 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 2 Nov 2024 15:16:30 +0000 Subject: [PATCH 15/69] minor fixes Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 8 ++++---- cluster-provision/gocli/cri/docker/docker.go | 16 ++++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 58e4f59295..47f30d5eab 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -3,14 +3,14 @@ package controlplane func buildEtcdCmdArgs() map[string]string { return map[string]string{ "--advertise-client-urls": "https://0.0.0.0:2379", - "--cert-file": "/etc/kubernetes/pki/etcd/kube-api-server.crt", + "--cert-file": "/etc/kubernetes/pki/etcd/apiserver.crt", "--client-cert-auth": "true", "--data-dir": "/var/lib/etcd", "--experimental-initial-corrupt-check": "true", "--experimental-watch-progress-notify-interval": "5s", "--initial-advertise-peer-urls": "https://0.0.0.0:2380", "--initial-cluster": "node01=https://0.0.0.0:2380", - "--key-file": "/etc/kubernetes/pki/etcd/kube-api-server.key", + "--key-file": "/etc/kubernetes/pki/etcd/apiserver.pem", "--listen-client-urls": "https://0.0.0.0:2379", "--listen-metrics-urls": "http://0.0.0.0:2381", "--name": "node01", @@ -28,14 +28,14 @@ func buildControllerMgrCmdArgs() map[string]string { "--cluster-cidr": "10.244.0.0/16,fd10:244::/112", "--cluster-name": "kubernetes", "--cluster-signing-cert-file": "/etc/kubernetes/pki/ca.crt", - "--cluster-signing-key-file": "/etc/kubernetes/pki/ca.key", + "--cluster-signing-key-file": "/etc/kubernetes/pki/key.pem", "--controllers": "*,csrapproving,csrsigning,bootstrapsigner,tokencleaner", "--kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", "--node-cidr-mask-size-ipv6": "116", "--leader-elect": "true", "-v": "5", "--root-ca-file": "/etc/kubernetes/pki/ca.crt", - "--service-account-private-key-file": "/etc/kubernetes/pki/service-accounts.key", + "--service-account-private-key-file": "/etc/kubernetes/pki/service-accounts.pem", "--service-cluster-ip-range": "10.96.0.0/12,fd10:96::/108", "--use-service-account-credentials": "true", } diff --git a/cluster-provision/gocli/cri/docker/docker.go b/cluster-provision/gocli/cri/docker/docker.go index 43cd814e87..beebcaa66f 100644 --- a/cluster-provision/gocli/cri/docker/docker.go +++ b/cluster-provision/gocli/cri/docker/docker.go @@ -5,6 +5,7 @@ import ( "os/exec" "strconv" "strings" + "errors" "github.com/sirupsen/logrus" "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" @@ -55,15 +56,6 @@ func (dc *DockerClient) Start(containerID string) error { } func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string, error) { - ports := "" - for containerPort, hostPort := range createOpts.Ports { - ports += "-p " + containerPort + ":" + hostPort - } - - for hostFile, containerFile := range createOpts.Mounts { - ports += "-v " + hostFile + ":" + containerFile - } - args := []string{ "--name=" + createOpts.Name, "--privileged=" + strconv.FormatBool(createOpts.Privileged), @@ -76,6 +68,10 @@ func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string args = append(args, "-p", containerPort+":"+hostPort) } + for hostFile, containerFile := range createOpts.Mounts { + args = append(args, "-v", hostFile+":"+containerFile) + } + if len(createOpts.Capabilities) > 0 { args = append(args, "--cap-add="+strings.Join(createOpts.Capabilities, ",")) } @@ -90,7 +86,7 @@ func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string containerID, err := cmd.CombinedOutput() if err != nil { - return "", err + return "", errors.New(string(containerID)) } logrus.Info("created registry container with id: ", string(containerID)) return strings.TrimSuffix(string(containerID), "\n"), nil From 24cda577a742b7d2c03ad9806cc76f2fa1ad928e Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 2 Nov 2024 17:17:35 +0200 Subject: [PATCH 16/69] k8s version Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 3 ++- cluster-provision/gocli/control-plane/main.go | 5 +++-- cluster-provision/gocli/providers/start.go | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 7751f9e954..8c99397e30 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -9,11 +9,12 @@ type RunControlPlaneComponentsPhase struct { containerRuntime cri.ContainerClient } -func NewRunControlPlaneComponentsPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath string) *RunControlPlaneComponentsPhase { +func NewRunControlPlaneComponentsPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath, k8sVersion string) *RunControlPlaneComponentsPhase { return &RunControlPlaneComponentsPhase{ dnsmasqID: dnsmasqID, pkiPath: pkiPath, containerRuntime: containerRuntime, + k8sVersion: k8sVersion, } } diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index b4c95d82ca..4244fdc037 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -24,13 +24,14 @@ type ControlPlaneRunner struct { dnsmasqID string containerRuntime cri.ContainerClient Client k8s.K8sDynamicClient + k8sVersion string } type Phase interface { Run() error } -func NewControlPlaneRunner(dnsmasqID string) *ControlPlaneRunner { +func NewControlPlaneRunner(dnsmasqID string, k8sVersion string) *ControlPlaneRunner { var containerRuntime cri.ContainerClient if podman.IsAvailable() { containerRuntime = podman.NewPodman() @@ -59,7 +60,7 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } - if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath).Run(); err != nil { + if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath, cp.k8sVersion).Run(); err != nil { return nil, err } diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 3b6bae6bc0..2b97086c08 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "strings" "sync" "github.com/docker/docker/api/types/container" @@ -95,7 +96,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc var c *rest.Config if kp.Nodes > 1 { - runner := controlplane.NewControlPlaneRunner(dnsmasq) + runner := controlplane.NewControlPlaneRunner(dnsmasq, strings.Split(kp.Version, "-")[1]) c, err = runner.Start() if err != nil { return err From fdb01e65f2e2a0a1ef7d15996e4f94f9b727c0f5 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 2 Nov 2024 17:33:29 +0200 Subject: [PATCH 17/69] use versionmap Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 8c99397e30..5abd28bc2a 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -2,6 +2,13 @@ package controlplane import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +var versionMap = map[string]string{ + "1.30": "v1.30.2", + "1.29": "v1.29.6", + "1.28": "v1.28.11", + "1.31": "v1.31.0", +} + type RunControlPlaneComponentsPhase struct { dnsmasqID string k8sVersion string @@ -30,7 +37,7 @@ func (p *RunControlPlaneComponentsPhase) Run() error { } func (p *RunControlPlaneComponentsPhase) runApiServer() error { - apiServerImage := registry + "/" + apiServer + ":" + p.k8sVersion + apiServerImage := registry + "/" + apiServer + ":" + versionMap[p.k8sVersion] err := p.containerRuntime.ImagePull(apiServerImage) if err != nil { return err From ffc6026222184aaf605bf4a9ab4d9d3227ab88b5 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 2 Nov 2024 19:35:05 +0200 Subject: [PATCH 18/69] use versionmap Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 5abd28bc2a..32a8eabf8a 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -72,7 +72,7 @@ func (p *RunControlPlaneComponentsPhase) runApiServer() error { } func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { - ctrlMgrImage := registry + "/" + controllerManager + ":" + p.k8sVersion + ctrlMgrImage := registry + "/" + controllerManager + ":" + versionMap[p.k8sVersion] err := p.containerRuntime.ImagePull(ctrlMgrImage) if err != nil { return err @@ -94,7 +94,7 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { Command: cmd, } - apiserverContainer, err := p.containerRuntime.Create(ctrlMgrImage, createOpts) + apiserverContainer, err := p.containerRuntime.Create(ctrlMgrImage, createOpts) // todo: variable names if err != nil { return err } @@ -107,7 +107,7 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { } func (p *RunControlPlaneComponentsPhase) runScheduler() error { - schedulerImage := registry + "/" + scheduler + ":" + p.k8sVersion + schedulerImage := registry + "/" + scheduler + ":" + versionMap[p.k8sVersion] err := p.containerRuntime.ImagePull(schedulerImage) if err != nil { return err From b4b6c2dad844aad227e6002c8612a210a48d6f22 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sun, 3 Nov 2024 11:22:06 +0000 Subject: [PATCH 19/69] control plane fixes Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 20 +++++----- .../control-plane/control-plane-components.go | 9 ++++- .../gocli/control-plane/kubeconfig.go | 40 ++++++++++--------- cluster-provision/gocli/control-plane/main.go | 1 + 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 47f30d5eab..d467446a27 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -22,15 +22,15 @@ func buildEtcdCmdArgs() map[string]string { func buildControllerMgrCmdArgs() map[string]string { return map[string]string{ "--allocate-node-cidrs": "true", - "--authorization-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", - "--authentication-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", + "--authorization-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager/.kubeconfig", + "--authentication-kubeconfig": "/etc/kubernetes/pki/kube-controller-manager/.kubeconfig", "--bind-address": "127.0.0.1", "--cluster-cidr": "10.244.0.0/16,fd10:244::/112", "--cluster-name": "kubernetes", "--cluster-signing-cert-file": "/etc/kubernetes/pki/ca.crt", "--cluster-signing-key-file": "/etc/kubernetes/pki/key.pem", "--controllers": "*,csrapproving,csrsigning,bootstrapsigner,tokencleaner", - "--kubeconfig": "/etc/kubernetes/pki/kube-controller-manager.kubeconfig", + "--kubeconfig": "/etc/kubernetes/pki/kube-controller-manager/.kubeconfig", "--node-cidr-mask-size-ipv6": "116", "--leader-elect": "true", "-v": "5", @@ -43,7 +43,7 @@ func buildControllerMgrCmdArgs() map[string]string { func buildApiServerCmdArgs() map[string]string { return map[string]string{ - "--advertise-address": "192.168.66.110", + "--advertise-address": "192.168.66.101", "--allow-privileged": "true", "--audit-log-format": "json", "--audit-log-path": "/var/log/k8s-audit/k8s-audit.log", @@ -52,17 +52,17 @@ func buildApiServerCmdArgs() map[string]string { "--enable-admission-plugins": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", "--enable-bootstrap-token-auth": "true", "--etcd-cafile": "/etc/kubernetes/pki/ca.crt", - "--etcd-certfile": "/etc/kubernetes/pki/kube-api-server.crt", - "--etcd-keyfile": "/etc/kubernetes/pki/kube-api-server.key", + "--etcd-certfile": "/etc/kubernetes/pki/apiserver.crt", + "--etcd-keyfile": "/etc/kubernetes/pki/apiserver.pem", "--etcd-servers": "https://127.0.0.1:2379", "--kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", "--secure-port": "6443", "--v": "3", "--service-account-issuer": "https://kubernetes.default.svc.cluster.local", - "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.key", - "--service-account-signing-key-file": " /etc/kubernetes/pki/service-accounts.key", + "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.pem", + "--service-account-signing-key-file": "/etc/kubernetes/pki/service-accounts.pem", "--service-cluster-ip-range": "10.96.0.0/24", - "--tls-cert-file": "/etc/kubernetes/pki/kube-api-server.crt", - "--tls-private-key-file": "/etc/kubernetes/pki/kube-api-server.key", + "--tls-cert-file": "/etc/kubernetes/pki/apiserver.crt", + "--tls-private-key-file": "/etc/kubernetes/pki/apiserver.pem", } } diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 32a8eabf8a..0fe24c9a3f 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -1,6 +1,9 @@ package controlplane -import "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +import ( + "time" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +) var versionMap = map[string]string{ "1.30": "v1.30.2", @@ -32,6 +35,8 @@ func (p *RunControlPlaneComponentsPhase) Run() error { if err != nil { return err } + // wait 10 seconds before component starts to allow them to do all their necessary bootstrapping + time.Sleep(time.Second*10) } return nil } @@ -113,7 +118,7 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { return err } - cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig"} + cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/kube-scheduler/.kubeconfig"} createOpts := &cri.CreateOpts{ Name: "kube-scheduler", diff --git a/cluster-provision/gocli/control-plane/kubeconfig.go b/cluster-provision/gocli/control-plane/kubeconfig.go index a05cb6ad71..86ed71ed64 100644 --- a/cluster-provision/gocli/control-plane/kubeconfig.go +++ b/cluster-provision/gocli/control-plane/kubeconfig.go @@ -2,6 +2,7 @@ package controlplane import ( "os" + "fmt" "path" "k8s.io/client-go/tools/clientcmd" @@ -41,38 +42,41 @@ func (p *KubeConfigPhase) Run() error { } kubeconfig := buildKubeConfigFromCerts(ca, clientCert, key, "https://127.0.0.1:6443", userName) // todo: handle this better - err = clientcmd.WriteToFile(kubeconfig, path.Join(p.pkiPath, component, ".kubeconfig")) + err = clientcmd.WriteToFile(kubeconfig, p.pkiPath+"/"+component+"/.kubeconfig") if err != nil { - return err + return fmt.Errorf("error writing to file: %w", err) } - return nil + } return nil } func buildKubeConfigFromCerts(ca, clientCert, clientKey []byte, server, user string) clientcmdapi.Config { - clusters := make(map[string]*clientcmdapi.Cluster) - clusters["kubernetes"] = &clientcmdapi.Cluster{ - Server: server, - CertificateAuthorityData: ca, + clusters := map[string]*clientcmdapi.Cluster{ + "kubernetes": { + Server: server, + CertificateAuthorityData: ca, + }, } - contexts := make(map[string]*clientcmdapi.Context) - contexts["default"] = &clientcmdapi.Context{ - Cluster: "kubernetes", - AuthInfo: user, + contexts := map[string]*clientcmdapi.Context{ + "default": { + Cluster: "kubernetes", + AuthInfo: user, + }, } - authinfos := make(map[string]*clientcmdapi.AuthInfo) - authinfos[user] = &clientcmdapi.AuthInfo{ - ClientCertificateData: clientCert, - ClientKeyData: clientKey, + authInfos := map[string]*clientcmdapi.AuthInfo{ + user: { + ClientCertificateData: clientCert, + ClientKeyData: clientKey, + }, } - clientConfig := clientcmdapi.Config{ + + return clientcmdapi.Config{ Kind: "Config", APIVersion: "v1", Clusters: clusters, Contexts: contexts, CurrentContext: "default", - AuthInfos: authinfos, + AuthInfos: authInfos, } - return clientConfig } diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 4244fdc037..6ffdd8df88 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -44,6 +44,7 @@ func NewControlPlaneRunner(dnsmasqID string, k8sVersion string) *ControlPlaneRun return &ControlPlaneRunner{ containerRuntime: containerRuntime, dnsmasqID: dnsmasqID, + k8sVersion: k8sVersion, } } From cab837c3de89e346cf2ee013d7b39f759b55388b Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sun, 3 Nov 2024 13:25:35 +0200 Subject: [PATCH 20/69] add apiserver port parameter Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/kubeconfig.go | 4 ++-- cluster-provision/gocli/control-plane/main.go | 8 +++++--- cluster-provision/gocli/providers/start.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cluster-provision/gocli/control-plane/kubeconfig.go b/cluster-provision/gocli/control-plane/kubeconfig.go index 86ed71ed64..93770296c6 100644 --- a/cluster-provision/gocli/control-plane/kubeconfig.go +++ b/cluster-provision/gocli/control-plane/kubeconfig.go @@ -1,8 +1,8 @@ package controlplane import ( - "os" "fmt" + "os" "path" "k8s.io/client-go/tools/clientcmd" @@ -46,7 +46,7 @@ func (p *KubeConfigPhase) Run() error { if err != nil { return fmt.Errorf("error writing to file: %w", err) } - + } return nil } diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 6ffdd8df88..b18068f4e1 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -25,13 +25,14 @@ type ControlPlaneRunner struct { containerRuntime cri.ContainerClient Client k8s.K8sDynamicClient k8sVersion string + apiServerPort uint } type Phase interface { Run() error } -func NewControlPlaneRunner(dnsmasqID string, k8sVersion string) *ControlPlaneRunner { +func NewControlPlaneRunner(dnsmasqID string, k8sVersion string, apiServerPort uint) *ControlPlaneRunner { var containerRuntime cri.ContainerClient if podman.IsAvailable() { containerRuntime = podman.NewPodman() @@ -44,7 +45,8 @@ func NewControlPlaneRunner(dnsmasqID string, k8sVersion string) *ControlPlaneRun return &ControlPlaneRunner{ containerRuntime: containerRuntime, dnsmasqID: dnsmasqID, - k8sVersion: k8sVersion, + k8sVersion: k8sVersion, + apiServerPort: apiServerPort, } } @@ -65,7 +67,7 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } - config, err := k8s.NewConfig(path.Join(defaultPkiPath, "admin.kubeconfig"), 6443) + config, err := k8s.NewConfig(path.Join(defaultPkiPath, "admin/.kubeconfig"), uint16(cp.apiServerPort)) if err != nil { return nil, err } diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 2b97086c08..b0690ee25e 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -96,7 +96,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc var c *rest.Config if kp.Nodes > 1 { - runner := controlplane.NewControlPlaneRunner(dnsmasq, strings.Split(kp.Version, "-")[1]) + runner := controlplane.NewControlPlaneRunner(dnsmasq, strings.Split(kp.Version, "-")[1], uint(kp.APIServerPort)) c, err = runner.Start() if err != nil { return err From 9b64a557b7846110c25c52f37f256a538f50ab5c Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sun, 3 Nov 2024 14:06:24 +0200 Subject: [PATCH 21/69] fix registry container Signed-off-by: aerosouund --- cluster-provision/gocli/cri/docker/docker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-provision/gocli/cri/docker/docker.go b/cluster-provision/gocli/cri/docker/docker.go index beebcaa66f..70920645bb 100644 --- a/cluster-provision/gocli/cri/docker/docker.go +++ b/cluster-provision/gocli/cri/docker/docker.go @@ -1,11 +1,11 @@ package docker import ( + "errors" "os" "os/exec" "strconv" "strings" - "errors" "github.com/sirupsen/logrus" "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" @@ -88,7 +88,7 @@ func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string if err != nil { return "", errors.New(string(containerID)) } - logrus.Info("created registry container with id: ", string(containerID)) + logrus.Infof("created %s container with id: %s\n", createOpts.Name, string(containerID)) return strings.TrimSuffix(string(containerID), "\n"), nil } From 5f1cf63f425b252e999343f5300d6d63177b236c Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sun, 3 Nov 2024 14:39:00 +0200 Subject: [PATCH 22/69] no newline Signed-off-by: aerosouund --- cluster-provision/gocli/cri/docker/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/cri/docker/docker.go b/cluster-provision/gocli/cri/docker/docker.go index 70920645bb..fcd7171e68 100644 --- a/cluster-provision/gocli/cri/docker/docker.go +++ b/cluster-provision/gocli/cri/docker/docker.go @@ -88,7 +88,7 @@ func (dc *DockerClient) Create(image string, createOpts *cri.CreateOpts) (string if err != nil { return "", errors.New(string(containerID)) } - logrus.Infof("created %s container with id: %s\n", createOpts.Name, string(containerID)) + logrus.Infof("created %s container with id: %s", createOpts.Name, string(containerID)) return strings.TrimSuffix(string(containerID), "\n"), nil } From 9ad1c14f89e884efde7fca6d209bf5737ad775c7 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 11:49:47 +0200 Subject: [PATCH 23/69] use 110 as the container control plane ip Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/certs/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/certs/constants.go b/cluster-provision/gocli/control-plane/certs/constants.go index 1897eb570a..f97884f227 100644 --- a/cluster-provision/gocli/control-plane/certs/constants.go +++ b/cluster-provision/gocli/control-plane/certs/constants.go @@ -6,4 +6,4 @@ var ApiServerDnsNames = []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.svc.cluster.local", "api-server.kubernetes.local"} -var ApiServerIPs = []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("10.96.0.1"), net.ParseIP("192.168.66.101")} +var ApiServerIPs = []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("10.96.0.1"), net.ParseIP("192.168.66.110")} From cad2f5aafcef537a4d85819a6f26f80713586c87 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 11:51:31 +0200 Subject: [PATCH 24/69] forward port 6443 to the container control plane Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index e4ab69dc5e..0557eae998 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -3,7 +3,7 @@ set -ex NUM_NODES=${NUM_NODES-1} -NUM_SECONDARY_NICS=${NUM_SECONDARY_NICS:-0} +NUM_SECONDARY_NICS=0 SEQ_START=1 ip link add br0 type bridge @@ -20,28 +20,32 @@ for snet in $(seq 1 ${NUM_SECONDARY_NICS}); do done # if the number is one then do the normal thing, if its higher than one then do the manual ip thingy - -for i in $(seq 1 ${NUM_NODES}); do - if [ $NUM_NODES > 1 ]; then +i=1 +while [ $i -le ${NUM_NODES} ]; do + if [ ${NUM_NODES} -gt 1 ] && [ $i -eq 1 ]; then ip tuntap add dev tap01 mode tap user $(whoami) ip link set tap01 master br0 ip link set dev tap01 up - ip addr add 192.168.66.101/24 dev tap01 - ip -6 addr add fd00::101 dev tap01 - i++ - fi; + ip addr add 192.168.66.110/24 dev tap01 + ip -6 addr add fd00::110 dev tap01 + iptables -t nat -A PREROUTING -p tcp -i eth0 -m tcp --dport 6443 -j DNAT --to-destination 192.168.66.110:6443 + ((i++)) + fi n="$(printf "%02d" ${i})" ip tuntap add dev tap${n} mode tap user $(whoami) ip link set tap${n} master br0 ip link set dev tap${n} up DHCP_HOSTS="${DHCP_HOSTS} --dhcp-host=52:55:00:d1:55:${n},192.168.66.1${n},[fd00::1${n}],node${n},infinite" + for s in $(seq 1 ${NUM_SECONDARY_NICS}); do tap_name=stap$(($i - 1))-$(($s - 1)) ip tuntap add dev $tap_name mode tap user $(whoami) ip link set $tap_name master br${s} ip link set dev $tap_name up done + + ((i++)) # Increment for the main loop done # Make sure that all VMs can reach the internet @@ -52,4 +56,4 @@ ip6tables -t nat -A POSTROUTING -o eth0 -j MASQUERADE ip6tables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT ip6tables -A FORWARD -i br0 -o eth0 -j ACCEPT -exec dnsmasq --interface=br0 --enable-ra --dhcp-option=option6:dns-server,[::] -d ${DHCP_HOSTS} --dhcp-range=192.168.66.10,192.168.66.200,infinite --dhcp-range=::10,::200,constructor:br0,static +exec dnsmasq --interface=br0 --enable-ra --dhcp-option=option6:dns-server,[::] -d ${DHCP_HOSTS} --dhcp-range=192.168.66.10,192.168.66.200,infinite --dhcp-range=::10,::200,constructor:br0,static \ No newline at end of file From 64d8ecc3509171c46bc28803009385c91ff27819 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 14:17:31 +0200 Subject: [PATCH 25/69] join the control plane container Signed-off-by: aerosouund --- cluster-provision/gocli/opts/nodes/nodes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-provision/gocli/opts/nodes/nodes.go b/cluster-provision/gocli/opts/nodes/nodes.go index 999a976022..4905440d2b 100644 --- a/cluster-provision/gocli/opts/nodes/nodes.go +++ b/cluster-provision/gocli/opts/nodes/nodes.go @@ -25,11 +25,11 @@ func NewNodesProvisioner(sc libssh.Client, singleStack bool) *nodesProvisioner { func (n *nodesProvisioner) Exec() error { var ( nodeIP = "" - controlPlaneIP = "192.168.66.101" + controlPlaneIP = "192.168.66.110" ) if n.singleStack { - controlPlaneIP = "[fd00::101]" + controlPlaneIP = "[fd00::110]" nodeIP = "--node-ip=::" } From de88a87c717627fc5ede12074c715d8fda8096d0 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 14:36:56 +0200 Subject: [PATCH 26/69] scheduler Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 0fe24c9a3f..2c6ee3827b 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -2,6 +2,7 @@ package controlplane import ( "time" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" ) @@ -36,7 +37,7 @@ func (p *RunControlPlaneComponentsPhase) Run() error { return err } // wait 10 seconds before component starts to allow them to do all their necessary bootstrapping - time.Sleep(time.Second*10) + time.Sleep(time.Second * 10) } return nil } @@ -118,7 +119,7 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { return err } - cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/kube-scheduler/.kubeconfig"} + cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/pki/kube-scheduler/.kubeconfig"} createOpts := &cri.CreateOpts{ Name: "kube-scheduler", From 8488868da913251d4ddc9a00c3567458a1367fe0 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 14:44:12 +0200 Subject: [PATCH 27/69] exec sleep Signed-off-by: aerosouund --- cluster-provision/gocli/providers/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index b0690ee25e..75775ee363 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -128,7 +128,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc Env: []string{ fmt.Sprintf("NODE_NUM=%s", nodeNum), }, - Cmd: []string{"/bin/bash", "-c", qemuCMD}, + Cmd: []string{"/bin/bash", "-c", "sleep 9000000"}, } var deviceMappings []container.DeviceMapping From c15a34532e2f8d1eaba7b60076d5b287f50ce398 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 14:58:24 +0200 Subject: [PATCH 28/69] hack Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index 0557eae998..f0819b2a46 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -32,7 +32,7 @@ while [ $i -le ${NUM_NODES} ]; do ((i++)) fi - n="$(printf "%02d" ${i})" + n="$(printf "%02d" ${i-1})" ip tuntap add dev tap${n} mode tap user $(whoami) ip link set tap${n} master br0 ip link set dev tap${n} up From c1387fc41ab260ed78aaf4fc6d1fc30946bbb3f6 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 15:03:24 +0200 Subject: [PATCH 29/69] bash arithmetic Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index f0819b2a46..b0175fa369 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -32,7 +32,7 @@ while [ $i -le ${NUM_NODES} ]; do ((i++)) fi - n="$(printf "%02d" ${i-1})" + n="$(printf "%02d" $((i - 1)))" ip tuntap add dev tap${n} mode tap user $(whoami) ip link set tap${n} master br0 ip link set dev tap${n} up From ddb41dc411010433b73a693217206cfbf864241c Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 15:08:47 +0200 Subject: [PATCH 30/69] use other iface Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index b0175fa369..3c2d9b4336 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -22,17 +22,16 @@ done # if the number is one then do the normal thing, if its higher than one then do the manual ip thingy i=1 while [ $i -le ${NUM_NODES} ]; do - if [ ${NUM_NODES} -gt 1 ] && [ $i -eq 1 ]; then - ip tuntap add dev tap01 mode tap user $(whoami) - ip link set tap01 master br0 - ip link set dev tap01 up - ip addr add 192.168.66.110/24 dev tap01 - ip -6 addr add fd00::110 dev tap01 + if [ ${NUM_NODES} -gt 1 ]; then + ip tuntap add dev tap101 mode tap user $(whoami) + ip link set tap101 master br0 + ip link set dev tap101 up + ip addr add 192.168.66.110/24 dev tap101 + ip -6 addr add fd00::110 dev tap101 iptables -t nat -A PREROUTING -p tcp -i eth0 -m tcp --dport 6443 -j DNAT --to-destination 192.168.66.110:6443 - ((i++)) fi - n="$(printf "%02d" $((i - 1)))" + n="$(printf "%02d" ${i})" ip tuntap add dev tap${n} mode tap user $(whoami) ip link set tap${n} master br0 ip link set dev tap${n} up From b4fc880e9f4c561e09f708baba5e3d6952368848 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 15:16:57 +0200 Subject: [PATCH 31/69] create iface once Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index 3c2d9b4336..928ca35a6f 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -22,7 +22,7 @@ done # if the number is one then do the normal thing, if its higher than one then do the manual ip thingy i=1 while [ $i -le ${NUM_NODES} ]; do - if [ ${NUM_NODES} -gt 1 ]; then + if [ ${NUM_NODES} -gt 1 ] && [ $i -eq 1 ]; then ip tuntap add dev tap101 mode tap user $(whoami) ip link set tap101 master br0 ip link set dev tap101 up From 038e5b4a9db046c1c6adeb499ffab33c1e1d8273 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 15:20:48 +0200 Subject: [PATCH 32/69] execute qemu Signed-off-by: aerosouund --- cluster-provision/gocli/providers/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 75775ee363..b0690ee25e 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -128,7 +128,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc Env: []string{ fmt.Sprintf("NODE_NUM=%s", nodeNum), }, - Cmd: []string{"/bin/bash", "-c", "sleep 9000000"}, + Cmd: []string{"/bin/bash", "-c", qemuCMD}, } var deviceMappings []container.DeviceMapping From 31eac328f195b54ff23e71ec8a5e55d91a5ddaf3 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 15:23:54 +0200 Subject: [PATCH 33/69] dont start k8s Signed-off-by: aerosouund --- cluster-provision/gocli/providers/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index b0690ee25e..5015f0d84c 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -323,7 +323,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) opts = append(opts, psaOpt) } - if nodeIdx == 1 { + if nodeIdx == 100 { n := node01.NewNode01Provisioner(sshClient, kp.SingleStack, kp.NoEtcdFsync) opts = append(opts, n) From 87d48a34027751eb077d480a72784caf7910b08d Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:01:50 +0200 Subject: [PATCH 34/69] run bootstrap auth resources phase Signed-off-by: aerosouund --- .../gocli/control-plane/bootstrap.go | 19 +++++++- .../control-plane/config/cluster-info.yaml | 2 +- cluster-provision/gocli/control-plane/main.go | 6 +++ cluster-provision/gocli/providers/start.go | 47 +++++++++---------- 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cluster-provision/gocli/control-plane/bootstrap.go b/cluster-provision/gocli/control-plane/bootstrap.go index 1bf51bc032..78a9e213a9 100644 --- a/cluster-provision/gocli/control-plane/bootstrap.go +++ b/cluster-provision/gocli/control-plane/bootstrap.go @@ -3,7 +3,11 @@ package controlplane import ( "bytes" _ "embed" + "encoding/base64" "fmt" + "os" + "path" + "strings" "github.com/sirupsen/logrus" k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" @@ -19,10 +23,23 @@ var kk []byte var clusterInfo []byte type BootstrapAuthResourcesPhase struct { - client k8s.K8sDynamicClient + client k8s.K8sDynamicClient + pkiPath string +} + +func NewBootstrapAuthResourcesPhase(client k8s.K8sDynamicClient, pkiPath string) *BootstrapAuthResourcesPhase { + return &BootstrapAuthResourcesPhase{ + client: client, + pkiPath: pkiPath, + } } func (p *BootstrapAuthResourcesPhase) Run() error { + caData, err := os.ReadFile(path.Join(p.pkiPath + "ca.crt")) + if err != nil { + return err + } + clusterInfo = []byte(strings.Replace(string(clusterInfo), "CADATA", base64.StdEncoding.EncodeToString(caData), 1)) for _, manifest := range [][]byte{token, kk, clusterInfo} { yamlDocs := bytes.Split(manifest, []byte("---\n")) for _, yamlDoc := range yamlDocs { diff --git a/cluster-provision/gocli/control-plane/config/cluster-info.yaml b/cluster-provision/gocli/control-plane/config/cluster-info.yaml index 7aa5226e61..4c0fdc42e3 100644 --- a/cluster-provision/gocli/control-plane/config/cluster-info.yaml +++ b/cluster-provision/gocli/control-plane/config/cluster-info.yaml @@ -8,7 +8,7 @@ data: apiVersion: v1 clusters: - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZURENDQXpTZ0F3SUJBZ0lVVUxKYkdXeHI5cGxiZEIwMnh4U3N6UGVMNXlvd0RRWUpLb1pJaHZjTkFRRU4KQlFBd1FURUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2xkaGMyaHBibWQwYjI0eEVEQU9CZ05WQkFjTQpCMU5sWVhSMGJHVXhDekFKQmdOVkJBTU1Ba05CTUI0WERUSTBNRGt3TnpFeU16WTFObG9YRFRNME1Ea3dPREV5Ck16WTFObG93UVRFTE1Ba0dBMVVFQmhNQ1ZWTXhFekFSQmdOVkJBZ01DbGRoYzJocGJtZDBiMjR4RURBT0JnTlYKQkFjTUIxTmxZWFIwYkdVeEN6QUpCZ05WQkFNTUFrTkJNSUlDSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQWc4QQpNSUlDQ2dLQ0FnRUFzVDQzSFNqWDY1WFlIVVdqVkdhU0RpUlBBOU9maWM1NTNxaGhQa2lsR2dsblNsR21WYUcrCittWFBnS2E0WWUyUDRuTkRydzFGM1VlUEZ5Z0dxZE4rbnZhRmFDZmlXR1dCUUxzNFV1Nm1nZXJYdnJRaVFzZDMKVnlrc21MWUxNZlF6M0cyK3Jvbk9uQ056UTd1ZEdkTS94ckYzaEN1MWF3Q2xzMUlrMDQ5YUIzWUZQSnc5WHdFNQpqYy9YcGlqbXJGVVFZOGh1UHI2aWpWVGJPVXpzcmNJNmJqeDhtcDdGU0dQQ2ZCUGhPaUFJekRLcmlXOEFjaCtXCkJER0RsRGMxV01qTEdLVVorVUxjUFlzcVVTKzlrWUNxQmJtSUQvYkdRdWdXZm8vSUpkOFVCVmkwaStXVkhJeUsKNVY0QUJzakdYT3hRbHk1Y3BucDM0bUp6blAzRzF4bWIwMlRtcm1uZjdqVDJnQm1sS2F5TWZQMjJCeFFqL3pWZQpUeVJTOWs5dDk5dDBRTDhwNytla1BkekYwTFpnMXVhTU1iZEZaWWVSRjlLd3NNTzRiaU1sSWFMb21wQ2ZndFlZClZ1YUVqSnpLY0pERW84SXAvZTJJazRzaUE2SEdJRE0vWFpVTGVFaDR6NCt0V3pxQUE2MUlJNXM2U0c1OG5iN08KN1VXWXJiK3NRaFZkaTR4cVE5b3ZFRFF5L1pHTUVsbkNPcDF1NUFLTFZDQ2w2U1RIdWNXNnR5UEN4VWdqWm9GQwpOb096NUwvdXQ2YmZOdDJmZVB2a3JEejcyUVVWWG1tWmJmODYrLyszOFZPOCttdTNHanBXK1NoVFUwaW91NUpzCjlHd0dsZEJTNlptelN5blQ0ZXNSa0xWWkdVTkxZMGRRaG9BZ0w0VDg2TElRZ2N3WU1ZNnhVdXNDQXdFQUFhTTgKTURvd0RBWURWUjBUQkFVd0F3RUIvekFMQmdOVkhROEVCQU1DQVFZd0hRWURWUjBPQkJZRUZKeEl3cHFoQmQwNApBclV1MzIvLzJHV3M2ZWFjTUEwR0NTcUdTSWIzRFFFQkRRVUFBNElDQVFBa3pRR1ZWcXFWT3Q4NjRxblU5MFZLCnlGREM0eG4zcTdUeDVZYVV4SU1qQTF5Q2hlSjAzQ3c1QnNMT0VHdXc0Rmw5NWJHVU9ldzVieGp0L29lNGV6WEoKY09hWC9xd0FCRGNIbjBFSlNPczc0K3h0NkZhWnphYjNVaFphVXlSZ2FnQUVqL2RPaTZKYzZVSHZJbXVWbHVYTQpGWnp2dk1oSWJuak1tSmJ1N2xlenRXTE5BMWEybnM2d3M1LzdCdzdtQm1DUFhycDdaUzNqaWxSLzA1UjNRaTNUCkNzVGFMeGZxdU14WEJuZVdUSmRmWTBxZ1VBZGNhMlp6UTBDa2dyaXdienlCRldPU25TRGdNQzBNZjhsd3FGaGsKY3FybUk3bGlSZHd6RVQ5ekQ5aGFBVkpkRDdMVWRRMzFvbWdoVWp6cUFpeWZWU2t3dE5mczlkeExhWC9JTURxaQowRVVXR2ZEZHFyMm9xNExDTW4zNVQ3c1pLaDBvZ3MrdXA5UVVPYzY3UmNpcXdoQkwrOCs4Q0NHNlE3Q0E2TkNaCkk0cGQ1L1AwK0h6N0c2RlVoRWl4azJUbEZvd3hEdUFtd0V6T0ZUcG5GcmpjUVFyNlEvWDBYcTdYZFFQSzhhTUUKTmFNY1dTdCsyS1ZSbHdVTzRJOHRxYW05SkpqOGRPRHh6cVJNcnBNaUM0SkkvSGtvb0xkaDNxMkhSdUd1NUFQQgpXMGxheW5CejcyYUVzVkQycVhNQVZHaHNhdlVMNDlYdm8vY2hWcXpFY2t1QzRFVDcvQ2ZCakNTNTcxUUROL1Q0CkZGMThhZkh0eWNQbkF4TUNoTkpIZEVJZTBvbFg4eDJxcHdlRUg1eUpiYXk2U21od2NVU2NUUUZVUStISTBsQzUKY3Zjb2NpZ1g2elRhU2dKOVBVQWV1QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + certificate-authority-data: CADATA server: https://192.168.66.110:6443 contexts: null current-context: "" diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index b18068f4e1..fe5f5e797a 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -81,10 +81,16 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } + if err := NewBootstrapAuthResourcesPhase(k8sClient, defaultPkiPath).Run(); err != nil { + return nil, err + } + if err := NewKubeProxyPhase(k8sClient).Run(); err != nil { return nil, err } + // new cni phase ? + return config, nil } diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 5015f0d84c..c6661e58e0 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -25,7 +25,6 @@ import ( etcd "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" - "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/labelnodes" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" @@ -222,7 +221,7 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc kp.Client = k8sClient default: // we need to be able to reach the control plane from the vm network - c.Host = "https://192.168.66.101:6443" + c.Host = "https://192.168.66.110:6443" kubeconfig := controlplane.RestConfigToKubeconfig(c) configBytes, err := clientcmd.Write(*kubeconfig) if err != nil { @@ -235,22 +234,22 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc } // and also from the host network - kubeConf, err := os.Create(".kubeconfig") - if err != nil { - return err - } - kubeconfig = controlplane.RestConfigToKubeconfig(c) - configBytes, err = clientcmd.Write(*kubeconfig) - if err != nil { - return err - } - - _, err = kubeConf.Write(configBytes) - if err != nil { - return err - } - - config, err := k8s.NewConfig(".kubeconfig", uint16(kp.APIServerPort)) + // kubeConf, err := os.Create(".kubeconfig") + // if err != nil { + // return err + // } + // kubeconfig = controlplane.RestConfigToKubeconfig(c) + // configBytes, err = clientcmd.Write(*kubeconfig) + // if err != nil { + // return err + // } + + // _, err = kubeConf.Write(configBytes) + // if err != nil { + // return err + // } + + config, err := k8s.NewConfig("/etc/kubevirtci/pki/admin/.kubeconfig", uint16(kp.APIServerPort)) // todo if err != nil { return err } @@ -323,7 +322,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) opts = append(opts, psaOpt) } - if nodeIdx == 100 { + if nodeIdx == 1 && kp.Nodes == 1 { n := node01.NewNode01Provisioner(sshClient, kp.SingleStack, kp.NoEtcdFsync) opts = append(opts, n) @@ -362,11 +361,11 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) func (kp *KubevirtProvider) provisionK8sOpts(sshClient libssh.Client) error { opts := []opts.Opt{} - labelSelector := "node-role.kubernetes.io/control-plane" - if kp.Nodes > 1 { - labelSelector = "!node-role.kubernetes.io/control-plane" - } - opts = append(opts, labelnodes.NewNodeLabler(sshClient, labelSelector)) + // labelSelector := "node-role.kubernetes.io/control-plane" + // if kp.Nodes > 1 { + // labelSelector = "!node-role.kubernetes.io/control-plane" + // } + // opts = append(opts, labelnodes.NewNodeLabler(sshClient, labelSelector)) if kp.CDI { opts = append(opts, cdi.NewCdiOpt(kp.Client, sshClient, kp.CDIVersion)) From 1a443b42b02b999124c53d4f72ac9dd66935169f Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:11:31 +0200 Subject: [PATCH 35/69] run bootstrap auth resources phase Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/bootstrap.go b/cluster-provision/gocli/control-plane/bootstrap.go index 78a9e213a9..5fc4b0f091 100644 --- a/cluster-provision/gocli/control-plane/bootstrap.go +++ b/cluster-provision/gocli/control-plane/bootstrap.go @@ -35,7 +35,7 @@ func NewBootstrapAuthResourcesPhase(client k8s.K8sDynamicClient, pkiPath string) } func (p *BootstrapAuthResourcesPhase) Run() error { - caData, err := os.ReadFile(path.Join(p.pkiPath + "ca.crt")) + caData, err := os.ReadFile(path.Join(p.pkiPath, "ca.crt")) if err != nil { return err } From 06fb476fafe6ed02a2ad9bc8657827b4ca0b11f5 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:33:11 +0200 Subject: [PATCH 36/69] cni phase Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cni.go | 52 + .../gocli/control-plane/config/cni.yaml | 5134 ++++++++++++++++ .../gocli/control-plane/config/cni_ipv6.yaml | 5136 +++++++++++++++++ cluster-provision/gocli/control-plane/main.go | 4 +- 4 files changed, 10325 insertions(+), 1 deletion(-) create mode 100644 cluster-provision/gocli/control-plane/cni.go create mode 100644 cluster-provision/gocli/control-plane/config/cni.yaml create mode 100644 cluster-provision/gocli/control-plane/config/cni_ipv6.yaml diff --git a/cluster-provision/gocli/control-plane/cni.go b/cluster-provision/gocli/control-plane/cni.go new file mode 100644 index 0000000000..dd2d3738b6 --- /dev/null +++ b/cluster-provision/gocli/control-plane/cni.go @@ -0,0 +1,52 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/cni.yaml +var cni4 []byte + +//go:embed config/cni_ipv6.yaml +var cni6 []byte + +type CNIPhase struct { + ipv6 bool + client k8s.K8sDynamicClient +} + +func NewCNIPhase(client k8s.K8sDynamicClient, ipv6 bool) *CNIPhase { + return &CNIPhase{ + ipv6: ipv6, + client: client, + } +} + +func (c *CNIPhase) Run() error { + cni := cni4 + if c.ipv6 { + cni = cni6 + } + + yamlDocs := bytes.Split(cni, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := c.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/config/cni.yaml b/cluster-provision/gocli/control-plane/config/cni.yaml new file mode 100644 index 0000000000..ee4b88154e --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/cni.yaml @@ -0,0 +1,5134 @@ +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "true", + "assign_ipv6": "true" + }, + "container_settings": { + "allow_ip_forwarding": true + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + ignoredInterfaces: + description: IgnoredInterfaces indicates the network interfaces that + needs to be excluded when reading device routes. + items: + type: string + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: bgpfilters.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPFilter + listKind: BGPFilterList + plural: bgpfilters + singular: bgpfilter + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of + the BGP Filter. + properties: + exportV4: + description: The ordered set of IPv4 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + exportV6: + description: The ordered set of IPv6 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + importV4: + description: The ordered set of IPv4 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + importV6: + description: The ordered set of IPv6 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + filters: + description: The ordered set of BGPFilters applied on this BGP peer. + items: + type: string + type: array + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, + this is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + reachableBy: + description: Add an exact, i.e. /32, static route toward peer IP in + order to prevent route flapping. ReachableBy contains the address + of the gateway which peer can be reached by. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + ttlSecurity: + description: TTLSecurity enables the generalized TTL security mechanism + (GTSM) which protects against spoofed packets by ignoring received + packets with a smaller than expected TTL value. The provided value + is the number of hops (edges) between the peers. + type: integer + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: Classes declares the types of information to monitor + for this calico/node, and allows for selective status reporting + about certain subsets of information. + items: + type: string + type: array + node: + description: The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: UpdatePeriodSeconds is the period at which CalicoNodeStatus + should be updated. Set to 0 to disable CalicoNodeStatus refresh. + Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: LastUpdated is a timestamp representing the server time + when CalicoNodeStatus object last updated. It is represented in + RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enable" or "Disable". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfCTLBLogFilter: + description: 'BPFCTLBLogFilter specifies, what is logged by connect + time load balancer when BPFLogLevel is debug. Currently has to be + specified as ''all'' when BPFLogFilters is set to see CTLB logs. + [Default: unset - means logs are emitted when BPFLogLevel id debug + and BPFLogFilters not set.]' + type: string + bpfConnectTimeLoadBalancing: + description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls + whether Felix installs the connect-time load balancer. The connect-time + load balancer is required for the host to be able to reach Kubernetes + services and it improves the performance of pod-to-service connections.When + set to TCP, connect time load balancing is available only for services + with TCP ports. [Default: TCP]' + enum: + - TCP + - Enabled + - Disabled + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. + This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: + true]' + type: boolean + bpfDSROptoutCIDRs: + description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded + from DSR. That is, clients in those CIDRs will accesses nodeports + as if BPFExternalServiceMode was set to Tunnel. + items: + type: string + type: array + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableGROForIfaces: + description: BPFDisableGROForIfaces is a regular expression that controls + which interfaces Felix should disable the Generic Receive Offload + [GRO] option. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all host interfaces + with BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled, Strict or Loose. [Default: + Loose]' + pattern: ^(?i)(Disabled|Strict|Loose)?$ + type: string + bpfExcludeCIDRsFromNAT: + description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to + be excluded from NAT resolution so that host can handle them. A + typical usecase is node local DNS cache. + items: + type: string + type: array + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + pattern: ^(?i)(Tunnel|DSR)?$ + type: string + bpfForceTrackPacketsFromIfaces: + description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic + from these interfaces to skip Calico''s iptables NOTRACK rule, allowing + traffic from those interfaces to be tracked by Linux conntrack. Should + only be used for interfaces that are not used for the Calico fabric. For + example, a docker bridge device for non-Calico-networked containers. + [Default: docker+]' + items: + type: string + type: array + bpfHostConntrackBypass: + description: 'BPFHostConntrackBypass Controls whether to bypass Linux + conntrack in BPF mode for workloads and services. [Default: true + - bypass Linux conntrack]' + type: boolean + bpfHostNetworkedNATWithoutCTLB: + description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls + whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing + determines the CTLB behavior. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + bpfL3IfacePattern: + description: BPFL3IfacePattern is a regular expression that allows + to list tunnel devices like wireguard or vxlan (i.e., L3 devices) + in addition to BPFDataIfacePattern. That is, tunnel interfaces not + created by Calico, that Calico workload traffic flows over as well + as any interfaces that handle incoming traffic to nodeports and + services from outside the cluster. + type: string + bpfLogFilters: + additionalProperties: + type: string + description: "BPFLogFilters is a map of key=values where the value + is a pcap filter expression and the key is an interface name with + 'all' denoting all interfaces, 'weps' all workload endpoints and + 'heps' all host endpoints. \n When specified as an env var, it accepts + a comma-separated list of key=values. [Default: unset - means all + debug logs are emitted]" + type: object + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + pattern: ^(?i)(Off|Info|Debug)?$ + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + pattern: ^(?i)(insert|append)?$ + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: "DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix's (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s] \n Deprecated: replaced + by the generic HealthTimeoutOverrides." + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneHangAfter: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + pattern: ^(?i)(Drop|Accept|Return)?$ + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override feature detection + based on auto-detected platform capabilities. Values are specified + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" + or "false" will force the feature, empty or omitted values are auto-detected. + pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ + type: string + featureGates: + description: FeatureGates is used to enable or disable tech-preview + Calico features. Values are specified in a comma separated list + with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". + This is used to enable features that are not fully production ready. + pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$ + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + non-OpenStack floating IP addresses. (OpenStack-derived floating + IPs are always programmed, regardless of this setting.) + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + healthTimeoutOverrides: + description: HealthTimeoutOverrides allows the internal watchdog timeouts + of individual subcomponents to be overridden. This is useful for + working around "false positive" liveness timeouts that can occur + in particularly stressful workloads or if CPU is constrained. For + a list of active subcomponents, see Felix's logs. + items: + properties: + name: + type: string + timeout: + type: string + required: + - name + - timeout + type: object + type: array + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is Auto. + pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + type: string + iptablesFilterAllowAction: + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesFilterDenyAction: + description: IptablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default Calico blocks traffic + with an iptables "DROP" action. If you want to use "REJECT" action + instead you can configure it in here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesMangleAllowAction: + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$ + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + pattern: ^(?i)(Drop|Reject|Disabled)?$ + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + windowsManageFirewallRules: + description: 'WindowsManageFirewallRules configures whether or not + Felix will program Windows Firewall rules. (to allow inbound access + to its own metrics ports) [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + pattern: ^(?i)(Disabled|Any)?$ + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + performanceHints: + description: "PerformanceHints contains a list of hints to Calico's + policy engine to help process the policy more efficiently. Hints + never change the enforcement behaviour of the policy. \n Currently, + the only available hint is \"AssumeNeededOnEveryNode\". When that + hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for \"preloading\" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work done + to preload the policy (and to maintain it) is wasted." + items: + type: string + type: array + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. + type: string + allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object + strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: AllowedUse controls what the IP pool will be used for. If + not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When natOutgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + performanceHints: + description: "PerformanceHints contains a list of hints to Calico's + policy engine to help process the policy more efficiently. Hints + never change the enforcement behaviour of the policy. \n Currently, + the only available hint is \"AssumeNeededOnEveryNode\". When that + hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for \"preloading\" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work done + to preload the policy (and to maintain it) is wasted." + items: + type: string + type: array + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-cni-plugin + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - bgpfilters + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: [ "crd.projectcalico.org" ] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-node-rbac.yaml +# CNI cluster role +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-cni-plugin +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - clusterinformations + - ippools + - ipreservations + - ipamconfigs + verbs: + - get + - list + - create + - update + - delete +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-cni-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-cni-plugin +subjects: +- kind: ServiceAccount + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: quay.io/calico/cni:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: quay.io/calico/cni:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: quay.io/calico/node:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v3.27.2 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + - name: IP_AUTODETECTION_METHOD + value: "interface=eth.*" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: IP6 + value: "autodetect" + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "true" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV6POOL_NAT_OUTGOING + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + type: DirectoryOrCreate + - name: var-lib-calico + hostPath: + path: /var/lib/calico + type: DirectoryOrCreate + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + securityContext: + seLinuxOptions: + type: spc_t + containers: + - name: calico-kube-controllers + image: quay.io/calico/kube-controllers:v3.27.2 + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/config/cni_ipv6.yaml b/cluster-provision/gocli/control-plane/config/cni_ipv6.yaml new file mode 100644 index 0000000000..b83516e24c --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/cni_ipv6.yaml @@ -0,0 +1,5136 @@ +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "false", + "assign_ipv6": "true" + }, + "container_settings": { + "allow_ip_forwarding": true + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + ignoredInterfaces: + description: IgnoredInterfaces indicates the network interfaces that + needs to be excluded when reading device routes. + items: + type: string + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: bgpfilters.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPFilter + listKind: BGPFilterList + plural: bgpfilters + singular: bgpfilter + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of + the BGP Filter. + properties: + exportV4: + description: The ordered set of IPv4 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + exportV6: + description: The ordered set of IPv6 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + importV4: + description: The ordered set of IPv4 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + importV6: + description: The ordered set of IPv6 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + source: + type: string + required: + - action + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + filters: + description: The ordered set of BGPFilters applied on this BGP peer. + items: + type: string + type: array + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, + this is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + reachableBy: + description: Add an exact, i.e. /32, static route toward peer IP in + order to prevent route flapping. ReachableBy contains the address + of the gateway which peer can be reached by. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + ttlSecurity: + description: TTLSecurity enables the generalized TTL security mechanism + (GTSM) which protects against spoofed packets by ignoring received + packets with a smaller than expected TTL value. The provided value + is the number of hops (edges) between the peers. + type: integer + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: Classes declares the types of information to monitor + for this calico/node, and allows for selective status reporting + about certain subsets of information. + items: + type: string + type: array + node: + description: The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: UpdatePeriodSeconds is the period at which CalicoNodeStatus + should be updated. Set to 0 to disable CalicoNodeStatus refresh. + Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: LastUpdated is a timestamp representing the server time + when CalicoNodeStatus object last updated. It is represented in + RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enable" or "Disable". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfCTLBLogFilter: + description: 'BPFCTLBLogFilter specifies, what is logged by connect + time load balancer when BPFLogLevel is debug. Currently has to be + specified as ''all'' when BPFLogFilters is set to see CTLB logs. + [Default: unset - means logs are emitted when BPFLogLevel id debug + and BPFLogFilters not set.]' + type: string + bpfConnectTimeLoadBalancing: + description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls + whether Felix installs the connect-time load balancer. The connect-time + load balancer is required for the host to be able to reach Kubernetes + services and it improves the performance of pod-to-service connections.When + set to TCP, connect time load balancing is available only for services + with TCP ports. [Default: TCP]' + enum: + - TCP + - Enabled + - Disabled + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. + This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: + true]' + type: boolean + bpfDSROptoutCIDRs: + description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded + from DSR. That is, clients in those CIDRs will accesses nodeports + as if BPFExternalServiceMode was set to Tunnel. + items: + type: string + type: array + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableGROForIfaces: + description: BPFDisableGROForIfaces is a regular expression that controls + which interfaces Felix should disable the Generic Receive Offload + [GRO] option. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all host interfaces + with BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled, Strict or Loose. [Default: + Loose]' + pattern: ^(?i)(Disabled|Strict|Loose)?$ + type: string + bpfExcludeCIDRsFromNAT: + description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to + be excluded from NAT resolution so that host can handle them. A + typical usecase is node local DNS cache. + items: + type: string + type: array + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + pattern: ^(?i)(Tunnel|DSR)?$ + type: string + bpfForceTrackPacketsFromIfaces: + description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic + from these interfaces to skip Calico''s iptables NOTRACK rule, allowing + traffic from those interfaces to be tracked by Linux conntrack. Should + only be used for interfaces that are not used for the Calico fabric. For + example, a docker bridge device for non-Calico-networked containers. + [Default: docker+]' + items: + type: string + type: array + bpfHostConntrackBypass: + description: 'BPFHostConntrackBypass Controls whether to bypass Linux + conntrack in BPF mode for workloads and services. [Default: true + - bypass Linux conntrack]' + type: boolean + bpfHostNetworkedNATWithoutCTLB: + description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls + whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing + determines the CTLB behavior. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + bpfL3IfacePattern: + description: BPFL3IfacePattern is a regular expression that allows + to list tunnel devices like wireguard or vxlan (i.e., L3 devices) + in addition to BPFDataIfacePattern. That is, tunnel interfaces not + created by Calico, that Calico workload traffic flows over as well + as any interfaces that handle incoming traffic to nodeports and + services from outside the cluster. + type: string + bpfLogFilters: + additionalProperties: + type: string + description: "BPFLogFilters is a map of key=values where the value + is a pcap filter expression and the key is an interface name with + 'all' denoting all interfaces, 'weps' all workload endpoints and + 'heps' all host endpoints. \n When specified as an env var, it accepts + a comma-separated list of key=values. [Default: unset - means all + debug logs are emitted]" + type: object + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + pattern: ^(?i)(Off|Info|Debug)?$ + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + pattern: ^(?i)(insert|append)?$ + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: "DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix's (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s] \n Deprecated: replaced + by the generic HealthTimeoutOverrides." + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneHangAfter: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + pattern: ^(?i)(Drop|Accept|Return)?$ + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override feature detection + based on auto-detected platform capabilities. Values are specified + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" + or "false" will force the feature, empty or omitted values are auto-detected. + pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ + type: string + featureGates: + description: FeatureGates is used to enable or disable tech-preview + Calico features. Values are specified in a comma separated list + with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". + This is used to enable features that are not fully production ready. + pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$ + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + non-OpenStack floating IP addresses. (OpenStack-derived floating + IPs are always programmed, regardless of this setting.) + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + healthTimeoutOverrides: + description: HealthTimeoutOverrides allows the internal watchdog timeouts + of individual subcomponents to be overridden. This is useful for + working around "false positive" liveness timeouts that can occur + in particularly stressful workloads or if CPU is constrained. For + a list of active subcomponents, see Felix's logs. + items: + properties: + name: + type: string + timeout: + type: string + required: + - name + - timeout + type: object + type: array + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is Auto. + pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ + type: string + iptablesFilterAllowAction: + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesFilterDenyAction: + description: IptablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default Calico blocks traffic + with an iptables "DROP" action. If you want to use "REJECT" action + instead you can configure it in here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesMangleAllowAction: + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$ + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + pattern: ^(?i)(Drop|Reject|Disabled)?$ + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + windowsManageFirewallRules: + description: 'WindowsManageFirewallRules configures whether or not + Felix will program Windows Firewall rules. (to allow inbound access + to its own metrics ports) [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + pattern: ^(?i)(Disabled|Any)?$ + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + performanceHints: + description: "PerformanceHints contains a list of hints to Calico's + policy engine to help process the policy more efficiently. Hints + never change the enforcement behaviour of the policy. \n Currently, + the only available hint is \"AssumeNeededOnEveryNode\". When that + hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for \"preloading\" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work done + to preload the policy (and to maintain it) is wasted." + items: + type: string + type: array + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. + type: string + allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object + strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: AllowedUse controls what the IP pool will be used for. If + not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When natOutgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + performanceHints: + description: "PerformanceHints contains a list of hints to Calico's + policy engine to help process the policy more efficiently. Hints + never change the enforcement behaviour of the policy. \n Currently, + the only available hint is \"AssumeNeededOnEveryNode\". When that + hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for \"preloading\" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work done + to preload the policy (and to maintain it) is wasted." + items: + type: string + type: array + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-cni-plugin + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - bgpfilters + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: [ "crd.projectcalico.org" ] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-node-rbac.yaml +# CNI cluster role +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-cni-plugin +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - clusterinformations + - ippools + - ipreservations + - ipamconfigs + verbs: + - get + - list + - create + - update + - delete +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-cni-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-cni-plugin +subjects: +- kind: ServiceAccount + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: quay.io/calico/cni:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: quay.io/calico/cni:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: quay.io/calico/node:v3.27.2 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v3.27.2 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + - name: IP_AUTODETECTION_METHOD + value: "interface=eth.*" + # Auto-detect the BGP IP address. + - name: IP + value: "none" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Never" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: IP6 + value: "autodetect" + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "true" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV6POOL_NAT_OUTGOING + value: "true" + - name: CALICO_ROUTER_ID + value: "hash" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + type: DirectoryOrCreate + - name: var-lib-calico + hostPath: + path: /var/lib/calico + type: DirectoryOrCreate + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + securityContext: + seLinuxOptions: + type: spc_t + containers: + - name: calico-kube-controllers + image: quay.io/calico/kube-controllers:v3.27.2 + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index fe5f5e797a..6d3ade662c 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -89,7 +89,9 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } - // new cni phase ? + if err := NewCNIPhase(k8sClient, false).Run(); err != nil { + return nil, err + } return config, nil } From 209a1d22b3f490599bad044714d05439768832a7 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:44:47 +0200 Subject: [PATCH 37/69] sysctl Signed-off-by: aerosouund --- cluster-provision/gocli/opts/nodes/nodes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-provision/gocli/opts/nodes/nodes.go b/cluster-provision/gocli/opts/nodes/nodes.go index 4905440d2b..27742327a0 100644 --- a/cluster-provision/gocli/opts/nodes/nodes.go +++ b/cluster-provision/gocli/opts/nodes/nodes.go @@ -34,6 +34,8 @@ func (n *nodesProvisioner) Exec() error { } cmds := []string{ + "sysctl net/netfilter/nf_conntrack_tcp_timeout_close_wait=3600", + "sysctl --system", "source /var/lib/kubevirtci/shared_vars.sh", `timeout=30; interval=5; while ! hostnamectl | grep Transient; do echo "Waiting for dhclient to set the hostname from dnsmasq"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`, `echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice --fail-swap-on=false ` + nodeIP + ` --feature-gates=CPUManager=true,NodeSwap=true --cpu-manager-policy=static --kube-reserved=cpu=250m --system-reserved=cpu=250m" | tee /etc/sysconfig/kubelet > /dev/null`, From c51aa94451ca9b3541280c0d6a798436fbad4510 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:47:59 +0200 Subject: [PATCH 38/69] privileged kubeproxy Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/config/kube-proxy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml index 7db931bda0..f2c8baeaf5 100644 --- a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml +++ b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml @@ -129,7 +129,7 @@ spec: add: - NET_ADMIN - SYS_RESOURCE - privileged: false + privileged: true readOnlyRootFilesystem: false runAsNonRoot: false image: registry.k8s.io/kube-proxy:v1.28.0 From aa5060bd1dbcc8443ec8bd77af274292ce29d6ca Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:48:26 +0200 Subject: [PATCH 39/69] privileged kubeproxy Signed-off-by: aerosouund --- cluster-provision/gocli/opts/nodes/nodes.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cluster-provision/gocli/opts/nodes/nodes.go b/cluster-provision/gocli/opts/nodes/nodes.go index 27742327a0..4905440d2b 100644 --- a/cluster-provision/gocli/opts/nodes/nodes.go +++ b/cluster-provision/gocli/opts/nodes/nodes.go @@ -34,8 +34,6 @@ func (n *nodesProvisioner) Exec() error { } cmds := []string{ - "sysctl net/netfilter/nf_conntrack_tcp_timeout_close_wait=3600", - "sysctl --system", "source /var/lib/kubevirtci/shared_vars.sh", `timeout=30; interval=5; while ! hostnamectl | grep Transient; do echo "Waiting for dhclient to set the hostname from dnsmasq"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`, `echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice --fail-swap-on=false ` + nodeIP + ` --feature-gates=CPUManager=true,NodeSwap=true --cpu-manager-policy=static --kube-reserved=cpu=250m --system-reserved=cpu=250m" | tee /etc/sysconfig/kubelet > /dev/null`, From caf4edc56ce8f3d59a37a24f908df91ee383fd46 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Tue, 5 Nov 2024 18:51:21 +0200 Subject: [PATCH 40/69] allow privilage escalation smh Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/config/kube-proxy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml index f2c8baeaf5..e6223aa605 100644 --- a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml +++ b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml @@ -124,7 +124,6 @@ spec: containers: - name: kube-proxy securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_ADMIN From 51b2a3f9300f0bbfb93877f4c26b65afaf66863c Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 08:51:56 +0200 Subject: [PATCH 41/69] ip Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index d467446a27..6c60e8279b 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -43,7 +43,7 @@ func buildControllerMgrCmdArgs() map[string]string { func buildApiServerCmdArgs() map[string]string { return map[string]string{ - "--advertise-address": "192.168.66.101", + "--advertise-address": "192.168.66.110", "--allow-privileged": "true", "--audit-log-format": "json", "--audit-log-path": "/var/log/k8s-audit/k8s-audit.log", From 77a60b213b046ad485bcee564db5b35e273416ed Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 09:18:34 +0200 Subject: [PATCH 42/69] corends Signed-off-by: aerosouund --- .../gocli/control-plane/config/coredns.yaml | 201 ++++++++++++++++++ .../gocli/control-plane/coredns.go | 42 ++++ cluster-provision/gocli/control-plane/main.go | 4 + 3 files changed, 247 insertions(+) create mode 100644 cluster-provision/gocli/control-plane/config/coredns.yaml create mode 100644 cluster-provision/gocli/control-plane/coredns.go diff --git a/cluster-provision/gocli/control-plane/config/coredns.yaml b/cluster-provision/gocli/control-plane/config/coredns.yaml new file mode 100644 index 0000000000..bf6bfe6b7e --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/coredns.yaml @@ -0,0 +1,201 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes CLUSTER_DOMAIN REVERSE_CIDRS { + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . UPSTREAMNAMESERVER { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + }STUBDOMAINS +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + app.kubernetes.io/name: coredns +spec: + # replicas: not specified here: + # 1. Default is 1. + # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + app.kubernetes.io/name: coredns + template: + metadata: + labels: + k8s-app: kube-dns + app.kubernetes.io/name: coredns + spec: + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + nodeSelector: + kubernetes.io/os: linux + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["kube-dns"] + topologyKey: kubernetes.io/hostname + containers: + - name: coredns + image: coredns/coredns:1.9.4 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + app.kubernetes.io/name: coredns +spec: + selector: + k8s-app: kube-dns + app.kubernetes.io/name: coredns + clusterIP: CLUSTER_DNS_IP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP \ No newline at end of file diff --git a/cluster-provision/gocli/control-plane/coredns.go b/cluster-provision/gocli/control-plane/coredns.go new file mode 100644 index 0000000000..3adca0e180 --- /dev/null +++ b/cluster-provision/gocli/control-plane/coredns.go @@ -0,0 +1,42 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/coredns.yaml +var coredns []byte + +type CoreDNSPhase struct { + client k8s.K8sDynamicClient +} + +func NewCoreDNSPhase(client k8s.K8sDynamicClient) *CoreDNSPhase { + return &CoreDNSPhase{ + client: client, + } +} + +func (c *CoreDNSPhase) Run() error { + yamlDocs := bytes.Split(coredns, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := c.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 6d3ade662c..fc3834bbe3 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -93,6 +93,10 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } + if err := NewCoreDNSPhase(k8sClient).Run(); err != nil { + return nil, err + } + return config, nil } From dd80ea7d64769a71d10c0ec5949ff1f9b6055250 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 09:22:24 +0200 Subject: [PATCH 43/69] ip Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/config/coredns.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cluster-provision/gocli/control-plane/config/coredns.yaml b/cluster-provision/gocli/control-plane/config/coredns.yaml index bf6bfe6b7e..20257bf47a 100644 --- a/cluster-provision/gocli/control-plane/config/coredns.yaml +++ b/cluster-provision/gocli/control-plane/config/coredns.yaml @@ -176,9 +176,6 @@ kind: Service metadata: name: kube-dns namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" @@ -188,7 +185,7 @@ spec: selector: k8s-app: kube-dns app.kubernetes.io/name: coredns - clusterIP: CLUSTER_DNS_IP + clusterIP: 10.96.0.10 ports: - name: dns port: 53 From 8c8b74a7847eea5bf495b64652e102a60da801d1 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 09:36:55 +0200 Subject: [PATCH 44/69] coredns Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/config/coredns.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-provision/gocli/control-plane/config/coredns.yaml b/cluster-provision/gocli/control-plane/config/coredns.yaml index 20257bf47a..e2af542171 100644 --- a/cluster-provision/gocli/control-plane/config/coredns.yaml +++ b/cluster-provision/gocli/control-plane/config/coredns.yaml @@ -59,18 +59,18 @@ data: lameduck 5s } ready - kubernetes CLUSTER_DOMAIN REVERSE_CIDRS { + kubernetes cluster.local 10.96.0.0/12 { fallthrough in-addr.arpa ip6.arpa } prometheus :9153 - forward . UPSTREAMNAMESERVER { + forward . /etc/resolv.conf { max_concurrent 1000 } cache 30 loop reload loadbalance - }STUBDOMAINS + } --- apiVersion: apps/v1 kind: Deployment From fe051071bc3aa89a475a8470394b0160c889e1f2 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 12:19:35 +0200 Subject: [PATCH 45/69] add control plane container parameter Signed-off-by: aerosouund --- cluster-provision/gocli/opts/nodes/nodes.go | 29 ++++++++++++++----- .../gocli/opts/nodes/nodes_test.go | 2 +- cluster-provision/gocli/providers/start.go | 22 ++------------ 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/cluster-provision/gocli/opts/nodes/nodes.go b/cluster-provision/gocli/opts/nodes/nodes.go index 4905440d2b..8272b4dedd 100644 --- a/cluster-provision/gocli/opts/nodes/nodes.go +++ b/cluster-provision/gocli/opts/nodes/nodes.go @@ -11,28 +11,41 @@ import ( var cgroupv2 []byte type nodesProvisioner struct { - sshClient libssh.Client - singleStack bool + sshClient libssh.Client + singleStack bool + controlPlaneContainer bool } -func NewNodesProvisioner(sc libssh.Client, singleStack bool) *nodesProvisioner { +func NewNodesProvisioner(sc libssh.Client, singleStack bool, controlPlaneContainer bool) *nodesProvisioner { return &nodesProvisioner{ - sshClient: sc, - singleStack: singleStack, + sshClient: sc, + singleStack: singleStack, + controlPlaneContainer: controlPlaneContainer, } } func (n *nodesProvisioner) Exec() error { var ( - nodeIP = "" - controlPlaneIP = "192.168.66.110" + nodeIP string + controlPlaneIP string ) + controlPlaneIP = "192.168.66.101" + if n.singleStack { - controlPlaneIP = "[fd00::110]" + controlPlaneIP = "[fd00::101]" nodeIP = "--node-ip=::" } + if n.controlPlaneContainer { + if n.singleStack { + controlPlaneIP = "[fd00::110]" + } else { + controlPlaneIP = "192.168.66.110" + nodeIP = "" + } + } + cmds := []string{ "source /var/lib/kubevirtci/shared_vars.sh", `timeout=30; interval=5; while ! hostnamectl | grep Transient; do echo "Waiting for dhclient to set the hostname from dnsmasq"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`, diff --git a/cluster-provision/gocli/opts/nodes/nodes_test.go b/cluster-provision/gocli/opts/nodes/nodes_test.go index 397676b75d..20791d7338 100644 --- a/cluster-provision/gocli/opts/nodes/nodes_test.go +++ b/cluster-provision/gocli/opts/nodes/nodes_test.go @@ -24,7 +24,7 @@ var _ = Describe("NodesProvisioner", func() { BeforeEach(func() { mockCtrl = gomock.NewController(GinkgoT()) sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) - opt = NewNodesProvisioner(sshClient, false) + opt = NewNodesProvisioner(sshClient, false, false) AddExpectCalls(sshClient) }) diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index c6661e58e0..14f3c52a9e 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -227,29 +227,12 @@ func (kp *KubevirtProvider) Start(ctx context.Context, cancel context.CancelFunc if err != nil { return err } - err = sshClient.SCP("/etc/kubernetes/admin.conf", bytes.NewReader(configBytes)) if err != nil { return err } - // and also from the host network - // kubeConf, err := os.Create(".kubeconfig") - // if err != nil { - // return err - // } - // kubeconfig = controlplane.RestConfigToKubeconfig(c) - // configBytes, err = clientcmd.Write(*kubeconfig) - // if err != nil { - // return err - // } - - // _, err = kubeConf.Write(configBytes) - // if err != nil { - // return err - // } - - config, err := k8s.NewConfig("/etc/kubevirtci/pki/admin/.kubeconfig", uint16(kp.APIServerPort)) // todo + config, err := k8s.NewConfig("/etc/kubevirtci/pki/admin/.kubeconfig", uint16(kp.APIServerPort)) if err != nil { return err } @@ -325,7 +308,6 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) if nodeIdx == 1 && kp.Nodes == 1 { n := node01.NewNode01Provisioner(sshClient, kp.SingleStack, kp.NoEtcdFsync) opts = append(opts, n) - } else { if kp.GPU != "" { // move the assigned PCI device to a vfio-pci driver to prepare for assignment @@ -336,7 +318,7 @@ func (kp *KubevirtProvider) provisionNode(sshClient libssh.Client, nodeIdx int) bindVfioOpt := bindvfio.NewBindVfioOpt(sshClient, gpuDeviceID) opts = append(opts, bindVfioOpt) } - n := nodesprovision.NewNodesProvisioner(sshClient, kp.SingleStack) + n := nodesprovision.NewNodesProvisioner(sshClient, kp.SingleStack, kp.Nodes > 1) opts = append(opts, n) } From 5c853b4973fd67946de41263fd0ad5404a4e47f0 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 13:23:50 +0200 Subject: [PATCH 46/69] create local volume opt Signed-off-by: aerosouund --- .../gocli/opts/localvolume/local_volume.go | 27 ++++ .../localvolume/manifests/local-volume.yaml | 130 ++++++++++++++++++ cluster-provision/gocli/providers/start.go | 2 + 3 files changed, 159 insertions(+) create mode 100644 cluster-provision/gocli/opts/localvolume/local_volume.go create mode 100644 cluster-provision/gocli/opts/localvolume/manifests/local-volume.yaml diff --git a/cluster-provision/gocli/opts/localvolume/local_volume.go b/cluster-provision/gocli/opts/localvolume/local_volume.go new file mode 100644 index 0000000000..1b5240ce50 --- /dev/null +++ b/cluster-provision/gocli/opts/localvolume/local_volume.go @@ -0,0 +1,27 @@ +package localvolume + +import ( + _ "embed" + + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +//go:embed manifests/local-volume.yaml +var lv []byte + +type localVolumeOpt struct { + sshClient libssh.Client +} + +func NewLocalVolumeOpt(sshClient libssh.Client) *localVolumeOpt { + return &localVolumeOpt{ + sshClient: sshClient, + } +} + +func (lv *localVolumeOpt) Exec() error { + if err := lv.sshClient.Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f /provision/local-volume.yaml"); err != nil { + return err + } + return nil +} diff --git a/cluster-provision/gocli/opts/localvolume/manifests/local-volume.yaml b/cluster-provision/gocli/opts/localvolume/manifests/local-volume.yaml new file mode 100644 index 0000000000..93012326df --- /dev/null +++ b/cluster-provision/gocli/opts/localvolume/manifests/local-volume.yaml @@ -0,0 +1,130 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-storage-config +data: + storageClassMap: | + local: + hostDir: /mnt/local-storage/local + mountDir: /mnt/local-storage/local +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-pv-binding +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: default +roleRef: + kind: ClusterRole + name: system:persistent-volume-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-storage-provisioner-node-clusterrole +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-node-binding +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: default +roleRef: + kind: ClusterRole + name: local-storage-provisioner-node-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-storage-provisioner-jobs-role +rules: +- apiGroups: + - 'batch' + resources: + - jobs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-storage-provisioner-jobs-rolebinding +subjects: +- kind: ServiceAccount + name: local-storage-admin +roleRef: + kind: Role + name: local-storage-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-storage-admin +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + labels: + app: local-volume-provisioner +spec: + selector: + matchLabels: + app: local-volume-provisioner + template: + metadata: + labels: + app: local-volume-provisioner + spec: + serviceAccountName: local-storage-admin + containers: + - image: "quay.io/external_storage/local-volume-provisioner:v2.5.0" + name: provisioner + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JOB_CONTAINER_IMAGE + value: "quay.io/external_storage/local-volume-provisioner:v2.5.0" + volumeMounts: + - mountPath: /etc/provisioner/config + name: provisioner-config + readOnly: true + - mountPath: /mnt/local-storage + name: local-storage + mountPropagation: "HostToContainer" + volumes: + - name: provisioner-config + configMap: + name: local-storage-config + - name: local-storage + hostPath: + path: /mnt/local-storage diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 14f3c52a9e..8a8de8bab6 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -25,6 +25,7 @@ import ( etcd "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/localvolume" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/node01" @@ -348,6 +349,7 @@ func (kp *KubevirtProvider) provisionK8sOpts(sshClient libssh.Client) error { // labelSelector = "!node-role.kubernetes.io/control-plane" // } // opts = append(opts, labelnodes.NewNodeLabler(sshClient, labelSelector)) + opts = append(opts, localvolume.NewLocalVolumeOpt(sshClient)) if kp.CDI { opts = append(opts, cdi.NewCdiOpt(kp.Client, sshClient, kp.CDIVersion)) From 5fdd8c95175096fb8dade0771e4d621f9bd1bbb9 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 13:52:57 +0200 Subject: [PATCH 47/69] add prefix to the control plane container Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 6 +++--- cluster-provision/gocli/control-plane/etcd.go | 6 ++++-- cluster-provision/gocli/control-plane/main.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 2c6ee3827b..cdc4fcd1d0 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -57,7 +57,7 @@ func (p *RunControlPlaneComponentsPhase) runApiServer() error { } createOpts := &cri.CreateOpts{ - Name: "api-server", + Name: p.k8sVersion + "-api-server", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, @@ -92,7 +92,7 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { } createOpts := &cri.CreateOpts{ - Name: "kube-controller-manager", + Name: p.k8sVersion + "-kube-controller-manager", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, @@ -122,7 +122,7 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/pki/kube-scheduler/.kubeconfig"} createOpts := &cri.CreateOpts{ - Name: "kube-scheduler", + Name: p.k8sVersion + "-kube-scheduler", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, diff --git a/cluster-provision/gocli/control-plane/etcd.go b/cluster-provision/gocli/control-plane/etcd.go index 2b80867e47..b10d2cd39b 100644 --- a/cluster-provision/gocli/control-plane/etcd.go +++ b/cluster-provision/gocli/control-plane/etcd.go @@ -6,13 +6,15 @@ type RunETCDPhase struct { dnsmasqID string containerRuntime cri.ContainerClient pkiPath string + k8sVersion string } -func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath string) Phase { +func NewRunETCDPhase(dnsmasqID string, containerRuntime cri.ContainerClient, pkiPath, k8sVersion string) Phase { return &RunETCDPhase{ dnsmasqID: dnsmasqID, containerRuntime: containerRuntime, pkiPath: pkiPath, + k8sVersion: k8sVersion, } } @@ -31,7 +33,7 @@ func (p *RunETCDPhase) Run() error { } createOpts := &cri.CreateOpts{ - Name: "etcd", + Name: p.k8sVersion + "-etcd", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/etcd", }, diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index fc3834bbe3..9e4ea4d64d 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -55,7 +55,7 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } - if err := NewRunETCDPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath).Run(); err != nil { + if err := NewRunETCDPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath, cp.k8sVersion).Run(); err != nil { return nil, err } From e3abee420b6926624037a318ff38460e5a4092c7 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 13:55:36 +0200 Subject: [PATCH 48/69] add prefix Signed-off-by: aerosouund --- .../gocli/control-plane/control-plane-components.go | 6 +++--- cluster-provision/gocli/control-plane/etcd.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index cdc4fcd1d0..700f770468 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -57,7 +57,7 @@ func (p *RunControlPlaneComponentsPhase) runApiServer() error { } createOpts := &cri.CreateOpts{ - Name: p.k8sVersion + "-api-server", + Name: "k8s-" + p.k8sVersion + "-api-server", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, @@ -92,7 +92,7 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { } createOpts := &cri.CreateOpts{ - Name: p.k8sVersion + "-kube-controller-manager", + Name: "k8s-" + p.k8sVersion + "-kube-controller-manager", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, @@ -122,7 +122,7 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { cmd := []string{"kube-scheduler", "--kubeconfig=/etc/kubernetes/pki/kube-scheduler/.kubeconfig"} createOpts := &cri.CreateOpts{ - Name: p.k8sVersion + "-kube-scheduler", + Name: "k8s-" + p.k8sVersion + "-kube-scheduler", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/", }, diff --git a/cluster-provision/gocli/control-plane/etcd.go b/cluster-provision/gocli/control-plane/etcd.go index b10d2cd39b..b668eb8d38 100644 --- a/cluster-provision/gocli/control-plane/etcd.go +++ b/cluster-provision/gocli/control-plane/etcd.go @@ -33,7 +33,7 @@ func (p *RunETCDPhase) Run() error { } createOpts := &cri.CreateOpts{ - Name: p.k8sVersion + "-etcd", + Name: "k8s-" + p.k8sVersion + "-etcd", Mounts: map[string]string{ p.pkiPath: "/etc/kubernetes/pki/etcd", }, From a5fc4df3ce7c354f8915410e33938294fc41bc2f Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:28:00 +0200 Subject: [PATCH 49/69] runtime wont be available in the container Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 9e4ea4d64d..40d2ea8e93 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -38,7 +38,7 @@ func NewControlPlaneRunner(dnsmasqID string, k8sVersion string, apiServerPort ui containerRuntime = podman.NewPodman() } - if docker.IsAvailable() { + if true { containerRuntime = docker.NewDockerClient() } From f8f4de6efaaea330d8768d3ac33c44dfff235075 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:36:06 +0200 Subject: [PATCH 50/69] mount runtime Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index d7d83f4583..2abf07e14c 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -29,6 +29,7 @@ if [ "${KUBEVIRTCI_RUNTIME}" = "podman" ]; then elif [ "${KUBEVIRTCI_RUNTIME}" = "docker" ]; then _cri_bin=docker _cri_socket="/var/run/docker.sock" + _bin_dir=$(which docker) else _cri_socket=$(detect_podman_socket) if [ -n "$_cri_socket" ]; then @@ -45,7 +46,7 @@ else fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" -_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock" +_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock" -v "${_bin_dir}:/usr/local/bin/${_cri_bin}" # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists if [ -d /lib/modules ]; then From 48bdb9a67a84da032c19f64ebff4e83e08cac71a Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:37:06 +0200 Subject: [PATCH 51/69] mount runtime Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index 2abf07e14c..3f2f91fad3 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -46,7 +46,7 @@ else fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" -_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock" -v "${_bin_dir}:/usr/local/bin/${_cri_bin}" +_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v ${_bin_dir}:/usr/local/bin/${_cri_bin}" # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists if [ -d /lib/modules ]; then From 63d6f332149d15e3a855438fd6dcf30172cbae07 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:42:26 +0200 Subject: [PATCH 52/69] mount runtime Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index 3f2f91fad3..9871ad2f88 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -46,7 +46,7 @@ else fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" -_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v ${_bin_dir}:/usr/local/bin/${_cri_bin}" +_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v ${_bin_dir}:/usr/bin/${_cri_bin}" # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists if [ -d /lib/modules ]; then From 33879316bdcae54c88ae1c196972a0f384e71578 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:43:42 +0200 Subject: [PATCH 53/69] debug Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index 9871ad2f88..80cc230582 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -47,6 +47,7 @@ fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" _cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v ${_bin_dir}:/usr/bin/${_cri_bin}" +echo $_cli # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists if [ -d /lib/modules ]; then From d5664b3fffd4d75aace9baea064a17e00fa2267a Mon Sep 17 00:00:00 2001 From: aerosouund Date: Wed, 6 Nov 2024 14:54:15 +0200 Subject: [PATCH 54/69] mount binaries Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index 80cc230582..c0e0bae782 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -46,7 +46,7 @@ else fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" -_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v ${_bin_dir}:/usr/bin/${_cri_bin}" +_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v /lib:/lib -v /lib64:/lib64 -v /usr/lib:/usr/lib -v /usr/bin:/usr/bin" echo $_cli # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists From 1f4d6498419843b47883110c7bdde10aa6aae485 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 17:49:10 +0200 Subject: [PATCH 55/69] use default name Signed-off-by: aerosouund --- cluster-up/cluster/k8s-provider-common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 32c4ede425..47ccc96d98 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -68,8 +68,8 @@ function up() { # Set server and disable tls check export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig - kubectl config set-cluster kubernetes --server="https://$(_main_ip):$(_port k8s)" - kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true + kubectl config set-cluster default --server="https://$(_main_ip):$(_port k8s)" + kubectl config set-cluster default --insecure-skip-tls-verify=true # Workaround https://github.com/containers/conmon/issues/315 by not dumping the file to stdout for the time being if [[ ${_cri_bin} = podman* ]]; then From 68f50147b8dcbb7d3211ede7a48060f8f26945c1 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 18:33:31 +0200 Subject: [PATCH 56/69] kwok opt Signed-off-by: aerosouund --- cluster-provision/gocli/cmd/run.go | 1 + cluster-provision/gocli/go.mod | 23 +- cluster-provision/gocli/go.sum | 50 +- cluster-provision/gocli/opts/kwok/kwok.go | 47 + .../gocli/opts/kwok/manifests/kwok.yaml | 2408 +++++++++++++++++ .../gocli/opts/kwok/manifests/stage-fast.yaml | 255 ++ cluster-provision/gocli/pkg/k8s/k8s.go | 2 + cluster-provision/gocli/providers/opts.go | 5 + cluster-provision/gocli/providers/start.go | 5 + cluster-provision/gocli/providers/types.go | 5 + cluster-provision/gocli/vendor/modules.txt | 32 +- .../gocli/vendor/sigs.k8s.io/kwok/LICENSE | 201 ++ .../kwok/pkg/apis/v1alpha1/attach_types.go | 84 + .../pkg/apis/v1alpha1/cluster_attach_types.go | 76 + .../pkg/apis/v1alpha1/cluster_exec_types.go | 80 + .../pkg/apis/v1alpha1/cluster_logs_types.go | 79 + .../v1alpha1/cluster_port_forward_types.go | 80 + .../v1alpha1/cluster_resource_usage_types.go | 80 + .../kwok/pkg/apis/v1alpha1/condition.go | 70 + .../sigs.k8s.io/kwok/pkg/apis/v1alpha1/doc.go | 29 + .../kwok/pkg/apis/v1alpha1/exec_types.go | 113 + .../pkg/apis/v1alpha1/groupversion_info.go | 39 + .../kwok/pkg/apis/v1alpha1/logs_types.go | 86 + .../kwok/pkg/apis/v1alpha1/metric_types.go | 165 ++ .../kwok/pkg/apis/v1alpha1/object_selector.go | 27 + .../pkg/apis/v1alpha1/port_forward_types.go | 101 + .../kwok/pkg/apis/v1alpha1/register.go | 29 + .../pkg/apis/v1alpha1/resource_usage_types.go | 93 + .../kwok/pkg/apis/v1alpha1/stage_types.go | 271 ++ .../apis/v1alpha1/zz_generated.deepcopy.go | 1972 ++++++++++++++ .../apis/v1alpha1/zz_generated.defaults.go | 70 + .../cluster/ephemeral-provider-common.sh | 4 + cluster-up/cluster/k8s-provider-common.sh | 11 - 33 files changed, 6528 insertions(+), 65 deletions(-) create mode 100644 cluster-provision/gocli/opts/kwok/kwok.go create mode 100644 cluster-provision/gocli/opts/kwok/manifests/kwok.yaml create mode 100644 cluster-provision/gocli/opts/kwok/manifests/stage-fast.yaml create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/LICENSE create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/attach_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_attach_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_exec_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_logs_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_port_forward_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_resource_usage_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/condition.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/doc.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/exec_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/groupversion_info.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/logs_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/metric_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/object_selector.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/port_forward_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/register.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/resource_usage_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/stage_types.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.deepcopy.go create mode 100644 cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.defaults.go diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index 1392c03772..5122e325d1 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -50,6 +50,7 @@ func NewRunCommand() *cobra.Command { run.Flags().Bool("skip-cnao-cr", false, "skip deploying cnao custom resource. if true, only cnao CRDS will be deployed") run.Flags().Bool("deploy-multus", false, "deploy multus") run.Flags().Bool("deploy-cdi", false, "deploy cdi") + run.Flags().Bool("deploy-kwok", false, "deploy kwok") run.Flags().String("cdi-version", "", "cdi version") run.Flags().String("aaq-version", "", "aaq version") run.Flags().Bool("deploy-aaq", false, "deploy aaq") diff --git a/cluster-provision/gocli/go.mod b/cluster-provision/gocli/go.mod index 68a4e03870..1960dd640f 100644 --- a/cluster-provision/gocli/go.mod +++ b/cluster-provision/gocli/go.mod @@ -10,38 +10,38 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/docker/docker v26.1.5+incompatible github.com/docker/go-connections v0.5.0 - github.com/onsi/ginkgo/v2 v2.18.0 + github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 github.com/rook/rook/pkg/apis v0.0.0-20240801164516-9c98467f3c32 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.24.0 golang.org/x/net v0.26.0 gopkg.in/yaml.v3 v3.0.1 istio.io/operator v0.0.0-20200714085832-f408beefc360 - k8s.io/api v0.30.3 + k8s.io/api v0.30.4 k8s.io/apiextensions-apiserver v0.30.1 - k8s.io/apimachinery v0.30.3 + k8s.io/apimachinery v0.30.4 k8s.io/client-go v12.0.0+incompatible k8s.io/pod-security-admission v0.22.0-beta.0.0.20240531013614-68e02f3c6735 kubevirt.io/application-aware-quota-api v1.2.3 kubevirt.io/containerized-data-importer-api v1.59.1-0.20240610172909-253d5a6e7f95 + sigs.k8s.io/kwok v0.6.1 sigs.k8s.io/yaml v1.4.0 ) require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.4.0 // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containernetworking/cni v1.2.0-rc1 // indirect + github.com/containernetworking/cni v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect @@ -87,7 +87,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/natefinch/lumberjack v2.0.0+incompatible // indirect - github.com/nxadm/tail v1.4.11 // indirect github.com/openshift/api v0.0.0 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect @@ -99,19 +98,19 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/grpc v1.62.1 // indirect + google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.5.1 // indirect istio.io/api v1.22.0 // indirect istio.io/pkg v0.0.0-20191029184635-5c2f5ef63692 // indirect - k8s.io/component-base v0.30.1 // indirect + k8s.io/component-base v0.30.4 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect kubevirt.io/application-aware-quota v0.0.0-20240609100014-e72e988ec98a // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/controller-runtime v0.17.2 // indirect + sigs.k8s.io/controller-runtime v0.18.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) @@ -125,7 +124,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -133,7 +132,7 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.22.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect; gindirect ) diff --git a/cluster-provision/gocli/go.sum b/cluster-provision/gocli/go.sum index b2fbed04a0..30a3090619 100644 --- a/cluster-provision/gocli/go.sum +++ b/cluster-provision/gocli/go.sum @@ -625,8 +625,6 @@ github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig v2.14.1+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -721,8 +719,8 @@ github.com/containerd/containerd v1.7.17 h1:KjNnn0+tAVQHAoaWRjmdak9WlvnFR/8rU1CH github.com/containerd/containerd v1.7.17/go.mod h1:vK+hhT4TIv2uejlcDlbVIc8+h/BqtKLIyNrtCZol8lI= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containernetworking/cni v1.2.0-rc1 h1:AKI3+pXtgY4PDLN9+50o9IaywWVuey0Jkw3Lvzp0HCY= -github.com/containernetworking/cni v1.2.0-rc1/go.mod h1:Lt0TQcZQVDju64fYxUhDziTgXCDe3Olzi9I4zZJLWHg= +github.com/containernetworking/cni v1.2.2 h1:9IbP6KJQQxVKo4hhnm8r50YcVKrJbJu3Dqw+Rbt1vYk= +github.com/containernetworking/cni v1.2.2/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -739,7 +737,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -1198,8 +1196,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1348,8 +1346,8 @@ github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= -github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc= -github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1536,8 +1534,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -2204,8 +2202,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2437,15 +2435,15 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go. google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= -google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -2494,8 +2492,8 @@ google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwS google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2591,8 +2589,8 @@ k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/api v0.30.4 h1:XASIELmW8w8q0i1Y4124LqPoWMycLjyQti/fdYHYjCs= +k8s.io/api v0.30.4/go.mod h1:ZqniWRKu7WIeLijbbzetF4U9qZ03cg5IRwl8YVs8mX0= k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= @@ -2609,8 +2607,8 @@ k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu7 k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.30.4 h1:5QHQI2tInzr8LsT4kU/2+fSeibH1eIHswNx480cqIoY= +k8s.io/apimachinery v0.30.4/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w= k8s.io/cli-runtime v0.28.4/go.mod h1:MLGRB7LWTIYyYR3d/DOgtUC8ihsAPA3P8K8FDNIqJ0k= k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= @@ -2727,8 +2725,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/controller-runtime v0.2.2/go.mod h1:9dyohw3ZtoXQuV1e766PHUn+cmrRCIcBh6XIMFNMZ+I= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= -sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= -sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= @@ -2739,6 +2737,8 @@ sigs.k8s.io/kustomize/cmd/config v0.11.2/go.mod h1:PCpHxyu10daTnbMfn3xhH1vppn7L8 sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3/go.mod h1:/d88dHCvoy7d0AKFT0yytezSGZKjsZBVs9YTkBHSGFk= sigs.k8s.io/kustomize/kyaml v0.14.2/go.mod h1:AN1/IpawKilWD7V+YvQwRGUvuUOOWpjsHu6uHwonSF4= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= +sigs.k8s.io/kwok v0.6.1 h1:Ar8u7Y5TWvvOktAWapXRWJxwiwy9XFolJd4IPDR3fPg= +sigs.k8s.io/kwok v0.6.1/go.mod h1:1ZydjiCOLly7ogqEmwN8YBgbF7bC8Z7BKURLHXPVrFc= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/cluster-provision/gocli/opts/kwok/kwok.go b/cluster-provision/gocli/opts/kwok/kwok.go new file mode 100644 index 0000000000..4ac0c62e48 --- /dev/null +++ b/cluster-provision/gocli/opts/kwok/kwok.go @@ -0,0 +1,47 @@ +package kwok + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed manifests/kwok.yaml +var kw []byte + +//go:embed manifests/stage-fast.yaml +var sf []byte + +type KwokOpt struct { + client k8s.K8sDynamicClient +} + +func NewKwokOpt(client k8s.K8sDynamicClient) *KwokOpt { + return &KwokOpt{ + client: client, + } +} + +func (k *KwokOpt) Exec() error { + for _, yamlData := range [][]byte{kw, sf} { + yamlDocs := bytes.Split(yamlData, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := k.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + } + return nil +} diff --git a/cluster-provision/gocli/opts/kwok/manifests/kwok.yaml b/cluster-provision/gocli/opts/kwok/manifests/kwok.yaml new file mode 100644 index 0000000000..6710e1dd86 --- /dev/null +++ b/cluster-provision/gocli/opts/kwok/manifests/kwok.yaml @@ -0,0 +1,2408 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: attaches.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: Attach + listKind: AttachList + plural: attaches + singular: attach + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Attach provides attach configuration for a single pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for attach + properties: + attaches: + description: Attaches is a list of attaches to configure. + items: + description: AttachConfig holds information how to attach. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + logsFile: + description: LogsFile is the file from which the attach starts + type: string + type: object + type: array + required: + - attaches + type: object + status: + description: Status holds status for attach + properties: + conditions: + description: Conditions holds conditions for attach + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: clusterattaches.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ClusterAttach + listKind: ClusterAttachList + plural: clusterattaches + singular: clusterattach + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterAttach provides cluster-wide logging configuration + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for cluster attach. + properties: + attaches: + description: Attaches is a list of attach configurations. + items: + description: AttachConfig holds information how to attach. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + logsFile: + description: LogsFile is the file from which the attach starts + type: string + type: object + type: array + selector: + description: Selector is a selector to filter pods to configure. + properties: + matchNames: + description: |- + MatchNames is a list of names to match. + if not set, all names will be matched. + items: + type: string + type: array + matchNamespaces: + description: |- + MatchNamespaces is a list of namespaces to match. + if not set, all namespaces will be matched. + items: + type: string + type: array + type: object + required: + - attaches + type: object + status: + description: Status holds status for cluster attach + properties: + conditions: + description: Conditions holds conditions for cluster attach. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: clusterexecs.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ClusterExec + listKind: ClusterExecList + plural: clusterexecs + singular: clusterexec + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterExec provides cluster-wide exec configuration. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for cluster exec. + properties: + execs: + description: Execs is a list of exec to configure. + items: + description: ExecTarget holds information how to exec. + properties: + containers: + description: |- + Containers is a list of containers to exec. + if not set, all containers will be execed. + items: + type: string + type: array + local: + description: Local holds information how to exec to a local + target. + properties: + envs: + description: Envs is a list of environment variables to + exec with. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + minLength: 1 + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + securityContext: + description: SecurityContext is the user context to exec. + properties: + runAsGroup: + description: RunAsGroup is the existing gid to run exec + command in container process. + format: int64 + type: integer + runAsUser: + description: RunAsUser is the existing uid to run exec + command in container process. + format: int64 + type: integer + type: object + workDir: + description: WorkDir is the working directory to exec with. + type: string + type: object + type: object + type: array + selector: + description: Selector is a selector to filter pods to configure. + properties: + matchNames: + description: |- + MatchNames is a list of names to match. + if not set, all names will be matched. + items: + type: string + type: array + matchNamespaces: + description: |- + MatchNamespaces is a list of namespaces to match. + if not set, all namespaces will be matched. + items: + type: string + type: array + type: object + required: + - execs + type: object + status: + description: Status holds status for cluster exec + properties: + conditions: + description: Conditions holds conditions for cluster exec. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: clusterlogs.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ClusterLogs + listKind: ClusterLogsList + plural: clusterlogs + singular: clusterlogs + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterLogs provides cluster-wide logging configuration + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for cluster logs. + properties: + logs: + description: Forwards is a list of log configurations. + items: + description: Log holds information how to forward logs. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + follow: + description: Follow up if true + type: boolean + logsFile: + description: LogsFile is the file from which the log forward + starts + type: string + type: object + type: array + selector: + description: Selector is a selector to filter pods to configure. + properties: + matchNames: + description: |- + MatchNames is a list of names to match. + if not set, all names will be matched. + items: + type: string + type: array + matchNamespaces: + description: |- + MatchNamespaces is a list of namespaces to match. + if not set, all namespaces will be matched. + items: + type: string + type: array + type: object + required: + - logs + type: object + status: + description: Status holds status for cluster logs + properties: + conditions: + description: Conditions holds conditions for cluster logs. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: clusterportforwards.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ClusterPortForward + listKind: ClusterPortForwardList + plural: clusterportforwards + singular: clusterportforward + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterPortForward provides cluster-wide port forward configuration. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for cluster port forward. + properties: + forwards: + description: Forwards is a list of forwards to configure. + items: + description: Forward holds information how to forward based on ports. + properties: + command: + description: |- + Command is the command to run to forward with stdin/stdout. + if set, Target will be ignored. + items: + type: string + type: array + ports: + description: |- + Ports is a list of ports to forward. + if not set, all ports will be forwarded. + items: + format: int32 + type: integer + type: array + target: + description: Target is the target to forward to. + properties: + address: + description: Address is the address to forward to. + minLength: 1 + type: string + port: + description: Port is the port to forward to. + format: int32 + maximum: 65535 + minimum: 0 + type: integer + required: + - address + - port + type: object + type: object + type: array + selector: + description: Selector is a selector to filter pods to configure. + properties: + matchNames: + description: |- + MatchNames is a list of names to match. + if not set, all names will be matched. + items: + type: string + type: array + matchNamespaces: + description: |- + MatchNamespaces is a list of namespaces to match. + if not set, all namespaces will be matched. + items: + type: string + type: array + type: object + required: + - forwards + type: object + status: + description: Status holds status for cluster port forward + properties: + conditions: + description: Conditions holds conditions for cluster port forward. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: clusterresourceusages.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ClusterResourceUsage + listKind: ClusterResourceUsageList + plural: clusterresourceusages + singular: clusterresourceusage + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterResourceUsage provides cluster-wide resource usage. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for cluster resource usage. + properties: + selector: + description: Selector is a selector to filter pods to configure. + properties: + matchNames: + description: |- + MatchNames is a list of names to match. + if not set, all names will be matched. + items: + type: string + type: array + matchNamespaces: + description: |- + MatchNamespaces is a list of namespaces to match. + if not set, all namespaces will be matched. + items: + type: string + type: array + type: object + usages: + description: Usages is a list of resource usage for the pod. + items: + description: ResourceUsageContainer holds spec for resource usage + container. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + usage: + additionalProperties: + description: ResourceUsageValue holds value for resource usage. + properties: + expression: + description: Expression is the expression for resource + usage. + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the value for resource usage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + description: Usage is a list of resource usage for the container. + type: object + type: object + type: array + type: object + status: + description: Status holds status for cluster resource usage + properties: + conditions: + description: Conditions holds conditions for cluster resource usage + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: execs.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: Exec + listKind: ExecList + plural: execs + singular: exec + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Exec provides exec configuration for a single pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for exec + properties: + execs: + description: Execs is a list of execs to configure. + items: + description: ExecTarget holds information how to exec. + properties: + containers: + description: |- + Containers is a list of containers to exec. + if not set, all containers will be execed. + items: + type: string + type: array + local: + description: Local holds information how to exec to a local + target. + properties: + envs: + description: Envs is a list of environment variables to + exec with. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + minLength: 1 + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + securityContext: + description: SecurityContext is the user context to exec. + properties: + runAsGroup: + description: RunAsGroup is the existing gid to run exec + command in container process. + format: int64 + type: integer + runAsUser: + description: RunAsUser is the existing uid to run exec + command in container process. + format: int64 + type: integer + type: object + workDir: + description: WorkDir is the working directory to exec with. + type: string + type: object + type: object + type: array + required: + - execs + type: object + status: + description: Status holds status for exec + properties: + conditions: + description: Conditions holds conditions for exec + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: logs.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: Logs + listKind: LogsList + plural: logs + singular: logs + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Logs provides logging configuration for a single pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for logs + properties: + logs: + description: Logs is a list of logs to configure. + items: + description: Log holds information how to forward logs. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + follow: + description: Follow up if true + type: boolean + logsFile: + description: LogsFile is the file from which the log forward + starts + type: string + type: object + type: array + required: + - logs + type: object + status: + description: Status holds status for logs + properties: + conditions: + description: Conditions holds conditions for logs + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: metrics.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: Metric + listKind: MetricList + plural: metrics + singular: metric + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Metric provides metrics configuration. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for metrics. + properties: + metrics: + description: Metrics is a list of metric configurations. + items: + description: MetricConfig provides metric configuration to a single + metric + properties: + buckets: + description: Buckets is a list of buckets for a histogram metric. + items: + description: MetricBucket is a single bucket for a metric. + properties: + hidden: + description: |- + Hidden is means that this bucket not shown in the metric. + but value will be calculated and cumulative into the next bucket. + type: boolean + le: + description: Le is less-than or equal. + minimum: 0 + type: number + value: + description: Value is a CEL expression. + type: string + required: + - le + - value + type: object + type: array + x-kubernetes-list-map-keys: + - le + x-kubernetes-list-type: map + dimension: + description: Dimension is a dimension of the metric. + type: string + help: + description: Help provides information about this metric. + type: string + kind: + description: Kind is kind of metric + enum: + - counter + - gauge + - histogram + type: string + labels: + description: Labels are metric labels. + items: + description: MetricLabel holds label name and the value of + the label. + properties: + name: + description: Name is a label name. + minLength: 1 + type: string + value: + description: Value is a CEL expression. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + name: + description: Name is the fully-qualified name of the metric. + minLength: 1 + type: string + value: + description: Value is a CEL expression. + type: string + required: + - kind + - name + type: object + type: array + path: + description: Path is a restful service path. + minLength: 1 + type: string + required: + - metrics + - path + type: object + status: + description: Status holds status for metrics + properties: + conditions: + description: Conditions holds conditions for metrics. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: portforwards.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: PortForward + listKind: PortForwardList + plural: portforwards + singular: portforward + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PortForward provides port forward configuration for a single + pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for port forward. + properties: + forwards: + description: Forwards is a list of forwards to configure. + items: + description: Forward holds information how to forward based on ports. + properties: + command: + description: |- + Command is the command to run to forward with stdin/stdout. + if set, Target will be ignored. + items: + type: string + type: array + ports: + description: |- + Ports is a list of ports to forward. + if not set, all ports will be forwarded. + items: + format: int32 + type: integer + type: array + target: + description: Target is the target to forward to. + properties: + address: + description: Address is the address to forward to. + minLength: 1 + type: string + port: + description: Port is the port to forward to. + format: int32 + maximum: 65535 + minimum: 0 + type: integer + required: + - address + - port + type: object + type: object + type: array + required: + - forwards + type: object + status: + description: Status holds status for port forward + properties: + conditions: + description: Conditions holds conditions for port forward + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: resourceusages.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: ResourceUsage + listKind: ResourceUsageList + plural: resourceusages + singular: resourceusage + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ResourceUsage provides resource usage for a single pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds spec for resource usage. + properties: + usages: + description: Usages is a list of resource usage for the pod. + items: + description: ResourceUsageContainer holds spec for resource usage + container. + properties: + containers: + description: Containers is list of container names. + items: + type: string + type: array + usage: + additionalProperties: + description: ResourceUsageValue holds value for resource usage. + properties: + expression: + description: Expression is the expression for resource + usage. + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the value for resource usage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + description: Usage is a list of resource usage for the container. + type: object + type: object + type: array + type: object + status: + description: Status holds status for resource usage + properties: + conditions: + description: Conditions holds conditions for resource usage + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app: kwok-controller + name: stages.kwok.x-k8s.io +spec: + group: kwok.x-k8s.io + names: + kind: Stage + listKind: StageList + plural: stages + singular: stage + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Stage is an API that describes the staged change of a resource + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec holds information about the request being evaluated. + properties: + delay: + description: Delay means there is a delay in this stage. + properties: + durationFrom: + description: |- + DurationFrom is the expression used to get the value. + If it is a time.Time type, getting the value will be minus time.Now() to get DurationMilliseconds + If it is a string type, the value get will be parsed by time.ParseDuration. + properties: + expressionFrom: + description: ExpressionFrom is the expression used to get + the value. + type: string + type: object + durationMilliseconds: + description: |- + DurationMilliseconds indicates the stage delay time. + If JitterDurationMilliseconds is less than DurationMilliseconds, then JitterDurationMilliseconds is used. + format: int64 + minimum: 0 + type: integer + jitterDurationFrom: + description: |- + JitterDurationFrom is the expression used to get the value. + If it is a time.Time type, getting the value will be minus time.Now() to get JitterDurationMilliseconds + If it is a string type, the value get will be parsed by time.ParseDuration. + properties: + expressionFrom: + description: ExpressionFrom is the expression used to get + the value. + type: string + type: object + jitterDurationMilliseconds: + description: |- + JitterDurationMilliseconds is the duration plus an additional amount chosen uniformly + at random from the interval between DurationMilliseconds and JitterDurationMilliseconds. + format: int64 + minimum: 0 + type: integer + type: object + immediateNextStage: + description: ImmediateNextStage means that the next stage of matching + is performed immediately, without waiting for the Apiserver to push. + type: boolean + next: + description: Next indicates that this stage will be moved to. + properties: + delete: + description: Delete means that the resource will be deleted if + true. + type: boolean + event: + description: Event means that an event will be sent. + properties: + message: + description: Message is a human-readable description of the + status of this operation. + type: string + reason: + description: Reason is why the action was taken. It is human-readable. + type: string + type: + description: Type is the type of this event (Normal, Warning), + It is machine-readable. + type: string + type: object + finalizers: + description: Finalizers means that finalizers will be modified. + properties: + add: + description: Add means that the Finalizers will be added to + the resource. + items: + description: FinalizerItem describes the one of the finalizers. + properties: + value: + description: Value is the value of the finalizer. + type: string + type: object + type: array + empty: + description: Empty means that the Finalizers for that resource + will be emptied. + type: boolean + remove: + description: Remove means that the Finalizers will be removed + from the resource. + items: + description: FinalizerItem describes the one of the finalizers. + properties: + value: + description: Value is the value of the finalizer. + type: string + type: object + type: array + type: object + statusPatchAs: + description: |- + StatusPatchAs indicates the impersonating configuration for client when patching status. + In most cases this will be empty, in which case the default client service account will be used. + When this is not empty, a corresponding rbac change is required to grant `impersonate` privilege. + The support for this field is not available in Pod and Node resources. + properties: + username: + description: Username the target username for the client to + impersonate + type: string + required: + - username + type: object + statusSubresource: + default: status + description: |- + StatusSubresource indicates the name of the subresource that will be patched. The support for + this field is not available in Pod and Node resources. + type: string + statusTemplate: + description: StatusTemplate indicates the template for modifying + the status of the resource in the next. + type: string + type: object + resourceRef: + description: ResourceRef specifies the Kind and version of the resource. + properties: + apiGroup: + default: v1 + description: APIGroup of the referent. + type: string + kind: + description: Kind of the referent. + type: string + required: + - kind + type: object + selector: + description: Selector specifies the stags will be applied to the selected + resource. + properties: + matchAnnotations: + additionalProperties: + type: string + description: |- + MatchAnnotations is a map of {key,value} pairs. A single {key,value} in the matchAnnotations + map is equivalent to an element of matchExpressions, whose key field is ".metadata.annotations[key]", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + matchExpressions: + description: MatchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + SelectorRequirement is a resource selector requirement is a selector that contains values, a key, + and an operator that relates the key and values. + properties: + key: + description: The name of the scope that the selector applies + to. + type: string + operator: + description: Represents a scope's relationship to a set + of values. + type: string + values: + description: |- + An array of string values. + If the operator is In, NotIn, Intersection or NotIntersection, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values array must be empty. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + MatchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is ".metadata.labels[key]", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + weight: + default: 0 + description: |- + Weight means when multiple stages share the same ResourceRef and Selector, + a random stage will be matched as the next stage based on the weight. + minimum: 0 + type: integer + required: + - next + - resourceRef + type: object + status: + description: Status holds status for the Stage + properties: + conditions: + description: Conditions holds conditions for the Stage. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: kwok-controller + name: kwok-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: kwok-controller + name: kwok-controller +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - attaches + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - attaches/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterattaches + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterattaches/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterexecs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterexecs/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterlogs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterlogs/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterportforwards + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterportforwards/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterresourceusages + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - clusterresourceusages/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - execs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - execs/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - logs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - logs/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - metrics + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - metrics/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - portforwards + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - portforwards/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - resourceusages + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - resourceusages/status + verbs: + - patch + - update +- apiGroups: + - kwok.x-k8s.io + resources: + - stages + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kwok.x-k8s.io + resources: + - stages/status + verbs: + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: kwok-controller + name: kwok-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kwok-controller +subjects: +- kind: ServiceAccount + name: kwok-controller + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kwok-controller + name: kwok-controller + namespace: kube-system +spec: + ports: + - name: http + port: 10247 + protocol: TCP + targetPort: 10247 + selector: + app: kwok-controller + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kwok-controller + name: kwok-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: kwok-controller + template: + metadata: + labels: + app: kwok-controller + spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --manage-nodes-with-label-selector= + - --manage-single-node= + - --disregard-status-with-annotation-selector=kwok.x-k8s.io/status=custom + - --disregard-status-with-label-selector= + - --node-ip=$(POD_IP) + - --node-port=10247 + - --cidr=10.0.0.1/24 + - --node-lease-duration-seconds=40 + - --enable-crds=Stage + - --enable-crds=Metric + - --enable-crds=Attach + - --enable-crds=ClusterAttach + - --enable-crds=Exec + - --enable-crds=ClusterExec + - --enable-crds=Logs + - --enable-crds=ClusterLogs + - --enable-crds=PortForward + - --enable-crds=ClusterPortForward + - --enable-crds=ResourceUsage + - --enable-crds=ClusterResourceUsage + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: registry.k8s.io/kwok/kwok:v0.5.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 10247 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 10 + name: kwok-controller + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10247 + scheme: HTTP + initialDelaySeconds: 2 + periodSeconds: 20 + timeoutSeconds: 2 + startupProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10247 + scheme: HTTP + initialDelaySeconds: 2 + periodSeconds: 10 + timeoutSeconds: 2 + restartPolicy: Always + serviceAccountName: kwok-controller diff --git a/cluster-provision/gocli/opts/kwok/manifests/stage-fast.yaml b/cluster-provision/gocli/opts/kwok/manifests/stage-fast.yaml new file mode 100644 index 0000000000..e39b0a7e50 --- /dev/null +++ b/cluster-provision/gocli/opts/kwok/manifests/stage-fast.yaml @@ -0,0 +1,255 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: node-heartbeat-with-lease +spec: + delay: + durationMilliseconds: 600000 + jitterDurationMilliseconds: 610000 + next: + statusTemplate: | + {{ $now := Now }} + {{ $lastTransitionTime := or .metadata.creationTimestamp $now }} + conditions: + {{ range NodeConditions }} + - lastHeartbeatTime: {{ $now | Quote }} + lastTransitionTime: {{ $lastTransitionTime | Quote }} + message: {{ .message | Quote }} + reason: {{ .reason | Quote }} + status: {{ .status | Quote }} + type: {{ .type | Quote }} + {{ end }} + + addresses: + {{ with .status.addresses }} + {{ YAML . 1 }} + {{ else }} + {{ with NodeIP }} + - address: {{ . | Quote }} + type: InternalIP + {{ end }} + {{ with NodeName }} + - address: {{ . | Quote }} + type: Hostname + {{ end }} + {{ end }} + + {{ with NodePort }} + daemonEndpoints: + kubeletEndpoint: + Port: {{ . }} + {{ end }} + resourceRef: + apiGroup: v1 + kind: Node + selector: + matchExpressions: + - key: .status.phase + operator: In + values: + - Running + - key: .status.conditions.[] | select( .type == "Ready" ) | .status + operator: In + values: + - "True" +--- +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: node-initialize +spec: + next: + statusTemplate: | + {{ $now := Now }} + {{ $lastTransitionTime := or .metadata.creationTimestamp $now }} + conditions: + {{ range NodeConditions }} + - lastHeartbeatTime: {{ $now | Quote }} + lastTransitionTime: {{ $lastTransitionTime | Quote }} + message: {{ .message | Quote }} + reason: {{ .reason | Quote }} + status: {{ .status | Quote }} + type: {{ .type | Quote}} + {{ end }} + + addresses: + {{ with .status.addresses }} + {{ YAML . 1 }} + {{ else }} + {{ with NodeIP }} + - address: {{ . | Quote }} + type: InternalIP + {{ end }} + {{ with NodeName }} + - address: {{ . | Quote }} + type: Hostname + {{ end }} + {{ end }} + + {{ with NodePort }} + daemonEndpoints: + kubeletEndpoint: + Port: {{ . }} + {{ end }} + + allocatable: + {{ with .status.allocatable }} + {{ YAML . 1 }} + {{ else }} + cpu: 1k + memory: 1Ti + pods: 1M + {{ end }} + capacity: + {{ with .status.capacity }} + {{ YAML . 1 }} + {{ else }} + cpu: 1k + memory: 1Ti + pods: 1M + {{ end }} + {{ with .status.nodeInfo }} + nodeInfo: + architecture: {{ with .architecture }} {{ . }} {{ else }} "amd64" {{ end }} + bootID: {{ with .bootID }} {{ . }} {{ else }} "" {{ end }} + containerRuntimeVersion: {{ with .containerRuntimeVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kernelVersion: {{ with .kernelVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kubeProxyVersion: {{ with .kubeProxyVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kubeletVersion: {{ with .kubeletVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + machineID: {{ with .machineID }} {{ . }} {{ else }} "" {{ end }} + operatingSystem: {{ with .operatingSystem }} {{ . }} {{ else }} "linux" {{ end }} + osImage: {{ with .osImage }} {{ . }} {{ else }} "" {{ end }} + systemUUID: {{ with .systemUUID }} {{ . }} {{ else }} "" {{ end }} + {{ end }} + phase: Running + resourceRef: + apiGroup: v1 + kind: Node + selector: + matchExpressions: + - key: .status.conditions.[] | select( .type == "Ready" ) | .status + operator: NotIn + values: + - "True" +--- +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-complete +spec: + next: + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + containerStatuses: + {{ range $index, $item := .spec.containers }} + {{ $origin := index $root.status.containerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: false + restartCount: 0 + started: false + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + phase: Succeeded + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: .metadata.deletionTimestamp + operator: DoesNotExist + - key: .status.phase + operator: In + values: + - Running + - key: .metadata.ownerReferences.[].kind + operator: In + values: + - Job +--- +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-delete +spec: + next: + delete: true + finalizers: + empty: true + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: .metadata.deletionTimestamp + operator: Exists +--- +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-ready +spec: + next: + statusTemplate: | + {{ $now := Now }} + + conditions: + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Initialized + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Ready + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: ContainersReady + {{ range .spec.readinessGates }} + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: {{ .conditionType | Quote }} + {{ end }} + + containerStatuses: + {{ range .spec.containers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: true + restartCount: 0 + state: + running: + startedAt: {{ $now | Quote }} + {{ end }} + + initContainerStatuses: + {{ range .spec.initContainers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: true + restartCount: 0 + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + + hostIP: {{ NodeIPWith .spec.nodeName | Quote }} + podIP: {{ PodIPWith .spec.nodeName ( or .spec.hostNetwork false ) ( or .metadata.uid "" ) ( or .metadata.name "" ) ( or .metadata.namespace "" ) | Quote }} + phase: Running + startTime: {{ $now | Quote }} + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: .metadata.deletionTimestamp + operator: DoesNotExist + - key: .status.podIP + operator: DoesNotExist diff --git a/cluster-provision/gocli/pkg/k8s/k8s.go b/cluster-provision/gocli/pkg/k8s/k8s.go index c5f91e1256..e5c68578db 100644 --- a/cluster-provision/gocli/pkg/k8s/k8s.go +++ b/cluster-provision/gocli/pkg/k8s/k8s.go @@ -14,6 +14,7 @@ import ( admissionv1 "k8s.io/pod-security-admission/admission/api/v1" aaqv1alpha1 "kubevirt.io/application-aware-quota-api/pkg/apis/core/v1alpha1" cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + kwokv1alpha1 "sigs.k8s.io/kwok/pkg/apis/v1alpha1" "github.com/cenkalti/backoff/v4" corev1 "k8s.io/api/core/v1" @@ -188,6 +189,7 @@ func initSchema() *runtime.Scheme { _ = monitoringv1alpha1.AddToScheme(s) _ = monitoringv1.AddToScheme(s) _ = istiov1alpha1.AddToScheme(s) + _ = kwokv1alpha1.AddToScheme(s) _ = admissionv1.AddToScheme(s) _ = cdiv1beta1.AddToScheme(s) _ = aaqv1alpha1.AddToScheme(s) diff --git a/cluster-provision/gocli/providers/opts.go b/cluster-provision/gocli/providers/opts.go index bd2851ff03..fd2f065760 100644 --- a/cluster-provision/gocli/providers/opts.go +++ b/cluster-provision/gocli/providers/opts.go @@ -196,6 +196,11 @@ func WithMultus(multus interface{}) KubevirtProviderOption { c.EnableMultus = multus.(bool) } } +func WithKwok(kwok interface{}) KubevirtProviderOption { + return func(c *KubevirtProvider) { + c.EnableKwok = kwok.(bool) + } +} func WithAAQ(aaq interface{}) KubevirtProviderOption { return func(c *KubevirtProvider) { c.AAQ = aaq.(bool) diff --git a/cluster-provision/gocli/providers/start.go b/cluster-provision/gocli/providers/start.go index 8a8de8bab6..8303728d5d 100644 --- a/cluster-provision/gocli/providers/start.go +++ b/cluster-provision/gocli/providers/start.go @@ -25,6 +25,7 @@ import ( etcd "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/kwok" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/localvolume" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" @@ -387,6 +388,10 @@ func (kp *KubevirtProvider) provisionK8sOpts(sshClient libssh.Client) error { opts = append(opts, istio.NewIstioOpt(sshClient, kp.Client, kp.EnableCNAO)) } + if kp.EnableKwok { + opts = append(opts, kwok.NewKwokOpt(kp.Client)) + } + for _, opt := range opts { if err := opt.Exec(); err != nil { return err diff --git a/cluster-provision/gocli/providers/types.go b/cluster-provision/gocli/providers/types.go index 8da12d58d5..24828500d4 100644 --- a/cluster-provision/gocli/providers/types.go +++ b/cluster-provision/gocli/providers/types.go @@ -37,6 +37,7 @@ type KubevirtProvider struct { NFSData string `flag:"nfs-data" json:"nfs_data"` EnableCeph bool `flag:"enable-ceph" json:"enable_ceph"` EnableIstio bool `flag:"enable-istio" json:"enable_istio"` + EnableKwok bool `flag:"deploy-kwok" json:"deploy_kwok"` EnableCNAO bool `flag:"enable-cnao" json:"enable_cnao"` SkipCnaoCR bool `flag:"skip-cnao-cr" json:"skip_cnao_cr"` EnableNFSCSI bool `flag:"enable-nfs-csi" json:"enable_nfs_csi"` @@ -289,6 +290,10 @@ var RunFlagMap = map[string]FlagConfig{ FlagType: "bool", ProviderOptFunc: WithMultus, }, + "deploy-kwok": { + FlagType: "bool", + ProviderOptFunc: WithKwok, + }, "deploy-aaq": { FlagType: "bool", ProviderOptFunc: WithAAQ, diff --git a/cluster-provision/gocli/vendor/modules.txt b/cluster-provision/gocli/vendor/modules.txt index f0a54cec8b..8e5a231650 100644 --- a/cluster-provision/gocli/vendor/modules.txt +++ b/cluster-provision/gocli/vendor/modules.txt @@ -2,9 +2,6 @@ ## explicit; go 1.16 # github.com/BurntSushi/toml v1.4.0 ## explicit; go 1.18 -# github.com/Masterminds/semver/v3 v3.2.1 -## explicit; go 1.18 -github.com/Masterminds/semver/v3 # github.com/Microsoft/go-winio v0.6.2 ## explicit; go 1.21 github.com/Microsoft/go-winio @@ -32,8 +29,8 @@ github.com/containerd/containerd/pkg/userns # github.com/containerd/log v0.1.0 ## explicit; go 1.20 github.com/containerd/log -# github.com/containernetworking/cni v1.2.0-rc1 -## explicit; go 1.18 +# github.com/containernetworking/cni v1.2.2 +## explicit; go 1.21 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke github.com/containernetworking/cni/pkg/types @@ -225,7 +222,7 @@ github.com/json-iterator/go github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1 github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils -# github.com/klauspost/compress v1.17.8 +# github.com/klauspost/compress v1.17.9 ## explicit; go 1.20 github.com/klauspost/compress github.com/klauspost/compress/fse @@ -282,9 +279,7 @@ github.com/munnerz/goautoneg # github.com/natefinch/lumberjack v2.0.0+incompatible ## explicit github.com/natefinch/lumberjack -# github.com/nxadm/tail v1.4.11 -## explicit; go 1.13 -# github.com/onsi/ginkgo/v2 v2.18.0 +# github.com/onsi/ginkgo/v2 v2.19.0 ## explicit; go 1.20 github.com/onsi/ginkgo/v2 github.com/onsi/ginkgo/v2/config @@ -355,7 +350,7 @@ github.com/sirupsen/logrus github.com/spf13/afero github.com/spf13/afero/internal/common github.com/spf13/afero/mem -# github.com/spf13/cobra v1.8.0 +# github.com/spf13/cobra v1.8.1 ## explicit; go 1.15 github.com/spf13/cobra # github.com/spf13/pflag v1.0.5 @@ -463,11 +458,11 @@ golang.org/x/text/unicode/norm # golang.org/x/time v0.5.0 ## explicit; go 1.18 golang.org/x/time/rate -# golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d +# golang.org/x/tools v0.22.0 ## explicit; go 1.19 golang.org/x/tools/cover golang.org/x/tools/go/ast/inspector -# google.golang.org/grpc v1.62.1 +# google.golang.org/grpc v1.64.0 ## explicit; go 1.19 google.golang.org/grpc/grpclog google.golang.org/grpc/internal/grpclog @@ -531,7 +526,7 @@ istio.io/operator/pkg/apis/istio/v1alpha1 # istio.io/pkg v0.0.0-20191029184635-5c2f5ef63692 ## explicit; go 1.12 istio.io/pkg/log -# k8s.io/api v0.30.3 +# k8s.io/api v0.30.4 ## explicit; go 1.22.0 k8s.io/api/admission/v1 k8s.io/api/admissionregistration/v1 @@ -592,7 +587,7 @@ k8s.io/api/storagemigration/v1alpha1 ## explicit; go 1.20 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 -# k8s.io/apimachinery v0.30.3 +# k8s.io/apimachinery v0.30.4 ## explicit; go 1.22.0 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -771,7 +766,7 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/keyutil k8s.io/client-go/util/retry k8s.io/client-go/util/workqueue -# k8s.io/component-base v0.30.1 => k8s.io/component-base v0.28.4 +# k8s.io/component-base v0.30.4 => k8s.io/component-base v0.28.4 ## explicit; go 1.20 k8s.io/component-base/version # k8s.io/klog/v2 v2.130.1 @@ -821,13 +816,16 @@ kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1 # kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 ## explicit; go 1.17 kubevirt.io/controller-lifecycle-operator-sdk/api -# sigs.k8s.io/controller-runtime v0.17.2 -## explicit; go 1.21 +# sigs.k8s.io/controller-runtime v0.18.4 +## explicit; go 1.22.0 sigs.k8s.io/controller-runtime/pkg/scheme # sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd ## explicit; go 1.18 sigs.k8s.io/json sigs.k8s.io/json/internal/golang/encoding/json +# sigs.k8s.io/kwok v0.6.1 +## explicit; go 1.22.3 +sigs.k8s.io/kwok/pkg/apis/v1alpha1 # sigs.k8s.io/structured-merge-diff/v4 v4.4.1 ## explicit; go 1.13 sigs.k8s.io/structured-merge-diff/v4/fieldpath diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/LICENSE b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/attach_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/attach_types.go new file mode 100644 index 0000000000..c2f4dc392a --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/attach_types.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // AttachKind is the kind of the Logs. + AttachKind = "Attach" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +kubebuilder:subresource:status +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=attaches,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=attaches/status,verbs=update;patch + +// Attach provides attach configuration for a single pod. +type Attach struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for attach + Spec AttachSpec `json:"spec"` + // Status holds status for attach + //+k8s:conversion-gen=false + Status AttachStatus `json:"status,omitempty"` +} + +// AttachStatus holds status for attach +type AttachStatus struct { + // Conditions holds conditions for attach + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// AttachSpec holds spec for attach. +type AttachSpec struct { + // Attaches is a list of attaches to configure. + Attaches []AttachConfig `json:"attaches"` +} + +// AttachConfig holds information how to attach. +type AttachConfig struct { + // Containers is list of container names. + Containers []string `json:"containers,omitempty"` + // LogsFile is the file from which the attach starts + LogsFile *string `json:"logsFile,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// AttachList contains a list of Attach +type AttachList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Attach `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Attach{}, &AttachList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_attach_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_attach_types.go new file mode 100644 index 0000000000..1036f82a3a --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_attach_types.go @@ -0,0 +1,76 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ClusterAttachKind is the kind of the ClusterAttachKind. + ClusterAttachKind = "ClusterAttach" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterattaches,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterattaches/status,verbs=update;patch + +// ClusterAttach provides cluster-wide logging configuration +type ClusterAttach struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for cluster attach. + Spec ClusterAttachSpec `json:"spec"` + // Status holds status for cluster attach + //+k8s:conversion-gen=false + Status ClusterAttachStatus `json:"status,omitempty"` +} + +// ClusterAttachStatus holds status for cluster attach +type ClusterAttachStatus struct { + // Conditions holds conditions for cluster attach. + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ClusterAttachSpec holds spec for cluster attach. +type ClusterAttachSpec struct { + // Selector is a selector to filter pods to configure. + Selector *ObjectSelector `json:"selector,omitempty"` + // Attaches is a list of attach configurations. + Attaches []AttachConfig `json:"attaches"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ClusterAttachList contains a list of ClusterAttach +type ClusterAttachList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterAttach `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterAttach{}, &ClusterAttachList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_exec_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_exec_types.go new file mode 100644 index 0000000000..6e6eca089b --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_exec_types.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ClusterExecKind is the kind of the ClusterExec. + ClusterExecKind = "ClusterExec" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterexecs,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterexecs/status,verbs=update;patch + +// ClusterExec provides cluster-wide exec configuration. +type ClusterExec struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for cluster exec. + Spec ClusterExecSpec `json:"spec"` + // Status holds status for cluster exec + //+k8s:conversion-gen=false + Status ClusterExecStatus `json:"status,omitempty"` +} + +// ClusterExecStatus holds status for cluster exec +type ClusterExecStatus struct { + // Conditions holds conditions for cluster exec. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ClusterExecSpec holds spec for cluster exec. +type ClusterExecSpec struct { + // Selector is a selector to filter pods to configure. + Selector *ObjectSelector `json:"selector,omitempty"` + // Execs is a list of exec to configure. + Execs []ExecTarget `json:"execs"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ClusterExecList contains a list of ClusterExec +type ClusterExecList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterExec `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterExec{}, &ClusterExecList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_logs_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_logs_types.go new file mode 100644 index 0000000000..8c75ef4cee --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_logs_types.go @@ -0,0 +1,79 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ClusterLogsKind is the kind of the ClusterLogsKind. + ClusterLogsKind = "ClusterLogs" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterlogs,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterlogs/status,verbs=update;patch + +// ClusterLogs provides cluster-wide logging configuration +type ClusterLogs struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for cluster logs. + Spec ClusterLogsSpec `json:"spec"` + // Status holds status for cluster logs + //+k8s:conversion-gen=false + Status ClusterLogsStatus `json:"status,omitempty"` +} + +// ClusterLogsStatus holds status for cluster logs +type ClusterLogsStatus struct { + // Conditions holds conditions for cluster logs. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ClusterLogsSpec holds spec for cluster logs. +type ClusterLogsSpec struct { + // Selector is a selector to filter pods to configure. + Selector *ObjectSelector `json:"selector,omitempty"` + // Forwards is a list of log configurations. + Logs []Log `json:"logs"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ClusterLogsList contains a list of ClusterLogs +type ClusterLogsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterLogs `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterLogs{}, &ClusterLogsList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_port_forward_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_port_forward_types.go new file mode 100644 index 0000000000..0f01f07ceb --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_port_forward_types.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ClusterPortForwardKind is the kind of the ClusterPortForward. + ClusterPortForwardKind = "ClusterPortForward" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterportforwards,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterportforwards/status,verbs=update;patch + +// ClusterPortForward provides cluster-wide port forward configuration. +type ClusterPortForward struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for cluster port forward. + Spec ClusterPortForwardSpec `json:"spec"` + // Status holds status for cluster port forward + //+k8s:conversion-gen=false + Status ClusterPortForwardStatus `json:"status,omitempty"` +} + +// ClusterPortForwardStatus holds status for cluster port forward +type ClusterPortForwardStatus struct { + // Conditions holds conditions for cluster port forward. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ClusterPortForwardSpec holds spec for cluster port forward. +type ClusterPortForwardSpec struct { + // Selector is a selector to filter pods to configure. + Selector *ObjectSelector `json:"selector,omitempty"` + // Forwards is a list of forwards to configure. + Forwards []Forward `json:"forwards"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ClusterPortForwardList contains a list of ClusterPortForward +type ClusterPortForwardList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterPortForward `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterPortForward{}, &ClusterPortForwardList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_resource_usage_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_resource_usage_types.go new file mode 100644 index 0000000000..44bbbf0337 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/cluster_resource_usage_types.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ClusterResourceUsageKind is the kind for ClusterResourceUsage. + ClusterResourceUsageKind = "ClusterResourceUsage" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterresourceusages,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=clusterresourceusages/status,verbs=update;patch + +// ClusterResourceUsage provides cluster-wide resource usage. +type ClusterResourceUsage struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for cluster resource usage. + Spec ClusterResourceUsageSpec `json:"spec"` + // Status holds status for cluster resource usage + //+k8s:conversion-gen=false + Status ClusterResourceUsageStatus `json:"status,omitempty"` +} + +// ClusterResourceUsageStatus holds status for cluster resource usage +type ClusterResourceUsageStatus struct { + // Conditions holds conditions for cluster resource usage + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ClusterResourceUsageSpec holds spec for cluster resource usage. +type ClusterResourceUsageSpec struct { + // Selector is a selector to filter pods to configure. + Selector *ObjectSelector `json:"selector,omitempty"` + // Usages is a list of resource usage for the pod. + Usages []ResourceUsageContainer `json:"usages,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ClusterResourceUsageList is a list of ClusterResourceUsage. +type ClusterResourceUsageList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []ClusterResourceUsage `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ClusterResourceUsage{}, &ClusterResourceUsageList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/condition.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/condition.go new file mode 100644 index 0000000000..d6a4353bd2 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/condition.go @@ -0,0 +1,70 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Condition contains details for one aspect of the current state of this API Resource. +type Condition struct { + // Type of condition in CamelCase or in foo.example.com/CamelCase. + // Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + // useful (see .node.status.conditions), the ability to deconflict is important. + // The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` + // +kubebuilder:validation:MaxLength=316 + Type string `json:"type"` + // Status of the condition + // +kubebuilder:validation:Required + Status ConditionStatus `json:"status"` + // LastTransitionTime is the last time the condition transitioned from one status to another. + // This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + LastTransitionTime metav1.Time `json:"lastTransitionTime"` + // Reason contains a programmatic identifier indicating the reason for the condition's last transition. + // Producers of specific condition types may define expected values and meanings for this field, + // and whether the values are considered a guaranteed API. + // The value should be a CamelCase string. + // This field may not be empty. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=1024 + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$` + Reason string `json:"reason"` + // Message is a human readable message indicating details about the transition. + // This may be an empty string. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=32768 + Message string `json:"message"` +} + +// ConditionStatus is the status of a condition. +// +enum +type ConditionStatus string + +const ( + // ConditionTrue means a resource is in the condition. + ConditionTrue ConditionStatus = "True" + // ConditionFalse means a resource is not in the condition. + ConditionFalse ConditionStatus = "False" + // ConditionUnknown means kubernetes can't decide if a resource is in the condition or not. + ConditionUnknown ConditionStatus = "Unknown" +) diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/doc.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/doc.go new file mode 100644 index 0000000000..3abcd74c18 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/doc.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:defaulter-gen=TypeMeta +// +groupName=kwok.x-k8s.io + +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes/status,verbs=patch;update +// +kubebuilder:rbac:groups="",resources=pods,verbs=delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups="",resources=pods/status,verbs=patch;update +// +kubebuilder:rbac:groups="",resources=events,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=create;get;list;patch;update;watch + +// Package v1alpha1 implements the v1alpha1 apiVersion of kwok's configuration +package v1alpha1 diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/exec_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/exec_types.go new file mode 100644 index 0000000000..33d2e4bdd8 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/exec_types.go @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ExecKind is the kind of the Exec. + ExecKind = "Exec" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +kubebuilder:subresource:status +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=execs,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=execs/status,verbs=update;patch + +// Exec provides exec configuration for a single pod. +type Exec struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for exec + Spec ExecSpec `json:"spec"` + // Status holds status for exec + //+k8s:conversion-gen=false + Status ExecStatus `json:"status,omitempty"` +} + +// ExecStatus holds status for exec +type ExecStatus struct { + // Conditions holds conditions for exec + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ExecSpec holds spec for exec +type ExecSpec struct { + // Execs is a list of execs to configure. + Execs []ExecTarget `json:"execs"` +} + +// ExecTarget holds information how to exec. +type ExecTarget struct { + // Containers is a list of containers to exec. + // if not set, all containers will be execed. + Containers []string `json:"containers,omitempty"` + // Local holds information how to exec to a local target. + Local *ExecTargetLocal `json:"local,omitempty"` +} + +// ExecTargetLocal holds information how to exec to a local target. +type ExecTargetLocal struct { + // WorkDir is the working directory to exec with. + WorkDir string `json:"workDir,omitempty"` + // Envs is a list of environment variables to exec with. + Envs []EnvVar `json:"envs,omitempty"` + // SecurityContext is the user context to exec. + SecurityContext *SecurityContext `json:"securityContext,omitempty"` +} + +// EnvVar represents an environment variable present in a Container. +type EnvVar struct { + // Name of the environment variable. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Value of the environment variable. + Value string `json:"value,omitempty"` +} + +// SecurityContext specifies the existing uid and gid to run exec command in container process. +type SecurityContext struct { + // RunAsUser is the existing uid to run exec command in container process. + RunAsUser *int64 `json:"runAsUser,omitempty"` + // RunAsGroup is the existing gid to run exec command in container process. + RunAsGroup *int64 `json:"runAsGroup,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ExecList contains a list of Exec +type ExecList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Exec `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Exec{}, &ExecList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/groupversion_info.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..3787d95013 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/groupversion_info.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=kwok.x-k8s.io + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{ + Group: "kwok.x-k8s.io", + Version: "v1alpha1", + } + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/logs_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/logs_types.go new file mode 100644 index 0000000000..d0e779945e --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/logs_types.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // LogsKind is the kind of the Logs. + LogsKind = "Logs" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +kubebuilder:subresource:status +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=logs,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=logs/status,verbs=update;patch + +// Logs provides logging configuration for a single pod. +type Logs struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for logs + Spec LogsSpec `json:"spec"` + // Status holds status for logs + //+k8s:conversion-gen=false + Status LogsStatus `json:"status,omitempty"` +} + +// LogsStatus holds status for logs +type LogsStatus struct { + // Conditions holds conditions for logs + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// LogsSpec holds spec for logs. +type LogsSpec struct { + // Logs is a list of logs to configure. + Logs []Log `json:"logs"` +} + +// Log holds information how to forward logs. +type Log struct { + // Containers is list of container names. + Containers []string `json:"containers,omitempty"` + // LogsFile is the file from which the log forward starts + LogsFile *string `json:"logsFile,omitempty"` + // Follow up if true + Follow *bool `json:"follow,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// LogsList contains a list of Logs +type LogsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Logs `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Logs{}, &LogsList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/metric_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/metric_types.go new file mode 100644 index 0000000000..0505e14758 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/metric_types.go @@ -0,0 +1,165 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // MetricKind is the kind for metrics. + MetricKind = "Metric" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=metrics,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=metrics/status,verbs=update;patch + +// Metric provides metrics configuration. +type Metric struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for metrics. + Spec MetricSpec `json:"spec"` + // Status holds status for metrics + //+k8s:conversion-gen=false + Status MetricStatus `json:"status,omitempty"` +} + +// MetricStatus holds status for metrics +type MetricStatus struct { + // Conditions holds conditions for metrics. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// MetricSpec holds spec for metrics. +type MetricSpec struct { + // Path is a restful service path. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Path string `json:"path"` + // Metrics is a list of metric configurations. + Metrics []MetricConfig `json:"metrics"` +} + +// MetricConfig provides metric configuration to a single metric +type MetricConfig struct { + // Name is the fully-qualified name of the metric. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Help provides information about this metric. + Help string `json:"help,omitempty"` + // Kind is kind of metric + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=counter;gauge;histogram + Kind Kind `json:"kind"` + // Labels are metric labels. + // +patchMergeKey=name + // +patchStrategy=merge + // +listType=map + // +listMapKey=name + Labels []MetricLabel `json:"labels,omitempty"` + // Value is a CEL expression. + Value string `json:"value,omitempty"` + // Buckets is a list of buckets for a histogram metric. + // +patchMergeKey=le + // +patchStrategy=merge + // +listType=map + // +listMapKey=le + Buckets []MetricBucket `json:"buckets,omitempty"` + // Dimension is a dimension of the metric. + // +default="node" + Dimension Dimension `json:"dimension,omitempty"` +} + +// Kind is kind of metric configuration. +// +enum +type Kind string + +const ( + // KindCounter is a counter metric. + KindCounter Kind = "counter" + // KindGauge is a gauge metric. + KindGauge Kind = "gauge" + // KindHistogram is a histogram metric. + KindHistogram Kind = "histogram" +) + +// Dimension is a dimension of the metric. +// +enum +type Dimension string + +const ( + // DimensionNode is a node dimension. + DimensionNode Dimension = "node" + // DimensionPod is a pod dimension. + DimensionPod Dimension = "pod" + // DimensionContainer is a container dimension. + DimensionContainer Dimension = "container" +) + +// MetricLabel holds label name and the value of the label. +type MetricLabel struct { + // Name is a label name. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Value is a CEL expression. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Value string `json:"value"` +} + +// MetricBucket is a single bucket for a metric. +type MetricBucket struct { + // Le is less-than or equal. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=0 + Le float64 `json:"le"` + // Value is a CEL expression. + // +kubebuilder:validation:Required + Value string `json:"value"` + // Hidden is means that this bucket not shown in the metric. + // but value will be calculated and cumulative into the next bucket. + Hidden bool `json:"hidden,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// MetricList contains a list of Metric +type MetricList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Metric `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Metric{}, &MetricList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/object_selector.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/object_selector.go new file mode 100644 index 0000000000..eaeed02751 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/object_selector.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// ObjectSelector holds information how to match based on namespace and name. +type ObjectSelector struct { + // MatchNamespaces is a list of namespaces to match. + // if not set, all namespaces will be matched. + MatchNamespaces []string `json:"matchNamespaces,omitempty"` + // MatchNames is a list of names to match. + // if not set, all names will be matched. + MatchNames []string `json:"matchNames,omitempty"` +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/port_forward_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/port_forward_types.go new file mode 100644 index 0000000000..2db633f62c --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/port_forward_types.go @@ -0,0 +1,101 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // PortForwardKind is the kind of the PortForward. + PortForwardKind = "PortForward" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +kubebuilder:subresource:status +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=portforwards,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=portforwards/status,verbs=update;patch + +// PortForward provides port forward configuration for a single pod. +type PortForward struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for port forward. + Spec PortForwardSpec `json:"spec"` + // Status holds status for port forward + //+k8s:conversion-gen=false + Status PortForwardStatus `json:"status,omitempty"` +} + +// PortForwardStatus holds status for port forward +type PortForwardStatus struct { + // Conditions holds conditions for port forward + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// PortForwardSpec holds spec for port forward. +type PortForwardSpec struct { + // Forwards is a list of forwards to configure. + Forwards []Forward `json:"forwards"` +} + +// Forward holds information how to forward based on ports. +type Forward struct { + // Ports is a list of ports to forward. + // if not set, all ports will be forwarded. + Ports []int32 `json:"ports,omitempty"` + // Target is the target to forward to. + Target *ForwardTarget `json:"target,omitempty"` + // Command is the command to run to forward with stdin/stdout. + // if set, Target will be ignored. + Command []string `json:"command,omitempty"` +} + +// ForwardTarget holds information how to forward to a target. +type ForwardTarget struct { + // Port is the port to forward to. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + // Address is the address to forward to. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// PortForwardList contains a list of PortForward +type PortForwardList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []PortForward `json:"items"` +} + +func init() { + SchemeBuilder.Register(&PortForward{}, &PortForwardList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/register.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/register.go new file mode 100644 index 0000000000..d99263e6c6 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/register.go @@ -0,0 +1,29 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = GroupVersion + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/resource_usage_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/resource_usage_types.go new file mode 100644 index 0000000000..8915a31c68 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/resource_usage_types.go @@ -0,0 +1,93 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ResourceUsageKind is the kind for resource usage. + ResourceUsageKind = "ResourceUsage" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +kubebuilder:subresource:status +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=resourceusages,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=resourceusages/status,verbs=update;patch + +// ResourceUsage provides resource usage for a single pod. +type ResourceUsage struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata"` + // Spec holds spec for resource usage. + Spec ResourceUsageSpec `json:"spec"` + // Status holds status for resource usage + //+k8s:conversion-gen=false + Status ResourceUsageStatus `json:"status,omitempty"` +} + +// ResourceUsageStatus holds status for resource usage +type ResourceUsageStatus struct { + // Conditions holds conditions for resource usage + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// ResourceUsageSpec holds spec for resource usage. +type ResourceUsageSpec struct { + // Usages is a list of resource usage for the pod. + Usages []ResourceUsageContainer `json:"usages,omitempty"` +} + +// ResourceUsageContainer holds spec for resource usage container. +type ResourceUsageContainer struct { + // Containers is list of container names. + Containers []string `json:"containers,omitempty"` + // Usage is a list of resource usage for the container. + Usage map[string]ResourceUsageValue `json:"usage,omitempty"` +} + +// ResourceUsageValue holds value for resource usage. +type ResourceUsageValue struct { + // Value is the value for resource usage. + Value *resource.Quantity `json:"value,omitempty"` + // Expression is the expression for resource usage. + Expression *string `json:"expression,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// ResourceUsageList is a list of ResourceUsage. +type ResourceUsageList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []ResourceUsage `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ResourceUsage{}, &ResourceUsageList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/stage_types.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/stage_types.go new file mode 100644 index 0000000000..a26d04908c --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/stage_types.go @@ -0,0 +1,271 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // StageKind is the kind of the Stage resource. + StageKind = "Stage" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=stages,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=kwok.x-k8s.io,resources=stages/status,verbs=update;patch + +// Stage is an API that describes the staged change of a resource +type Stage struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata,omitempty"` + // Spec holds information about the request being evaluated. + Spec StageSpec `json:"spec"` + // Status holds status for the Stage + //+k8s:conversion-gen=false + Status StageStatus `json:"status,omitempty"` +} + +// StageStatus holds status for the Stage +type StageStatus struct { + // Conditions holds conditions for the Stage. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// StageSpec defines the specification for Stage. +type StageSpec struct { + // ResourceRef specifies the Kind and version of the resource. + ResourceRef StageResourceRef `json:"resourceRef"` + // Selector specifies the stags will be applied to the selected resource. + Selector *StageSelector `json:"selector,omitempty"` + // Weight means when multiple stages share the same ResourceRef and Selector, + // a random stage will be matched as the next stage based on the weight. + // +default=0 + // +kubebuilder:default=0 + // +kubebuilder:validation:Minimum=0 + Weight int `json:"weight,omitempty"` + // WeightFrom means is the expression used to get the value. + // If it is a number type, convert to int. + // If it is a string type, the value get will be parsed by strconv.ParseInt. + WeightFrom *ExpressionFromSource `json:"weightFrom,omitempty"` + // Delay means there is a delay in this stage. + Delay *StageDelay `json:"delay,omitempty"` + // Next indicates that this stage will be moved to. + Next StageNext `json:"next"` + // ImmediateNextStage means that the next stage of matching is performed immediately, without waiting for the Apiserver to push. + ImmediateNextStage *bool `json:"immediateNextStage,omitempty"` +} + +// StageResourceRef specifies the kind and version of the resource. +type StageResourceRef struct { + // APIGroup of the referent. + // +default="v1" + // +kubebuilder:default="v1" + APIGroup string `json:"apiGroup,omitempty"` + // Kind of the referent. + Kind string `json:"kind"` +} + +// StageDelay describes the delay time before going to next. +type StageDelay struct { + // DurationMilliseconds indicates the stage delay time. + // If JitterDurationMilliseconds is less than DurationMilliseconds, then JitterDurationMilliseconds is used. + // +kubebuilder:validation:Minimum=0 + DurationMilliseconds *int64 `json:"durationMilliseconds,omitempty"` + // DurationFrom is the expression used to get the value. + // If it is a time.Time type, getting the value will be minus time.Now() to get DurationMilliseconds + // If it is a string type, the value get will be parsed by time.ParseDuration. + DurationFrom *ExpressionFromSource `json:"durationFrom,omitempty"` + + // JitterDurationMilliseconds is the duration plus an additional amount chosen uniformly + // at random from the interval between DurationMilliseconds and JitterDurationMilliseconds. + // +kubebuilder:validation:Minimum=0 + JitterDurationMilliseconds *int64 `json:"jitterDurationMilliseconds,omitempty"` + // JitterDurationFrom is the expression used to get the value. + // If it is a time.Time type, getting the value will be minus time.Now() to get JitterDurationMilliseconds + // If it is a string type, the value get will be parsed by time.ParseDuration. + JitterDurationFrom *ExpressionFromSource `json:"jitterDurationFrom,omitempty"` +} + +// StageNext describes a stage will be moved to. +type StageNext struct { + // Event means that an event will be sent. + Event *StageEvent `json:"event,omitempty"` + // Finalizers means that finalizers will be modified. + Finalizers *StageFinalizers `json:"finalizers,omitempty"` + // Delete means that the resource will be deleted if true. + Delete bool `json:"delete,omitempty"` + // Patches means that the resource will be patched. + Patches []StagePatch `json:"patches,omitempty"` + + // StatusTemplate indicates the template for modifying the status of the resource in the next. + // Deprecated: Use Patches instead. + //+k8s:conversion-gen=false + StatusTemplate string `json:"statusTemplate,omitempty"` + // StatusSubresource indicates the name of the subresource that will be patched. The support for + // this field is not available in Pod and Node resources. + // +default="status" + // +kubebuilder:default=status + // Deprecated: Use Patches instead. + //+k8s:conversion-gen=false + StatusSubresource *string `json:"statusSubresource,omitempty"` + // StatusPatchAs indicates the impersonating configuration for client when patching status. + // In most cases this will be empty, in which case the default client service account will be used. + // When this is not empty, a corresponding rbac change is required to grant `impersonate` privilege. + // The support for this field is not available in Pod and Node resources. + // Deprecated: Use Patches instead. + //+k8s:conversion-gen=false + StatusPatchAs *ImpersonationConfig `json:"statusPatchAs,omitempty"` +} + +// StagePatch describes the patch for the resource. +type StagePatch struct { + // Subresource indicates the name of the subresource that will be patched. + Subresource string `json:"subresource,omitempty"` + // Root indicates the root of the template calculated by the patch. + Root string `json:"root,omitempty"` + // Template indicates the template for modifying the resource in the next. + Template string `json:"template,omitempty"` + // Type indicates the type of the patch. + // +kubebuilder:validation:Enum=json;merge;strategic + Type *StagePatchType `json:"type,omitempty"` + // Impersonation indicates the impersonating configuration for client when patching status. + // In most cases this will be empty, in which case the default client service account will be used. + // When this is not empty, a corresponding rbac change is required to grant `impersonate` privilege. + // The support for this field is not available in Pod and Node resources. + Impersonation *ImpersonationConfig `json:"impersonation,omitempty"` +} + +// StagePatchType is the type of the patch. +type StagePatchType string + +const ( + // StagePatchTypeJSONPatch is the JSON patch type. + StagePatchTypeJSONPatch StagePatchType = "json" + // StagePatchTypeMergePatch is the merge patch type. + StagePatchTypeMergePatch StagePatchType = "merge" + // StagePatchTypeStrategicMergePatch is the strategic merge patch type. + StagePatchTypeStrategicMergePatch StagePatchType = "strategic" +) + +// ImpersonationConfig describes the configuration for impersonating clients +type ImpersonationConfig struct { + // Username the target username for the client to impersonate + Username string `json:"username"` +} + +// StageFinalizers describes the modifications in the finalizers of a resource. +type StageFinalizers struct { + // Add means that the Finalizers will be added to the resource. + Add []FinalizerItem `json:"add,omitempty"` + // Remove means that the Finalizers will be removed from the resource. + Remove []FinalizerItem `json:"remove,omitempty"` + // Empty means that the Finalizers for that resource will be emptied. + Empty bool `json:"empty,omitempty"` +} + +// FinalizerItem describes the one of the finalizers. +type FinalizerItem struct { + // Value is the value of the finalizer. + Value string `json:"value,omitempty"` +} + +// StageEvent describes one event in the Kubernetes. +type StageEvent struct { + // Type is the type of this event (Normal, Warning), It is machine-readable. + Type string `json:"type,omitempty"` + // Reason is why the action was taken. It is human-readable. + Reason string `json:"reason,omitempty"` + // Message is a human-readable description of the status of this operation. + Message string `json:"message,omitempty"` +} + +// StageSelector is a resource selector. the result of matchLabels and matchAnnotations and +// matchExpressions are ANDed. An empty resource selector matches all objects. A null +// resource selector matches no objects. +type StageSelector struct { + // MatchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + // map is equivalent to an element of matchExpressions, whose key field is ".metadata.labels[key]", the + // operator is "In", and the values array contains only "value". The requirements are ANDed. + MatchLabels map[string]string `json:"matchLabels,omitempty"` + // MatchAnnotations is a map of {key,value} pairs. A single {key,value} in the matchAnnotations + // map is equivalent to an element of matchExpressions, whose key field is ".metadata.annotations[key]", the + // operator is "In", and the values array contains only "value". The requirements are ANDed. + MatchAnnotations map[string]string `json:"matchAnnotations,omitempty"` + // MatchExpressions is a list of label selector requirements. The requirements are ANDed. + MatchExpressions []SelectorRequirement `json:"matchExpressions,omitempty"` +} + +// SelectorRequirement is a resource selector requirement is a selector that contains values, a key, +// and an operator that relates the key and values. +type SelectorRequirement struct { + // The name of the scope that the selector applies to. + Key string `json:"key"` + // Represents a scope's relationship to a set of values. + Operator SelectorOperator `json:"operator"` + // An array of string values. + // If the operator is In, NotIn, Intersection or NotIntersection, the values array must be non-empty. + // If the operator is Exists or DoesNotExist, the values array must be empty. + Values []string `json:"values,omitempty"` +} + +// SelectorOperator is a label selector operator is the set of operators that can be used in a selector requirement. +// +enum +type SelectorOperator string + +// The following are valid selector operators. +const ( + // SelectorOpIn is the set inclusion operator. + SelectorOpIn SelectorOperator = "In" + // SelectorOpNotIn is the negated set inclusion operator. + SelectorOpNotIn SelectorOperator = "NotIn" + // SelectorOpExists is the existence operator. + SelectorOpExists SelectorOperator = "Exists" + // SelectorOpDoesNotExist is the negated existence operator. + SelectorOpDoesNotExist SelectorOperator = "DoesNotExist" +) + +// ExpressionFromSource represents a source for the value of a from. +type ExpressionFromSource struct { + // ExpressionFrom is the expression used to get the value. + ExpressionFrom string `json:"expressionFrom,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// StageList contains a list of Stage +type StageList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Stage `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Stage{}, &StageList{}) +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..5609c20eb9 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,1972 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Attach) DeepCopyInto(out *Attach) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Attach. +func (in *Attach) DeepCopy() *Attach { + if in == nil { + return nil + } + out := new(Attach) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Attach) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AttachConfig) DeepCopyInto(out *AttachConfig) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LogsFile != nil { + in, out := &in.LogsFile, &out.LogsFile + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachConfig. +func (in *AttachConfig) DeepCopy() *AttachConfig { + if in == nil { + return nil + } + out := new(AttachConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AttachList) DeepCopyInto(out *AttachList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Attach, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachList. +func (in *AttachList) DeepCopy() *AttachList { + if in == nil { + return nil + } + out := new(AttachList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AttachList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AttachSpec) DeepCopyInto(out *AttachSpec) { + *out = *in + if in.Attaches != nil { + in, out := &in.Attaches, &out.Attaches + *out = make([]AttachConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachSpec. +func (in *AttachSpec) DeepCopy() *AttachSpec { + if in == nil { + return nil + } + out := new(AttachSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AttachStatus) DeepCopyInto(out *AttachStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachStatus. +func (in *AttachStatus) DeepCopy() *AttachStatus { + if in == nil { + return nil + } + out := new(AttachStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAttach) DeepCopyInto(out *ClusterAttach) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAttach. +func (in *ClusterAttach) DeepCopy() *ClusterAttach { + if in == nil { + return nil + } + out := new(ClusterAttach) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterAttach) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAttachList) DeepCopyInto(out *ClusterAttachList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterAttach, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAttachList. +func (in *ClusterAttachList) DeepCopy() *ClusterAttachList { + if in == nil { + return nil + } + out := new(ClusterAttachList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterAttachList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAttachSpec) DeepCopyInto(out *ClusterAttachSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ObjectSelector) + (*in).DeepCopyInto(*out) + } + if in.Attaches != nil { + in, out := &in.Attaches, &out.Attaches + *out = make([]AttachConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAttachSpec. +func (in *ClusterAttachSpec) DeepCopy() *ClusterAttachSpec { + if in == nil { + return nil + } + out := new(ClusterAttachSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAttachStatus) DeepCopyInto(out *ClusterAttachStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAttachStatus. +func (in *ClusterAttachStatus) DeepCopy() *ClusterAttachStatus { + if in == nil { + return nil + } + out := new(ClusterAttachStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterExec) DeepCopyInto(out *ClusterExec) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterExec. +func (in *ClusterExec) DeepCopy() *ClusterExec { + if in == nil { + return nil + } + out := new(ClusterExec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterExec) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterExecList) DeepCopyInto(out *ClusterExecList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterExec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterExecList. +func (in *ClusterExecList) DeepCopy() *ClusterExecList { + if in == nil { + return nil + } + out := new(ClusterExecList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterExecList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterExecSpec) DeepCopyInto(out *ClusterExecSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ObjectSelector) + (*in).DeepCopyInto(*out) + } + if in.Execs != nil { + in, out := &in.Execs, &out.Execs + *out = make([]ExecTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterExecSpec. +func (in *ClusterExecSpec) DeepCopy() *ClusterExecSpec { + if in == nil { + return nil + } + out := new(ClusterExecSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterExecStatus) DeepCopyInto(out *ClusterExecStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterExecStatus. +func (in *ClusterExecStatus) DeepCopy() *ClusterExecStatus { + if in == nil { + return nil + } + out := new(ClusterExecStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterLogs) DeepCopyInto(out *ClusterLogs) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLogs. +func (in *ClusterLogs) DeepCopy() *ClusterLogs { + if in == nil { + return nil + } + out := new(ClusterLogs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterLogs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterLogsList) DeepCopyInto(out *ClusterLogsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterLogs, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLogsList. +func (in *ClusterLogsList) DeepCopy() *ClusterLogsList { + if in == nil { + return nil + } + out := new(ClusterLogsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterLogsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterLogsSpec) DeepCopyInto(out *ClusterLogsSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ObjectSelector) + (*in).DeepCopyInto(*out) + } + if in.Logs != nil { + in, out := &in.Logs, &out.Logs + *out = make([]Log, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLogsSpec. +func (in *ClusterLogsSpec) DeepCopy() *ClusterLogsSpec { + if in == nil { + return nil + } + out := new(ClusterLogsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterLogsStatus) DeepCopyInto(out *ClusterLogsStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLogsStatus. +func (in *ClusterLogsStatus) DeepCopy() *ClusterLogsStatus { + if in == nil { + return nil + } + out := new(ClusterLogsStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPortForward) DeepCopyInto(out *ClusterPortForward) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPortForward. +func (in *ClusterPortForward) DeepCopy() *ClusterPortForward { + if in == nil { + return nil + } + out := new(ClusterPortForward) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterPortForward) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPortForwardList) DeepCopyInto(out *ClusterPortForwardList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterPortForward, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPortForwardList. +func (in *ClusterPortForwardList) DeepCopy() *ClusterPortForwardList { + if in == nil { + return nil + } + out := new(ClusterPortForwardList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterPortForwardList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPortForwardSpec) DeepCopyInto(out *ClusterPortForwardSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ObjectSelector) + (*in).DeepCopyInto(*out) + } + if in.Forwards != nil { + in, out := &in.Forwards, &out.Forwards + *out = make([]Forward, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPortForwardSpec. +func (in *ClusterPortForwardSpec) DeepCopy() *ClusterPortForwardSpec { + if in == nil { + return nil + } + out := new(ClusterPortForwardSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterPortForwardStatus) DeepCopyInto(out *ClusterPortForwardStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPortForwardStatus. +func (in *ClusterPortForwardStatus) DeepCopy() *ClusterPortForwardStatus { + if in == nil { + return nil + } + out := new(ClusterPortForwardStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourceUsage) DeepCopyInto(out *ClusterResourceUsage) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourceUsage. +func (in *ClusterResourceUsage) DeepCopy() *ClusterResourceUsage { + if in == nil { + return nil + } + out := new(ClusterResourceUsage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourceUsage) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourceUsageList) DeepCopyInto(out *ClusterResourceUsageList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterResourceUsage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourceUsageList. +func (in *ClusterResourceUsageList) DeepCopy() *ClusterResourceUsageList { + if in == nil { + return nil + } + out := new(ClusterResourceUsageList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterResourceUsageList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourceUsageSpec) DeepCopyInto(out *ClusterResourceUsageSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ObjectSelector) + (*in).DeepCopyInto(*out) + } + if in.Usages != nil { + in, out := &in.Usages, &out.Usages + *out = make([]ResourceUsageContainer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourceUsageSpec. +func (in *ClusterResourceUsageSpec) DeepCopy() *ClusterResourceUsageSpec { + if in == nil { + return nil + } + out := new(ClusterResourceUsageSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterResourceUsageStatus) DeepCopyInto(out *ClusterResourceUsageStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResourceUsageStatus. +func (in *ClusterResourceUsageStatus) DeepCopy() *ClusterResourceUsageStatus { + if in == nil { + return nil + } + out := new(ClusterResourceUsageStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. +func (in *Condition) DeepCopy() *Condition { + if in == nil { + return nil + } + out := new(Condition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvVar) DeepCopyInto(out *EnvVar) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvVar. +func (in *EnvVar) DeepCopy() *EnvVar { + if in == nil { + return nil + } + out := new(EnvVar) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Exec) DeepCopyInto(out *Exec) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Exec. +func (in *Exec) DeepCopy() *Exec { + if in == nil { + return nil + } + out := new(Exec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Exec) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecList) DeepCopyInto(out *ExecList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Exec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecList. +func (in *ExecList) DeepCopy() *ExecList { + if in == nil { + return nil + } + out := new(ExecList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExecList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecSpec) DeepCopyInto(out *ExecSpec) { + *out = *in + if in.Execs != nil { + in, out := &in.Execs, &out.Execs + *out = make([]ExecTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecSpec. +func (in *ExecSpec) DeepCopy() *ExecSpec { + if in == nil { + return nil + } + out := new(ExecSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecStatus) DeepCopyInto(out *ExecStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecStatus. +func (in *ExecStatus) DeepCopy() *ExecStatus { + if in == nil { + return nil + } + out := new(ExecStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecTarget) DeepCopyInto(out *ExecTarget) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Local != nil { + in, out := &in.Local, &out.Local + *out = new(ExecTargetLocal) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecTarget. +func (in *ExecTarget) DeepCopy() *ExecTarget { + if in == nil { + return nil + } + out := new(ExecTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecTargetLocal) DeepCopyInto(out *ExecTargetLocal) { + *out = *in + if in.Envs != nil { + in, out := &in.Envs, &out.Envs + *out = make([]EnvVar, len(*in)) + copy(*out, *in) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(SecurityContext) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecTargetLocal. +func (in *ExecTargetLocal) DeepCopy() *ExecTargetLocal { + if in == nil { + return nil + } + out := new(ExecTargetLocal) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpressionFromSource) DeepCopyInto(out *ExpressionFromSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpressionFromSource. +func (in *ExpressionFromSource) DeepCopy() *ExpressionFromSource { + if in == nil { + return nil + } + out := new(ExpressionFromSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FinalizerItem) DeepCopyInto(out *FinalizerItem) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FinalizerItem. +func (in *FinalizerItem) DeepCopy() *FinalizerItem { + if in == nil { + return nil + } + out := new(FinalizerItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Forward) DeepCopyInto(out *Forward) { + *out = *in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]int32, len(*in)) + copy(*out, *in) + } + if in.Target != nil { + in, out := &in.Target, &out.Target + *out = new(ForwardTarget) + **out = **in + } + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Forward. +func (in *Forward) DeepCopy() *Forward { + if in == nil { + return nil + } + out := new(Forward) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ForwardTarget) DeepCopyInto(out *ForwardTarget) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ForwardTarget. +func (in *ForwardTarget) DeepCopy() *ForwardTarget { + if in == nil { + return nil + } + out := new(ForwardTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationConfig) DeepCopyInto(out *ImpersonationConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationConfig. +func (in *ImpersonationConfig) DeepCopy() *ImpersonationConfig { + if in == nil { + return nil + } + out := new(ImpersonationConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Log) DeepCopyInto(out *Log) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LogsFile != nil { + in, out := &in.LogsFile, &out.LogsFile + *out = new(string) + **out = **in + } + if in.Follow != nil { + in, out := &in.Follow, &out.Follow + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Log. +func (in *Log) DeepCopy() *Log { + if in == nil { + return nil + } + out := new(Log) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Logs) DeepCopyInto(out *Logs) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Logs. +func (in *Logs) DeepCopy() *Logs { + if in == nil { + return nil + } + out := new(Logs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Logs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogsList) DeepCopyInto(out *LogsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Logs, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogsList. +func (in *LogsList) DeepCopy() *LogsList { + if in == nil { + return nil + } + out := new(LogsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LogsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogsSpec) DeepCopyInto(out *LogsSpec) { + *out = *in + if in.Logs != nil { + in, out := &in.Logs, &out.Logs + *out = make([]Log, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogsSpec. +func (in *LogsSpec) DeepCopy() *LogsSpec { + if in == nil { + return nil + } + out := new(LogsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogsStatus) DeepCopyInto(out *LogsStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogsStatus. +func (in *LogsStatus) DeepCopy() *LogsStatus { + if in == nil { + return nil + } + out := new(LogsStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metric) DeepCopyInto(out *Metric) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metric. +func (in *Metric) DeepCopy() *Metric { + if in == nil { + return nil + } + out := new(Metric) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Metric) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricBucket) DeepCopyInto(out *MetricBucket) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricBucket. +func (in *MetricBucket) DeepCopy() *MetricBucket { + if in == nil { + return nil + } + out := new(MetricBucket) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricConfig) DeepCopyInto(out *MetricConfig) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]MetricLabel, len(*in)) + copy(*out, *in) + } + if in.Buckets != nil { + in, out := &in.Buckets, &out.Buckets + *out = make([]MetricBucket, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricConfig. +func (in *MetricConfig) DeepCopy() *MetricConfig { + if in == nil { + return nil + } + out := new(MetricConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricLabel) DeepCopyInto(out *MetricLabel) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricLabel. +func (in *MetricLabel) DeepCopy() *MetricLabel { + if in == nil { + return nil + } + out := new(MetricLabel) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricList) DeepCopyInto(out *MetricList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Metric, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricList. +func (in *MetricList) DeepCopy() *MetricList { + if in == nil { + return nil + } + out := new(MetricList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MetricList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricSpec) DeepCopyInto(out *MetricSpec) { + *out = *in + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]MetricConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec. +func (in *MetricSpec) DeepCopy() *MetricSpec { + if in == nil { + return nil + } + out := new(MetricSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricStatus) DeepCopyInto(out *MetricStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricStatus. +func (in *MetricStatus) DeepCopy() *MetricStatus { + if in == nil { + return nil + } + out := new(MetricStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectSelector) DeepCopyInto(out *ObjectSelector) { + *out = *in + if in.MatchNamespaces != nil { + in, out := &in.MatchNamespaces, &out.MatchNamespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchNames != nil { + in, out := &in.MatchNames, &out.MatchNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSelector. +func (in *ObjectSelector) DeepCopy() *ObjectSelector { + if in == nil { + return nil + } + out := new(ObjectSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortForward) DeepCopyInto(out *PortForward) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortForward. +func (in *PortForward) DeepCopy() *PortForward { + if in == nil { + return nil + } + out := new(PortForward) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PortForward) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortForwardList) DeepCopyInto(out *PortForwardList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PortForward, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortForwardList. +func (in *PortForwardList) DeepCopy() *PortForwardList { + if in == nil { + return nil + } + out := new(PortForwardList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PortForwardList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortForwardSpec) DeepCopyInto(out *PortForwardSpec) { + *out = *in + if in.Forwards != nil { + in, out := &in.Forwards, &out.Forwards + *out = make([]Forward, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortForwardSpec. +func (in *PortForwardSpec) DeepCopy() *PortForwardSpec { + if in == nil { + return nil + } + out := new(PortForwardSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortForwardStatus) DeepCopyInto(out *PortForwardStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortForwardStatus. +func (in *PortForwardStatus) DeepCopy() *PortForwardStatus { + if in == nil { + return nil + } + out := new(PortForwardStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsage) DeepCopyInto(out *ResourceUsage) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsage. +func (in *ResourceUsage) DeepCopy() *ResourceUsage { + if in == nil { + return nil + } + out := new(ResourceUsage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceUsage) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsageContainer) DeepCopyInto(out *ResourceUsageContainer) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Usage != nil { + in, out := &in.Usage, &out.Usage + *out = make(map[string]ResourceUsageValue, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsageContainer. +func (in *ResourceUsageContainer) DeepCopy() *ResourceUsageContainer { + if in == nil { + return nil + } + out := new(ResourceUsageContainer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsageList) DeepCopyInto(out *ResourceUsageList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ResourceUsage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsageList. +func (in *ResourceUsageList) DeepCopy() *ResourceUsageList { + if in == nil { + return nil + } + out := new(ResourceUsageList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceUsageList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsageSpec) DeepCopyInto(out *ResourceUsageSpec) { + *out = *in + if in.Usages != nil { + in, out := &in.Usages, &out.Usages + *out = make([]ResourceUsageContainer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsageSpec. +func (in *ResourceUsageSpec) DeepCopy() *ResourceUsageSpec { + if in == nil { + return nil + } + out := new(ResourceUsageSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsageStatus) DeepCopyInto(out *ResourceUsageStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsageStatus. +func (in *ResourceUsageStatus) DeepCopy() *ResourceUsageStatus { + if in == nil { + return nil + } + out := new(ResourceUsageStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceUsageValue) DeepCopyInto(out *ResourceUsageValue) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + x := (*in).DeepCopy() + *out = &x + } + if in.Expression != nil { + in, out := &in.Expression, &out.Expression + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceUsageValue. +func (in *ResourceUsageValue) DeepCopy() *ResourceUsageValue { + if in == nil { + return nil + } + out := new(ResourceUsageValue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityContext) DeepCopyInto(out *SecurityContext) { + *out = *in + if in.RunAsUser != nil { + in, out := &in.RunAsUser, &out.RunAsUser + *out = new(int64) + **out = **in + } + if in.RunAsGroup != nil { + in, out := &in.RunAsGroup, &out.RunAsGroup + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityContext. +func (in *SecurityContext) DeepCopy() *SecurityContext { + if in == nil { + return nil + } + out := new(SecurityContext) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SelectorRequirement) DeepCopyInto(out *SelectorRequirement) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorRequirement. +func (in *SelectorRequirement) DeepCopy() *SelectorRequirement { + if in == nil { + return nil + } + out := new(SelectorRequirement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Stage) DeepCopyInto(out *Stage) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stage. +func (in *Stage) DeepCopy() *Stage { + if in == nil { + return nil + } + out := new(Stage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Stage) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageDelay) DeepCopyInto(out *StageDelay) { + *out = *in + if in.DurationMilliseconds != nil { + in, out := &in.DurationMilliseconds, &out.DurationMilliseconds + *out = new(int64) + **out = **in + } + if in.DurationFrom != nil { + in, out := &in.DurationFrom, &out.DurationFrom + *out = new(ExpressionFromSource) + **out = **in + } + if in.JitterDurationMilliseconds != nil { + in, out := &in.JitterDurationMilliseconds, &out.JitterDurationMilliseconds + *out = new(int64) + **out = **in + } + if in.JitterDurationFrom != nil { + in, out := &in.JitterDurationFrom, &out.JitterDurationFrom + *out = new(ExpressionFromSource) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageDelay. +func (in *StageDelay) DeepCopy() *StageDelay { + if in == nil { + return nil + } + out := new(StageDelay) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageEvent) DeepCopyInto(out *StageEvent) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageEvent. +func (in *StageEvent) DeepCopy() *StageEvent { + if in == nil { + return nil + } + out := new(StageEvent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageFinalizers) DeepCopyInto(out *StageFinalizers) { + *out = *in + if in.Add != nil { + in, out := &in.Add, &out.Add + *out = make([]FinalizerItem, len(*in)) + copy(*out, *in) + } + if in.Remove != nil { + in, out := &in.Remove, &out.Remove + *out = make([]FinalizerItem, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageFinalizers. +func (in *StageFinalizers) DeepCopy() *StageFinalizers { + if in == nil { + return nil + } + out := new(StageFinalizers) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageList) DeepCopyInto(out *StageList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Stage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageList. +func (in *StageList) DeepCopy() *StageList { + if in == nil { + return nil + } + out := new(StageList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StageList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageNext) DeepCopyInto(out *StageNext) { + *out = *in + if in.Event != nil { + in, out := &in.Event, &out.Event + *out = new(StageEvent) + **out = **in + } + if in.Finalizers != nil { + in, out := &in.Finalizers, &out.Finalizers + *out = new(StageFinalizers) + (*in).DeepCopyInto(*out) + } + if in.Patches != nil { + in, out := &in.Patches, &out.Patches + *out = make([]StagePatch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StatusSubresource != nil { + in, out := &in.StatusSubresource, &out.StatusSubresource + *out = new(string) + **out = **in + } + if in.StatusPatchAs != nil { + in, out := &in.StatusPatchAs, &out.StatusPatchAs + *out = new(ImpersonationConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageNext. +func (in *StageNext) DeepCopy() *StageNext { + if in == nil { + return nil + } + out := new(StageNext) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StagePatch) DeepCopyInto(out *StagePatch) { + *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(StagePatchType) + **out = **in + } + if in.Impersonation != nil { + in, out := &in.Impersonation, &out.Impersonation + *out = new(ImpersonationConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagePatch. +func (in *StagePatch) DeepCopy() *StagePatch { + if in == nil { + return nil + } + out := new(StagePatch) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageResourceRef) DeepCopyInto(out *StageResourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageResourceRef. +func (in *StageResourceRef) DeepCopy() *StageResourceRef { + if in == nil { + return nil + } + out := new(StageResourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageSelector) DeepCopyInto(out *StageSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchAnnotations != nil { + in, out := &in.MatchAnnotations, &out.MatchAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]SelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageSelector. +func (in *StageSelector) DeepCopy() *StageSelector { + if in == nil { + return nil + } + out := new(StageSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageSpec) DeepCopyInto(out *StageSpec) { + *out = *in + out.ResourceRef = in.ResourceRef + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(StageSelector) + (*in).DeepCopyInto(*out) + } + if in.WeightFrom != nil { + in, out := &in.WeightFrom, &out.WeightFrom + *out = new(ExpressionFromSource) + **out = **in + } + if in.Delay != nil { + in, out := &in.Delay, &out.Delay + *out = new(StageDelay) + (*in).DeepCopyInto(*out) + } + in.Next.DeepCopyInto(&out.Next) + if in.ImmediateNextStage != nil { + in, out := &in.ImmediateNextStage, &out.ImmediateNextStage + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageSpec. +func (in *StageSpec) DeepCopy() *StageSpec { + if in == nil { + return nil + } + out := new(StageSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageStatus) DeepCopyInto(out *StageStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageStatus. +func (in *StageStatus) DeepCopy() *StageStatus { + if in == nil { + return nil + } + out := new(StageStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.defaults.go b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.defaults.go new file mode 100644 index 0000000000..7fb9c96ac5 --- /dev/null +++ b/cluster-provision/gocli/vendor/sigs.k8s.io/kwok/pkg/apis/v1alpha1/zz_generated.defaults.go @@ -0,0 +1,70 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Metric{}, func(obj interface{}) { SetObjectDefaults_Metric(obj.(*Metric)) }) + scheme.AddTypeDefaultingFunc(&MetricList{}, func(obj interface{}) { SetObjectDefaults_MetricList(obj.(*MetricList)) }) + scheme.AddTypeDefaultingFunc(&Stage{}, func(obj interface{}) { SetObjectDefaults_Stage(obj.(*Stage)) }) + scheme.AddTypeDefaultingFunc(&StageList{}, func(obj interface{}) { SetObjectDefaults_StageList(obj.(*StageList)) }) + return nil +} + +func SetObjectDefaults_Metric(in *Metric) { + for i := range in.Spec.Metrics { + a := &in.Spec.Metrics[i] + if a.Dimension == "" { + a.Dimension = "node" + } + } +} + +func SetObjectDefaults_MetricList(in *MetricList) { + for i := range in.Items { + a := &in.Items[i] + SetObjectDefaults_Metric(a) + } +} + +func SetObjectDefaults_Stage(in *Stage) { + if in.Spec.ResourceRef.APIGroup == "" { + in.Spec.ResourceRef.APIGroup = "v1" + } + if in.Spec.Next.StatusSubresource == nil { + var ptrVar1 string = "status" + in.Spec.Next.StatusSubresource = &ptrVar1 + } +} + +func SetObjectDefaults_StageList(in *StageList) { + for i := range in.Items { + a := &in.Items[i] + SetObjectDefaults_Stage(a) + } +} diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index c0e0bae782..f0ac2f5322 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -248,6 +248,10 @@ function _add_common_params() { params=" --docker-proxy=$KUBEVIRTCI_PROXY $params" fi + if [ "$KUBEVIRT_DEPLOY_KWOK" == "true" ]; then + params=" --deploy-kwok $params" + fi + echo $params } diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 47ccc96d98..4b5c2d92ed 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -5,15 +5,6 @@ set -e # shellcheck source=cluster-up/cluster/ephemeral-provider-common.sh source "${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh" - - -function deploy_kwok() { - if [[ ${KUBEVIRT_DEPLOY_KWOK} == "true" ]]; then - $kubectl create -f /opt/kwok/kwok.yaml - $kubectl create -f /opt/kwok/stage-fast.yaml - fi -} - # copy_istio_cni_conf_files copy the generated Istio CNI net conf file # (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/') function copy_istio_cni_conf_files() { @@ -98,8 +89,6 @@ function up() { configure_prometheus configure_cpu_manager - deploy_kwok - until wait_for_kwok_ready; do echo "Waiting for cluster components..." sleep 5 From 1321a81d295840a72a8e312cdb14f66993c93139 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 19:34:05 +0200 Subject: [PATCH 57/69] authenticate the api server Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 6c60e8279b..ff58d01bc8 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -58,6 +58,8 @@ func buildApiServerCmdArgs() map[string]string { "--kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", "--secure-port": "6443", "--v": "3", + "--kubelet-client-certificate": "/etc/kubernetes/pki/apiserver-kubelet-client.crt", + "--kubelet-client-key": "/etc/kubernetes/pki/apiserver-kubelet-client.pem", "--service-account-issuer": "https://kubernetes.default.svc.cluster.local", "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.pem", "--service-account-signing-key-file": "/etc/kubernetes/pki/service-accounts.pem", From 500b641d48a85fbe734fbc6329aecc395813505f Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 19:44:51 +0200 Subject: [PATCH 58/69] add fake control plane label Signed-off-by: aerosouund --- cluster-up/cluster/k8s-provider-common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 4b5c2d92ed..14f1ead015 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -84,6 +84,8 @@ function up() { else label="node-role.kubernetes.io/control-plane" fi + + $kubectl label node node01 node-role.kubernetes.io/control-plane $kubectl label node -l $label node-role.kubernetes.io/worker='' configure_prometheus From 8ffbf2b39395ffe0ca7cc88e6820374628bd4527 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 19:52:34 +0200 Subject: [PATCH 59/69] label node correctly Signed-off-by: aerosouund --- cluster-up/cluster/k8s-provider-common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 14f1ead015..962c77fcbf 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -84,8 +84,8 @@ function up() { else label="node-role.kubernetes.io/control-plane" fi - - $kubectl label node node01 node-role.kubernetes.io/control-plane + + $kubectl label node node01 node-role.kubernetes.io/control-plane="" $kubectl label node -l $label node-role.kubernetes.io/worker='' configure_prometheus From 7fdcb41efe9fa97affd7c73479dd98ef30478177 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 8 Nov 2024 20:18:42 +0200 Subject: [PATCH 60/69] request header Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 50 +++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index ff58d01bc8..988797588b 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -43,28 +43,32 @@ func buildControllerMgrCmdArgs() map[string]string { func buildApiServerCmdArgs() map[string]string { return map[string]string{ - "--advertise-address": "192.168.66.110", - "--allow-privileged": "true", - "--audit-log-format": "json", - "--audit-log-path": "/var/log/k8s-audit/k8s-audit.log", - "--authorization-mode": "Node,RBAC", - "--client-ca-file": "/etc/kubernetes/pki/ca.crt", - "--enable-admission-plugins": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", - "--enable-bootstrap-token-auth": "true", - "--etcd-cafile": "/etc/kubernetes/pki/ca.crt", - "--etcd-certfile": "/etc/kubernetes/pki/apiserver.crt", - "--etcd-keyfile": "/etc/kubernetes/pki/apiserver.pem", - "--etcd-servers": "https://127.0.0.1:2379", - "--kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", - "--secure-port": "6443", - "--v": "3", - "--kubelet-client-certificate": "/etc/kubernetes/pki/apiserver-kubelet-client.crt", - "--kubelet-client-key": "/etc/kubernetes/pki/apiserver-kubelet-client.pem", - "--service-account-issuer": "https://kubernetes.default.svc.cluster.local", - "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.pem", - "--service-account-signing-key-file": "/etc/kubernetes/pki/service-accounts.pem", - "--service-cluster-ip-range": "10.96.0.0/24", - "--tls-cert-file": "/etc/kubernetes/pki/apiserver.crt", - "--tls-private-key-file": "/etc/kubernetes/pki/apiserver.pem", + "--advertise-address": "192.168.66.110", + "--allow-privileged": "true", + "--audit-log-format": "json", + "--audit-log-path": "/var/log/k8s-audit/k8s-audit.log", + "--authorization-mode": "Node,RBAC", + "--client-ca-file": "/etc/kubernetes/pki/ca.crt", + "--enable-admission-plugins": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + "--enable-bootstrap-token-auth": "true", + "--etcd-cafile": "/etc/kubernetes/pki/ca.crt", + "--etcd-certfile": "/etc/kubernetes/pki/apiserver.crt", + "--etcd-keyfile": "/etc/kubernetes/pki/apiserver.pem", + "--etcd-servers": "https://127.0.0.1:2379", + "--kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + "--secure-port": "6443", + "--v": "3", + "--kubelet-client-certificate": "/etc/kubernetes/pki/apiserver-kubelet-client.crt", + "--kubelet-client-key": "/etc/kubernetes/pki/apiserver-kubelet-client.pem", + "--service-account-issuer": "https://kubernetes.default.svc.cluster.local", + "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.pem", + "--service-account-signing-key-file": "/etc/kubernetes/pki/service-accounts.pem", + "--service-cluster-ip-range": "10.96.0.0/24", + "--tls-cert-file": "/etc/kubernetes/pki/apiserver.crt", + "--tls-private-key-file": "/etc/kubernetes/pki/apiserver.pem", + "--requestheader-client-ca-file": "/etc/kubernetes/pki/ca.crt", + "--requestheader-extra-headers-prefix": "X-Remote-Extra-", + "--requestheader-group-headers": "X-Remote-Group", + "--requestheader-username-headers": "X-Remote-User", } } From e09e1e7aa1cf987dc7ae86fa9458b7d36e4332e1 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 9 Nov 2024 14:39:29 +0200 Subject: [PATCH 61/69] different iptables Signed-off-by: aerosouund --- .../control-plane/config/kube-proxy.yaml | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml index e6223aa605..c2c7d715bb 100644 --- a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml +++ b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml @@ -20,38 +20,69 @@ data: config: |- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 + bindAddressHardFail: false clientConnection: acceptContentTypes: "" - burst: 10 - contentType: application/vnd.kubernetes.protobuf - kubeconfig: /var/lib/kube-proxy/kubeconfig - qps: 5 - clusterCIDR: "" - configSyncPeriod: 15m0s + burst: 0 + contentType: "" + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 0 + clusterCIDR: 10.244.0.0/16,fd10:244::/112 + configSyncPeriod: 0s conntrack: - maxPerCore: 32768 - min: 131072 - tcpCloseWaitTimeout: 1h0m0s - tcpEstablishedTimeout: 24h0m0s + maxPerCore: null + min: null + tcpBeLiberal: false + tcpCloseWaitTimeout: null + tcpEstablishedTimeout: null + udpStreamTimeout: 0s + udpTimeout: 0s + detectLocal: + bridgeInterface: "" + interfaceNamePrefix: "" + detectLocalMode: "" enableProfiling: false - healthzBindAddress: 0.0.0.0:10256 + healthzBindAddress: "" hostnameOverride: "" iptables: + localhostNodePorts: null masqueradeAll: false - masqueradeBit: 14 + masqueradeBit: null minSyncPeriod: 0s - syncPeriod: 30s + syncPeriod: 0s ipvs: excludeCIDRs: null minSyncPeriod: 0s scheduler: "" - syncPeriod: 30s + strictARP: false + syncPeriod: 0s + tcpFinTimeout: 0s + tcpTimeout: 0s + udpTimeout: 0s kind: KubeProxyConfiguration - metricsBindAddress: 0.0.0.0:10205 + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + metricsBindAddress: "" mode: iptables + nftables: + masqueradeAll: false + masqueradeBit: null + minSyncPeriod: 0s + syncPeriod: 0s nodePortAddresses: null - oomScoreAdj: -999 + oomScoreAdj: null portRange: "" + showHiddenMetricsForVersion: "" + winkernel: + enableDSR: false + forwardHealthCheckVip: false + networkName: "" + rootHnsEndpointName: "" + sourceVip: "" kubeconfig: |- kind: Config apiVersion: v1 From 2adba05cf3c4aa42fb0936fdb20e3a4a006d815b Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 9 Nov 2024 14:54:15 +0200 Subject: [PATCH 62/69] kubeconfig Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/config/kube-proxy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml index c2c7d715bb..a98dc156c9 100644 --- a/cluster-provision/gocli/control-plane/config/kube-proxy.yaml +++ b/cluster-provision/gocli/control-plane/config/kube-proxy.yaml @@ -25,7 +25,7 @@ data: acceptContentTypes: "" burst: 0 contentType: "" - kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + kubeconfig: /var/lib/kube-proxy/kubeconfig qps: 0 clusterCIDR: 10.244.0.0/16,fd10:244::/112 configSyncPeriod: 0s From 1216e701984f43e99584da574254115f29e73b55 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 13:19:07 +0200 Subject: [PATCH 63/69] konnectivity Signed-off-by: aerosouund --- .../gocli/control-plane/certs/certlist.go | 15 +++ cluster-provision/gocli/control-plane/cmd.go | 21 +++++ .../control-plane/config/egress-selector.yaml | 9 ++ .../config/konnectivity-agent.yaml | 76 ++++++++++++++++ .../control-plane/control-plane-components.go | 23 ++++- .../gocli/control-plane/konnectivity-agent.go | 42 +++++++++ .../control-plane/konnectivity-server.go | 91 +++++++++++++++++++ cluster-provision/gocli/control-plane/main.go | 5 + 8 files changed, 278 insertions(+), 4 deletions(-) create mode 100644 cluster-provision/gocli/control-plane/config/egress-selector.yaml create mode 100644 cluster-provision/gocli/control-plane/config/konnectivity-agent.yaml create mode 100644 cluster-provision/gocli/control-plane/konnectivity-agent.go create mode 100644 cluster-provision/gocli/control-plane/konnectivity-server.go diff --git a/cluster-provision/gocli/control-plane/certs/certlist.go b/cluster-provision/gocli/control-plane/certs/certlist.go index 304e5839ec..77c3f9d260 100644 --- a/cluster-provision/gocli/control-plane/certs/certlist.go +++ b/cluster-provision/gocli/control-plane/certs/certlist.go @@ -27,6 +27,7 @@ func GetAllCertificates() Certificates { KubeletClientCert(), ServiceAccountsCert(), AdminCert(), + KonnectivityCert(), } } @@ -83,6 +84,20 @@ func ControllerMgrCert() *Cert { } } +func KonnectivityCert() *Cert { + return &Cert{ + Name: "konnectivity", + LongName: "certificate for the konnectivity proxy", + BaseName: "konnectivity", + CAName: "ca", + Config: cert.Config{ + CommonName: "system:konnectivity-server", + Organization: []string{"system:konnectivity-server"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + }, + } +} + func ServiceAccountsCert() *Cert { return &Cert{ Name: "service-accounts", diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 988797588b..97889ebfb3 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -1,5 +1,25 @@ package controlplane +func buildKonnectivityArgs() map[string]string { + return map[string]string{ + "--uds-name": "/etc/kubernetes/pki/konnectivity/konnectivity-server.sock", + "--cluster-cert": "/etc/kubernetes/pki/apiserver.pem", + "--cluster-key": "/etc/kubernetes/pki/apiserver-key.pem", + "--mode": "grpc", + "--proxy-strategies": "default", + "--logtostderr": "true", + "--v": "4", + "--agent-port": "8132", + "--admin-port": "8133", + "--health-port": "8134", + "--advertise-address": "192.168.66.110", + "--authentication-audience": "system:konnectivity-server", + "--kubeconfig": "/etc/kubernetes/pki/konnectivity/.kubeconfig", + "--agent-namespace": "kube-system", + "--agent-service-account": "konnectivity-agent", + } +} + func buildEtcdCmdArgs() map[string]string { return map[string]string{ "--advertise-client-urls": "https://0.0.0.0:2379", @@ -64,6 +84,7 @@ func buildApiServerCmdArgs() map[string]string { "--service-account-key-file": "/etc/kubernetes/pki/service-accounts.pem", "--service-account-signing-key-file": "/etc/kubernetes/pki/service-accounts.pem", "--service-cluster-ip-range": "10.96.0.0/24", + "--egress-selector-config-file": "/etc/kubernetes/pki/egress-selector.yaml", "--tls-cert-file": "/etc/kubernetes/pki/apiserver.crt", "--tls-private-key-file": "/etc/kubernetes/pki/apiserver.pem", "--requestheader-client-ca-file": "/etc/kubernetes/pki/ca.crt", diff --git a/cluster-provision/gocli/control-plane/config/egress-selector.yaml b/cluster-provision/gocli/control-plane/config/egress-selector.yaml new file mode 100644 index 0000000000..7c855df13c --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/egress-selector.yaml @@ -0,0 +1,9 @@ +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: EgressSelectorConfiguration +egressSelections: +- name: cluster + connection: + proxyProtocol: GRPC + transport: + uds: + udsName: /etc/kubernetes/pki/konnectivity/konnectivity-server.sock diff --git a/cluster-provision/gocli/control-plane/config/konnectivity-agent.yaml b/cluster-provision/gocli/control-plane/config/konnectivity-agent.yaml new file mode 100644 index 0000000000..a9787a8be3 --- /dev/null +++ b/cluster-provision/gocli/control-plane/config/konnectivity-agent.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:konnectivity-server + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:konnectivity-server +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: konnectivity-agent + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: konnectivity-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + template: + metadata: + labels: + k8s-app: konnectivity-agent + spec: + priorityClassName: system-cluster-critical + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.37 + name: konnectivity-agent + command: ["/proxy-agent"] + args: [ + "--logtostderr=true", + "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "--proxy-server-host=192.168.66.110", + "--proxy-server-port=8132", + "--admin-server-port=8133", + "--health-server-port=8134", + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token" + ] + volumeMounts: + - mountPath: /var/run/secrets/tokens + name: konnectivity-agent-token + livenessProbe: + httpGet: + port: 8134 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + serviceAccountName: konnectivity-agent + volumes: + - name: konnectivity-agent-token + projected: + sources: + - serviceAccountToken: + path: konnectivity-agent-token + audience: system:konnectivity-server diff --git a/cluster-provision/gocli/control-plane/control-plane-components.go b/cluster-provision/gocli/control-plane/control-plane-components.go index 700f770468..d4d928e281 100644 --- a/cluster-provision/gocli/control-plane/control-plane-components.go +++ b/cluster-provision/gocli/control-plane/control-plane-components.go @@ -1,11 +1,17 @@ package controlplane import ( + _ "embed" + "os" + "path" "time" "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" ) +//go:embed config/egress-selector.yaml +var eg []byte + var versionMap = map[string]string{ "1.30": "v1.30.2", "1.29": "v1.29.6", @@ -49,6 +55,15 @@ func (p *RunControlPlaneComponentsPhase) runApiServer() error { return err } + f, err := os.Create(path.Join(p.pkiPath, "egress-selector.yaml")) + if err != nil { + return err + } + + if _, err = f.Write(eg); err != nil { + return err + } + args := buildApiServerCmdArgs() cmd := []string{"kube-apiserver"} @@ -100,12 +115,12 @@ func (p *RunControlPlaneComponentsPhase) runControllerMgr() error { Command: cmd, } - apiserverContainer, err := p.containerRuntime.Create(ctrlMgrImage, createOpts) // todo: variable names + ctrlMgr, err := p.containerRuntime.Create(ctrlMgrImage, createOpts) // todo: variable names if err != nil { return err } - err = p.containerRuntime.Start(apiserverContainer) + err = p.containerRuntime.Start(ctrlMgr) if err != nil { return err } @@ -130,12 +145,12 @@ func (p *RunControlPlaneComponentsPhase) runScheduler() error { Command: cmd, } - apiserverContainer, err := p.containerRuntime.Create(schedulerImage, createOpts) + sched, err := p.containerRuntime.Create(schedulerImage, createOpts) if err != nil { return err } - err = p.containerRuntime.Start(apiserverContainer) + err = p.containerRuntime.Start(sched) if err != nil { return err } diff --git a/cluster-provision/gocli/control-plane/konnectivity-agent.go b/cluster-provision/gocli/control-plane/konnectivity-agent.go new file mode 100644 index 0000000000..06ab2d48dc --- /dev/null +++ b/cluster-provision/gocli/control-plane/konnectivity-agent.go @@ -0,0 +1,42 @@ +package controlplane + +import ( + "bytes" + _ "embed" + "fmt" + + "github.com/sirupsen/logrus" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +//go:embed config/konnectivity-agent.yaml +var ka []byte + +type KonnectivityAgentPhase struct { + client k8s.K8sDynamicClient +} + +func NewKonnectivityAgentPhase(client k8s.K8sDynamicClient) *KonnectivityAgentPhase { + return &KonnectivityAgentPhase{ + client: client, + } +} + +func (k *KonnectivityAgentPhase) Run() error { + yamlDocs := bytes.Split(kp, []byte("---\n")) + for _, yamlDoc := range yamlDocs { + if len(yamlDoc) == 0 { + continue + } + + obj, err := k8s.SerializeIntoObject(yamlDoc) + if err != nil { + logrus.Info(err.Error()) + continue + } + if err := k.client.Apply(obj); err != nil { + return fmt.Errorf("error applying manifest %s", err) + } + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/konnectivity-server.go b/cluster-provision/gocli/control-plane/konnectivity-server.go new file mode 100644 index 0000000000..c066bcad04 --- /dev/null +++ b/cluster-provision/gocli/control-plane/konnectivity-server.go @@ -0,0 +1,91 @@ +package controlplane + +import ( + "fmt" + "os" + "path" + + "k8s.io/client-go/tools/clientcmd" + "kubevirt.io/kubevirtci/cluster-provision/gocli/cri" +) + +type KonnectivityPhase struct { + pkiPath string + dnsmasqID string + serverImage string + k8sVersion string + containerRuntime cri.ContainerClient +} + +func NewKonnectivityPhase(cr cri.ContainerClient, pkiPath, serverImage, dnsmasqID, k8sVersion string) *KonnectivityPhase { + return &KonnectivityPhase{ + dnsmasqID: dnsmasqID, + pkiPath: pkiPath, + containerRuntime: cr, + serverImage: serverImage, + k8sVersion: k8sVersion, + } +} + +func (k *KonnectivityPhase) Run() error { + if err := k.createKonnectivityKubeConfig(); err != nil { + return err + } + + img := registry + "/" + k.serverImage + err := k.containerRuntime.ImagePull(img) + if err != nil { + return err + } + + args := buildKonnectivityArgs() + + cmd := []string{"konnectivity-server"} + for flag, values := range args { + cmd = append(cmd, flag+"="+values) + } + + createOpts := &cri.CreateOpts{ + Name: "k8s-" + k.k8sVersion + "-konnectivity", + Mounts: map[string]string{ + k.pkiPath: "/etc/kubernetes/pki/", + }, + Network: "container:" + k.dnsmasqID, + Command: cmd, + } + + konnectivityContainer, err := k.containerRuntime.Create(img, createOpts) + if err != nil { + return err + } + + err = k.containerRuntime.Start(konnectivityContainer) + if err != nil { + return err + } + return nil +} + +func (k *KonnectivityPhase) createKonnectivityKubeConfig() error { + ca, err := os.ReadFile(path.Join(k.pkiPath, "ca.crt")) + if err != nil { + return err + } + + clientCert, err := os.ReadFile(path.Join(k.pkiPath, "konnectivity.crt")) + if err != nil { + return err + } + + key, err := os.ReadFile(path.Join(k.pkiPath, "konnectivity.pem")) + if err != nil { + return err + } + + kubeconfig := buildKubeConfigFromCerts(ca, clientCert, key, "https://127.0.0.1:6443", "system:konnectivity-server") + err = clientcmd.WriteToFile(kubeconfig, k.pkiPath+"/konnectivity/.kubeconfig") + if err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + return nil +} diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 40d2ea8e93..2e54c6ea51 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -18,6 +18,7 @@ const ( scheduler = "kube-scheduler" registry = "registry.k8s.io" defaultPkiPath = "/etc/kubevirtci/pki" + konnectivityImage = "kas-network-proxy/proxy-server:v0.28.6" ) type ControlPlaneRunner struct { @@ -63,6 +64,10 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } + if err := NewKonnectivityPhase(cp.containerRuntime, defaultPkiPath, konnectivityImage, cp.dnsmasqID, cp.k8sVersion).Run(); err != nil { + return nil, err + } + if err := NewRunControlPlaneComponentsPhase(cp.dnsmasqID, cp.containerRuntime, defaultPkiPath, cp.k8sVersion).Run(); err != nil { return nil, err } From a0024d83513eece956418604f02eeefda5522480 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 13:37:50 +0200 Subject: [PATCH 64/69] crt Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 97889ebfb3..9052db9815 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -3,8 +3,8 @@ package controlplane func buildKonnectivityArgs() map[string]string { return map[string]string{ "--uds-name": "/etc/kubernetes/pki/konnectivity/konnectivity-server.sock", - "--cluster-cert": "/etc/kubernetes/pki/apiserver.pem", - "--cluster-key": "/etc/kubernetes/pki/apiserver-key.pem", + "--cluster-cert": "/etc/kubernetes/pki/apiserver.crt", + "--cluster-key": "/etc/kubernetes/pki/apiserver.pem", "--mode": "grpc", "--proxy-strategies": "default", "--logtostderr": "true", @@ -12,7 +12,6 @@ func buildKonnectivityArgs() map[string]string { "--agent-port": "8132", "--admin-port": "8133", "--health-port": "8134", - "--advertise-address": "192.168.66.110", "--authentication-audience": "system:konnectivity-server", "--kubeconfig": "/etc/kubernetes/pki/konnectivity/.kubeconfig", "--agent-namespace": "kube-system", From 0f71564e923a3e05e1d0445814b388c5ad45e88e Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 13:39:26 +0200 Subject: [PATCH 65/69] port 0 Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/cmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-provision/gocli/control-plane/cmd.go b/cluster-provision/gocli/control-plane/cmd.go index 9052db9815..d83c5c4c4b 100644 --- a/cluster-provision/gocli/control-plane/cmd.go +++ b/cluster-provision/gocli/control-plane/cmd.go @@ -10,6 +10,7 @@ func buildKonnectivityArgs() map[string]string { "--logtostderr": "true", "--v": "4", "--agent-port": "8132", + "--server-port": "0", "--admin-port": "8133", "--health-port": "8134", "--authentication-audience": "system:konnectivity-server", From f38f943fcdc3594932ce271107c0a31c540fec96 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 13:44:00 +0200 Subject: [PATCH 66/69] agent Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster-provision/gocli/control-plane/main.go b/cluster-provision/gocli/control-plane/main.go index 2e54c6ea51..3d0053a98d 100644 --- a/cluster-provision/gocli/control-plane/main.go +++ b/cluster-provision/gocli/control-plane/main.go @@ -98,6 +98,10 @@ func (cp *ControlPlaneRunner) Start() (*rest.Config, error) { return nil, err } + if err := NewKonnectivityAgentPhase(k8sClient).Run(); err != nil { + return nil, err + } + if err := NewCoreDNSPhase(k8sClient).Run(); err != nil { return nil, err } From c8e9edbec7afee39a7edd3e13d4c81b540e069cb Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 13:46:06 +0200 Subject: [PATCH 67/69] already exists Signed-off-by: aerosouund --- cluster-provision/gocli/control-plane/konnectivity-agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/gocli/control-plane/konnectivity-agent.go b/cluster-provision/gocli/control-plane/konnectivity-agent.go index 06ab2d48dc..665168ce29 100644 --- a/cluster-provision/gocli/control-plane/konnectivity-agent.go +++ b/cluster-provision/gocli/control-plane/konnectivity-agent.go @@ -23,7 +23,7 @@ func NewKonnectivityAgentPhase(client k8s.K8sDynamicClient) *KonnectivityAgentPh } func (k *KonnectivityAgentPhase) Run() error { - yamlDocs := bytes.Split(kp, []byte("---\n")) + yamlDocs := bytes.Split(ka, []byte("---\n")) for _, yamlDoc := range yamlDocs { if len(yamlDoc) == 0 { continue From 384980621c84a2236aedab94b0dc849dcdf47792 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 15:44:09 +0200 Subject: [PATCH 68/69] mount etc Signed-off-by: aerosouund --- cluster-up/cluster/ephemeral-provider-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index f0ac2f5322..6786b1d9e2 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -46,7 +46,7 @@ else fi _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}" -_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v /lib:/lib -v /lib64:/lib64 -v /usr/lib:/usr/lib -v /usr/bin:/usr/bin" +_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock -v /lib:/lib -v /etc:/etc:ro -v /lib64:/lib64 -v /usr/lib:/usr/lib -v /usr/bin:/usr/bin" echo $_cli # gocli will try to mount /lib/modules to make it accessible to dnsmasq in # in case it exists From e3c68c7daf83292465634ab223c397a9621b5c8a Mon Sep 17 00:00:00 2001 From: aerosouund Date: Sat, 16 Nov 2024 19:45:04 +0200 Subject: [PATCH 69/69] dont even remember why i did this Signed-off-by: aerosouund --- cluster-provision/centos9/scripts/dnsmasq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-provision/centos9/scripts/dnsmasq.sh b/cluster-provision/centos9/scripts/dnsmasq.sh index 928ca35a6f..3673ea82fc 100755 --- a/cluster-provision/centos9/scripts/dnsmasq.sh +++ b/cluster-provision/centos9/scripts/dnsmasq.sh @@ -3,7 +3,7 @@ set -ex NUM_NODES=${NUM_NODES-1} -NUM_SECONDARY_NICS=0 +NUM_SECONDARY_NICS=${NUM_SECONDARY_NICS:-0} SEQ_START=1 ip link add br0 type bridge