Merge pull request #2059 from haircommander/min-kubelet-version

add minimumKubeletVersion

config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml

@@ -0,0 +1,44 @@
+apiVersion: apiextensions.k8s.io/v1 # Hack because controller-gen complains if we don't have this
+name: "Node"
+crdName: nodes.config.openshift.io
+featureGate: MinimumKubeletVersion
+tests:
+  onCreate:
+    - name: Should be able to create a minimal Node
+      initial: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec: {} # No spec is required for a Node
+      expected: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec: {}
+    - name: Should be able to create an empty minimumKubeletVersion
+      initial: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec:
+          minimumKubeletVersion: ""
+      expected: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec:
+          minimumKubeletVersion: ""
+    - name: Should be able to create a minimumKubeletVersion
+      initial: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec:
+          minimumKubeletVersion: 1.30.0
+      expected: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec:
+          minimumKubeletVersion: 1.30.0
+    - name: Should fail to create a bogus version
+      initial: |
+        apiVersion: config.openshift.io/v1
+        kind: Node
+        spec:
+          minimumKubeletVersion: bogus
+      expectedError: "Invalid value: \"string\": minmumKubeletVersion must be in a semver compatible format of x.y.z, or empty"

config/v1/types_node.go

@@ -46,6 +46,25 @@ type NodeSpec struct {
 	// the status and corresponding reaction of the cluster
 	// +optional
 	WorkerLatencyProfile WorkerLatencyProfileType `json:"workerLatencyProfile,omitempty"`
+
+	// minimumKubeletVersion is the lowest version of a kubelet that can join the cluster.
+	// Specifically, the apiserver will deny most authorization requests of kubelets that are older
+	// than the specified version, only allowing the kubelet to get and update its node object, and perform
+	// subjectaccessreviews.
+	// This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads,
+	// and will eventually be marked as not ready.
+	// Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99".
+	// Since the kubelet reports the version of the kubernetes release, not Openshift, this field references
+	// the underlying kubernetes version this version of Openshift is based off of.
+	// In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then
+	// they should set the minimumKubeletVersion to 1.30.0.
+	// When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version.
+	// Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier.
+	// +kubebuilder:validation:XValidation:rule="self == \"\" || self.matches('^[0-9]*.[0-9]*.[0-9]*$')",message="minmumKubeletVersion must be in a semver compatible format of x.y.z, or empty"
+	// +kubebuilder:validation:MaxLength:=8
+	// +openshift:enable:FeatureGate=MinimumKubeletVersion
+	// +optional
+	MinimumKubeletVersion string `json:"minimumKubeletVersion"`
 }
 
 type NodeStatus struct {

config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml

@@ -0,0 +1,158 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.openshift.io: https://github.com/openshift/api/pull/1107
+    api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/bootstrap-required: "true"
+    release.openshift.io/feature-set: CustomNoUpgrade
+  name: nodes.config.openshift.io
+spec:
+  group: config.openshift.io
+  names:
+    kind: Node
+    listKind: NodeList
+    plural: nodes
+    singular: node
+  scope: Cluster
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          Node holds cluster-wide information about node specific features.
+
+          Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer).
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: spec holds user settable values for configuration
+            properties:
+              cgroupMode:
+                description: CgroupMode determines the cgroups version on the node
+                enum:
+                - v1
+                - v2
+                - ""
+                type: string
+              minimumKubeletVersion:
+                description: |-
+                  minimumKubeletVersion is the lowest version of a kubelet that can join the cluster.
+                  Specifically, the apiserver will deny most authorization requests of kubelets that are older
+                  than the specified version, only allowing the kubelet to get and update its node object, and perform
+                  subjectaccessreviews.
+                  This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads,
+                  and will eventually be marked as not ready.
+                  Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99".
+                  Since the kubelet reports the version of the kubernetes release, not Openshift, this field references
+                  the underlying kubernetes version this version of Openshift is based off of.
+                  In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then
+                  they should set the minimumKubeletVersion to 1.30.0.
+                  When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version.
+                  Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier.
+                maxLength: 8
+                type: string
+                x-kubernetes-validations:
+                - message: minmumKubeletVersion must be in a semver compatible format
+                    of x.y.z, or empty
+                  rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$')
+              workerLatencyProfile:
+                description: |-
+                  WorkerLatencyProfile determins the how fast the kubelet is updating
+                  the status and corresponding reaction of the cluster
+                enum:
+                - Default
+                - MediumUpdateAverageReaction
+                - LowUpdateSlowReaction
+                type: string
+            type: object
+          status:
+            description: status holds observed values.
+            properties:
+              conditions:
+                description: conditions contain the details and the current state
+                  of the nodes.config object
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml → config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-Default.crd.yaml

@@ -7,6 +7,7 @@ metadata:
     include.release.openshift.io/ibm-cloud-managed: "true"
     include.release.openshift.io/self-managed-high-availability: "true"
     release.openshift.io/bootstrap-required: "true"
+    release.openshift.io/feature-set: Default
   name: nodes.config.openshift.io
 spec:
   group: config.openshift.io

config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml

@@ -0,0 +1,158 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.openshift.io: https://github.com/openshift/api/pull/1107
+    api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/bootstrap-required: "true"
+    release.openshift.io/feature-set: DevPreviewNoUpgrade
+  name: nodes.config.openshift.io
+spec:
+  group: config.openshift.io
+  names:
+    kind: Node
+    listKind: NodeList
+    plural: nodes
+    singular: node
+  scope: Cluster
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          Node holds cluster-wide information about node specific features.
+
+          Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer).
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: spec holds user settable values for configuration
+            properties:
+              cgroupMode:
+                description: CgroupMode determines the cgroups version on the node
+                enum:
+                - v1
+                - v2
+                - ""
+                type: string
+              minimumKubeletVersion:
+                description: |-
+                  minimumKubeletVersion is the lowest version of a kubelet that can join the cluster.
+                  Specifically, the apiserver will deny most authorization requests of kubelets that are older
+                  than the specified version, only allowing the kubelet to get and update its node object, and perform
+                  subjectaccessreviews.
+                  This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads,
+                  and will eventually be marked as not ready.
+                  Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99".
+                  Since the kubelet reports the version of the kubernetes release, not Openshift, this field references
+                  the underlying kubernetes version this version of Openshift is based off of.
+                  In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then
+                  they should set the minimumKubeletVersion to 1.30.0.
+                  When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version.
+                  Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier.
+                maxLength: 8
+                type: string
+                x-kubernetes-validations:
+                - message: minmumKubeletVersion must be in a semver compatible format
+                    of x.y.z, or empty
+                  rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$')
+              workerLatencyProfile:
+                description: |-
+                  WorkerLatencyProfile determins the how fast the kubelet is updating
+                  the status and corresponding reaction of the cluster
+                enum:
+                - Default
+                - MediumUpdateAverageReaction
+                - LowUpdateSlowReaction
+                type: string
+            type: object
+          status:
+            description: status holds observed values.
+            properties:
+              conditions:
+                description: conditions contain the details and the current state
+                  of the nodes.config object
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}