diff --git a/apis/apps/v1alpha1/apimanager_types.go b/apis/apps/v1alpha1/apimanager_types.go index 4967df91c..6a88b1c1d 100644 --- a/apis/apps/v1alpha1/apimanager_types.go +++ b/apis/apps/v1alpha1/apimanager_types.go @@ -45,7 +45,6 @@ const ( const ( defaultTenantName = "3scale" - defaultImageStreamImportInsecure = false defaultResourceRequirementsEnabled = true ) @@ -98,7 +97,7 @@ type APIManagerStatus struct { // +patchStrategy=merge Conditions common.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,2,rep,name=conditions"` - // APIManager Deployment Configs + // APIManager Deployments // +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Deployments",xDescriptors="urn:alm:descriptor:com.tectonic.ui:podStatuses" Deployments olm.DeploymentStatus `json:"deployments"` } @@ -130,11 +129,10 @@ func (s *APIManagerStatus) Equals(other *APIManagerStatus, logger logr.Logger) b // APIManager is the Schema for the apimanagers API // +kubebuilder:resource:path=apimanagers,scope=Namespaced // +operator-sdk:csv:customresourcedefinitions:displayName="APIManager" -// +operator-sdk:csv:customresourcedefinitions:resources={{"DeploymentConfig","apps.openshift.io/v1"}} +// +operator-sdk:csv:customresourcedefinitions:resources={{"Deployment","apps/v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"PersistentVolumeClaim","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Service","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Route","route.openshift.io/v1"}} -// +operator-sdk:csv:customresourcedefinitions:resources={{"ImageStream","image.openshift.io/v1"}} type APIManager struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -161,8 +159,6 @@ type APIManagerCommonSpec struct { // +optional TenantName *string `json:"tenantName,omitempty"` // +optional - ImageStreamTagImportInsecure *bool `json:"imageStreamTagImportInsecure,omitempty"` - // +optional ResourceRequirementsEnabled *bool `json:"resourceRequirementsEnabled,omitempty"` // +optional ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` @@ -977,7 +973,6 @@ func (apimanager *APIManager) setAPIManagerCommonSpecDefaults() bool { tmpDefaultAppLabel := Default3scaleAppLabel tmpDefaultTenantName := defaultTenantName - tmpDefaultImageStreamTagImportInsecure := defaultImageStreamImportInsecure tmpDefaultResourceRequirementsEnabled := defaultResourceRequirementsEnabled if spec.AppLabel == nil { @@ -990,11 +985,6 @@ func (apimanager *APIManager) setAPIManagerCommonSpecDefaults() bool { changed = true } - if spec.ImageStreamTagImportInsecure == nil { - spec.ImageStreamTagImportInsecure = &tmpDefaultImageStreamTagImportInsecure - changed = true - } - if spec.ResourceRequirementsEnabled == nil { spec.ResourceRequirementsEnabled = &tmpDefaultResourceRequirementsEnabled changed = true diff --git a/apis/apps/v1alpha1/apimanager_types_test.go b/apis/apps/v1alpha1/apimanager_types_test.go index 0ded8b340..792ac1bdb 100644 --- a/apis/apps/v1alpha1/apimanager_types_test.go +++ b/apis/apps/v1alpha1/apimanager_types_test.go @@ -14,7 +14,6 @@ import ( func TestSetDefaults(t *testing.T) { tmpDefaultAppLabel := Default3scaleAppLabel tmpDefaultTenantName := defaultTenantName - tmpDefaultImageStreamTagImportInsecure := defaultImageStreamImportInsecure tmpDefaultResourceRequirementsEnabled := defaultResourceRequirementsEnabled tmpDefaultApicastManagementAPI := defaultApicastManagementAPI tmpDefaultApicastOpenSSLVerify := defaultApicastOpenSSLVerify @@ -32,11 +31,10 @@ func TestSetDefaults(t *testing.T) { }, Spec: APIManagerSpec{ APIManagerCommonSpec: APIManagerCommonSpec{ - WildcardDomain: "test.3scale.com", - AppLabel: &tmpDefaultAppLabel, - TenantName: &tmpDefaultTenantName, - ImageStreamTagImportInsecure: &tmpDefaultImageStreamTagImportInsecure, - ResourceRequirementsEnabled: &tmpDefaultResourceRequirementsEnabled, + WildcardDomain: "test.3scale.com", + AppLabel: &tmpDefaultAppLabel, + TenantName: &tmpDefaultTenantName, + ResourceRequirementsEnabled: &tmpDefaultResourceRequirementsEnabled, }, Apicast: &ApicastSpec{ IncludeResponseCodes: &tmpDefaultApicastResponseCodes, diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 1287f3f84..d34eacd46 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -205,11 +205,6 @@ func (in *APIManagerCommonSpec) DeepCopyInto(out *APIManagerCommonSpec) { *out = new(string) **out = **in } - if in.ImageStreamTagImportInsecure != nil { - in, out := &in.ImageStreamTagImportInsecure, &out.ImageStreamTagImportInsecure - *out = new(bool) - **out = **in - } if in.ResourceRequirementsEnabled != nil { in, out := &in.ResourceRequirementsEnabled, &out.ResourceRequirementsEnabled *out = new(bool) diff --git a/bundle/manifests/3scale-operator.clusterserviceversion.yaml b/bundle/manifests/3scale-operator.clusterserviceversion.yaml index a9a3fcca2..e7c858243 100644 --- a/bundle/manifests/3scale-operator.clusterserviceversion.yaml +++ b/bundle/manifests/3scale-operator.clusterserviceversion.yaml @@ -242,12 +242,9 @@ spec: kind: APIManager name: apimanagers.apps.3scale.net resources: - - kind: DeploymentConfig + - kind: Deployment name: "" - version: apps.openshift.io/v1 - - kind: ImageStream - name: "" - version: image.openshift.io/v1 + version: apps/v1 - kind: PersistentVolumeClaim name: "" version: v1 @@ -264,7 +261,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:label statusDescriptors: - - description: APIManager Deployment Configs + - description: APIManager Deployments displayName: Deployments path: deployments x-descriptors: diff --git a/bundle/manifests/apps.3scale.net_apimanagers.yaml b/bundle/manifests/apps.3scale.net_apimanagers.yaml index a766ab353..83b4b7267 100644 --- a/bundle/manifests/apps.3scale.net_apimanagers.yaml +++ b/bundle/manifests/apps.3scale.net_apimanagers.yaml @@ -3860,8 +3860,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - imageStreamTagImportInsecure: - type: boolean monitoring: properties: enablePrometheusRules: @@ -10540,7 +10538,7 @@ spec: type: object type: array deployments: - description: APIManager Deployment Configs + description: APIManager Deployments properties: ready: description: Deployments are ready to serve requests diff --git a/config/crd/bases/apps.3scale.net_apimanagers.yaml b/config/crd/bases/apps.3scale.net_apimanagers.yaml index 7a9d90ab3..db8a72852 100644 --- a/config/crd/bases/apps.3scale.net_apimanagers.yaml +++ b/config/crd/bases/apps.3scale.net_apimanagers.yaml @@ -7190,8 +7190,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - imageStreamTagImportInsecure: - type: boolean monitoring: properties: enablePrometheusRules: @@ -19931,7 +19929,7 @@ spec: type: object type: array deployments: - description: APIManager Deployment Configs + description: APIManager Deployments properties: ready: description: Deployments are ready to serve requests diff --git a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml index 4533a416d..7d787f470 100644 --- a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml @@ -56,12 +56,9 @@ spec: kind: APIManager name: apimanagers.apps.3scale.net resources: - - kind: DeploymentConfig + - kind: Deployment name: "" - version: apps.openshift.io/v1 - - kind: ImageStream - name: "" - version: image.openshift.io/v1 + version: apps/v1 - kind: PersistentVolumeClaim name: "" version: v1 @@ -78,7 +75,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:label statusDescriptors: - - description: APIManager Deployment Configs + - description: APIManager Deployments displayName: Deployments path: deployments x-descriptors: diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index 46485f9fc..175ac98d7 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -19,10 +19,10 @@ package controllers import ( "context" "fmt" + "github.com/3scale/3scale-operator/pkg/upgrade" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" - + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apimachinerymetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,7 +40,6 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/operator" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/handlers" - "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/version" @@ -154,7 +153,7 @@ func (r *APIManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(secretToApimanagerEventMapper.Map), builder.WithPredicates(labelSelectorPredicate), ). - Owns(&appsv1.DeploymentConfig{}). + Owns(&k8sappsv1.Deployment{}). Watches(&source.Kind{Type: &routev1.Route{}}, handler.EnqueueRequestsFromMapFunc(handlers.Map)). Complete(r) } @@ -262,6 +261,9 @@ func (r *APIManagerReconciler) reconcileAPIManagerLogic(cr *appsv1alpha1.APIMana return result, err } + // 3scale 2.14 -> 2.15 + err = upgrade.DeleteImageStreams(r.WatchedNamespace, r.Client()) + return ctrl.Result{}, nil } @@ -337,12 +339,10 @@ func (r *APIManagerReconciler) dependencyReconcilerForComponents(cr *appsv1alpha if cr.Spec.System.DatabaseSpec != nil && cr.Spec.System.DatabaseSpec.PostgreSQL != nil { systemDatabaseReconcilerConstructor = operator.CompositeDependencyReconcilerConstructor( operator.NewSystemPostgreSQLReconciler, - operator.NewSystemPostgreSQLImageReconciler, ) } else { systemDatabaseReconcilerConstructor = operator.CompositeDependencyReconcilerConstructor( operator.NewSystemMySQLReconciler, - operator.NewSystemMySQLImageReconciler, ) } diff --git a/controllers/apps/apimanager_controller_test.go b/controllers/apps/apimanager_controller_test.go index 977f57834..c66467ddc 100644 --- a/controllers/apps/apimanager_controller_test.go +++ b/controllers/apps/apimanager_controller_test.go @@ -12,8 +12,8 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -143,10 +143,10 @@ var _ = Describe("APIManager controller", func() { return err == nil }, 5*time.Minute, 5*time.Second).Should(BeTrue()) - fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed DeploymentConfigs\n") - err = waitForAllAPIManagerStandardDeploymentConfigs(testNamespace, 5*time.Second, 15*time.Minute, GinkgoWriter) + fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed Deployments\n") + err = waitForAllAPIManagerStandardDeployments(testNamespace, 5*time.Second, 15*time.Minute, GinkgoWriter) Expect(err).ToNot(HaveOccurred()) - fmt.Fprintf(GinkgoWriter, "All APIManager managed DeploymentConfigs are ready\n") + fmt.Fprintf(GinkgoWriter, "All APIManager managed Deployments are ready\n") fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed Routes\n") err = waitForAllAPIManagerStandardRoutes(testNamespace, 5*time.Second, 15*time.Minute, wildcardDomain, GinkgoWriter) @@ -164,8 +164,8 @@ var _ = Describe("APIManager controller", func() { }) }) -func waitForAllAPIManagerStandardDeploymentConfigs(namespace string, retryInterval, timeout time.Duration, w io.Writer) error { - deploymentConfigNames := []string{ // TODO gather this from constants/somewhere centralized +func waitForAllAPIManagerStandardDeployments(namespace string, retryInterval, timeout time.Duration, w io.Writer) error { + deploymentNames := []string{ // TODO gather this from constants/somewhere centralized "apicast-production", "apicast-staging", "backend-cron", @@ -183,23 +183,23 @@ func waitForAllAPIManagerStandardDeploymentConfigs(namespace string, retryInterv "zync-database", } - for _, dcName := range deploymentConfigNames { - lookupKey := types.NamespacedName{Name: dcName, Namespace: namespace} - createdDeployment := &appsv1.DeploymentConfig{} + for _, dName := range deploymentNames { + lookupKey := types.NamespacedName{Name: dName, Namespace: namespace} + createdDeployment := &k8sappsv1.Deployment{} Eventually(func() bool { err := testK8sClient.Get(context.Background(), lookupKey, createdDeployment) if err != nil { return false } - if helper.IsDeploymentConfigAvailable(createdDeployment) { - fmt.Fprintf(w, "DeploymentConfig '%s' available\n", dcName) + if helper.IsDeploymentAvailable(createdDeployment) { + fmt.Fprintf(w, "Deployment '%s' available\n", dName) return true } availableReplicas := createdDeployment.Status.AvailableReplicas desiredReplicas := createdDeployment.Spec.Replicas - fmt.Fprintf(w, "Waiting for full availability of %s DeploymentConfig (%d/%d)\n", dcName, availableReplicas, desiredReplicas) + fmt.Fprintf(w, "Waiting for full availability of %s Deployment (%d/%d)\n", dName, availableReplicas, desiredReplicas) return false }, timeout, retryInterval).Should(BeTrue()) diff --git a/controllers/apps/apimanager_status_reconciler.go b/controllers/apps/apimanager_status_reconciler.go index 26f18742d..2e62af2c7 100644 --- a/controllers/apps/apimanager_status_reconciler.go +++ b/controllers/apps/apimanager_status_reconciler.go @@ -12,8 +12,8 @@ import ( "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/RHsyseng/operator-utils/pkg/olm" "github.com/go-logr/logr" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -67,7 +67,7 @@ func (s *APIManagerStatusReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{Requeue: true}, nil } - return reconcile.Result{}, fmt.Errorf("Failed to update status: %w", updateErr) + return reconcile.Result{}, fmt.Errorf("failed to update status: %w", updateErr) } return reconcile.Result{}, nil } @@ -88,7 +88,7 @@ func (s *APIManagerStatusReconciler) calculateStatus() (*appsv1alpha1.APIManager } newStatus.Conditions.SetCondition(availableCondition) - deploymentStatus := olm.GetDeploymentConfigStatus(deployments) + deploymentStatus := olm.GetDeploymentStatus(deployments) newStatus.Deployments = deploymentStatus return newStatus, nil @@ -124,17 +124,17 @@ func (s *APIManagerStatusReconciler) expectedDeploymentNames(instance *appsv1alp return deploymentLister.DeploymentNames() } -func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments []appsv1.DeploymentConfig) bool { +func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments []k8sappsv1.Deployment) bool { expectedDeploymentNames := s.expectedDeploymentNames(s.apimanagerResource) for _, deploymentName := range expectedDeploymentNames { - foundExistingDCIdx := -1 - for idx, existingDC := range existingDeployments { - if existingDC.Name == deploymentName { - foundExistingDCIdx = idx + foundExistingDeploymentIdx := -1 + for idx, existingDeployment := range existingDeployments { + if existingDeployment.Name == deploymentName { + foundExistingDeploymentIdx = idx break } } - if foundExistingDCIdx == -1 || !helper.IsDeploymentConfigAvailable(&existingDeployments[foundExistingDCIdx]) { + if foundExistingDeploymentIdx == -1 || !helper.IsDeploymentAvailable(&existingDeployments[foundExistingDeploymentIdx]) { return false } } @@ -142,13 +142,13 @@ func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments [] return true } -func (s *APIManagerStatusReconciler) existingDeployments() ([]appsv1.DeploymentConfig, error) { +func (s *APIManagerStatusReconciler) existingDeployments() ([]k8sappsv1.Deployment, error) { expectedDeploymentNames := s.expectedDeploymentNames(s.apimanagerResource) - var dcs []appsv1.DeploymentConfig - for _, dcName := range expectedDeploymentNames { - existingDeploymentConfig := &appsv1.DeploymentConfig{} - err := s.Client().Get(context.Background(), types.NamespacedName{Namespace: s.apimanagerResource.Namespace, Name: dcName}, existingDeploymentConfig) + var deployments []k8sappsv1.Deployment + for _, dName := range expectedDeploymentNames { + existingDeployment := &k8sappsv1.Deployment{} + err := s.Client().Get(context.Background(), types.NamespacedName{Namespace: s.apimanagerResource.Namespace, Name: dName}, existingDeployment) if err != nil && !errors.IsNotFound(err) { return nil, err } @@ -156,19 +156,19 @@ func (s *APIManagerStatusReconciler) existingDeployments() ([]appsv1.DeploymentC continue } - for _, ownerRef := range existingDeploymentConfig.GetOwnerReferences() { + for _, ownerRef := range existingDeployment.GetOwnerReferences() { if ownerRef.UID == s.apimanagerResource.UID { - dcs = append(dcs, *existingDeploymentConfig) + deployments = append(deployments, *existingDeployment) break } } } - sort.Slice(dcs, func(i, j int) bool { return dcs[i].Name < dcs[j].Name }) + sort.Slice(deployments, func(i, j int) bool { return deployments[i].Name < deployments[j].Name }) - return dcs, nil + return deployments, nil } -func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []appsv1.DeploymentConfig) (common.Condition, error) { +func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []k8sappsv1.Deployment) (common.Condition, error) { deploymentsAvailable := s.deploymentsAvailable(existingDeployments) defaultRoutesReady, err := s.defaultRoutesReady() @@ -207,7 +207,7 @@ func (s *APIManagerStatusReconciler) defaultRoutesReady() (bool, error) { routeList := &routev1.RouteList{} err := s.Client().List(context.TODO(), routeList, listOps...) if err != nil { - return false, fmt.Errorf("Failed to list routes: %w", err) + return false, fmt.Errorf("failed to list routes: %w", err) } routes := append([]routev1.Route(nil), routeList.Items...) diff --git a/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go b/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go index 115404dfc..2bfe99a2a 100644 --- a/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go +++ b/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go @@ -39,11 +39,10 @@ func getApiManger() (apimanager *appsv1alpha1.APIManager) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: &oneValue}, diff --git a/doc/apimanager-reference.md b/doc/apimanager-reference.md index 913de8d62..ebe021381 100644 --- a/doc/apimanager-reference.md +++ b/doc/apimanager-reference.md @@ -90,8 +90,7 @@ One APIManager custom resource per project is allowed. | WildcardDomain | `wildcardDomain` | string | Yes | N/A | Root domain for the wildcard routes. Eg. example.com will generate 3scale-admin.example.com. | | AppLabel | `appLabel` | string | No | `3scale-api-management` | The value of the `app` label that will be applied to the API management solution | TenantName | `tenantName` | string | No | `3scale` | Tenant name under the root that Admin UI will be available with -admin suffix. -| ImageStreamTagImportInsecure | `imageStreamTagImportInsecure` | bool | No | `false` | Set to true if the server may bypass certificate verification or connect directly over HTTP during image import | -| ImagePullSecrets | `imagePullSecrets` | \[\][corev1.LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#localobjectreference-v1-core) | No | `[ { name: "threescale-registry-auth" } ]` | List of image pull secrets to be used on the managed DeploymentConfigs ServiceAccounts. See [imagePullSecrets field in K8s ServiceAccount documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#serviceaccount-v1-core) for details on Image pull secrets. If not specified, `threescale-registry-auth` is used. Secret names that contain `dockercfg-` or `token-` anywhere in part of its name cannot be specified. If an update to this attribute is performed the corresponding DeploymentConfig pods have to be redeployed by the user to make the changes effective | +| ImagePullSecrets | `imagePullSecrets` | \[\][corev1.LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#localobjectreference-v1-core) | No | `[ { name: "threescale-registry-auth" } ]` | List of image pull secrets to be used on the managed Deployments ServiceAccounts. See [imagePullSecrets field in K8s ServiceAccount documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#serviceaccount-v1-core) for details on Image pull secrets. If not specified, `threescale-registry-auth` is used. Secret names that contain `dockercfg-` or `token-` anywhere in part of its name cannot be specified. If an update to this attribute is performed the corresponding Deployment pods have to be redeployed by the user to make the changes effective | | ResourceRequirementsEnabled | `resourceRequirementsEnabled` | bool | No | `true` | When true, 3Scale API management solution is deployed with the optimal resource requirements and limits. Setting this to false removes those resource requirements. ***Warning*** Only set it to false for development and evaluation environments. When set to `true`, default compute resources are set for the APIManager components. See [Default APIManager components compute resources](#Default-APIManager-components-compute-resources) to see the default assigned values | | ApicastSpec | `apicast` | \*ApicastSpec | No | See [ApicastSpec](#ApicastSpec) | Spec of the Apicast part | | BackendSpec | `backend` | \*BackendSpec | No | See [BackendSpec](#BackendSpec) reference | Spec of the Backend part | @@ -660,7 +659,7 @@ Each element of the Condition array has the following fields: * The *status* field is a string, with possible values **True**, **False**, and **Unknown**. * The *type* field is a string indicating the type of the condition. The types are: * `Available`: An APIManager is in `Available` state when *all* of the following scenarios are true: - * All expected DeploymentConfigs to be deployed exist and have the `Available` condition set to true + * All expected Deployments to be deployed exist and have the `Available` condition set to true * All 3scale default OpenShift routes exist and have the Admitted condition set to true. The default routes are: * Master route * Backend Listener route diff --git a/doc/operator-application-capabilities.md b/doc/operator-application-capabilities.md index aea388cd7..ad01564bd 100644 --- a/doc/operator-application-capabilities.md +++ b/doc/operator-application-capabilities.md @@ -28,8 +28,8 @@ The following diagram shows available custom resource definitions and their rela * [Backend custom resource status field](#backend-custom-resource-status-field) * [Link your 3scale backend to your 3scale tenant or provider account](#link-your-3scale-backend-to-your-3scale-tenant-or-provider-account) * [Product custom resource](#product-custom-resource) - * [Product Deployment Config: Apicast Hosted](#product-deployment-config-apicast-hosted) - * [Product Deployment Config:Apicast Self Managed](#product-deployment-configapicast-self-managed) + * [Product Deployment: Apicast Hosted](#product-deployment-apicast-hosted) + * [Product Deployment: Apicast Self Managed](#product-deployment-apicast-self-managed) * [Product authentication types](#product-authentication-types) * [User Key](#user-key) * [AppID and AppKey pair](#appid-and-appkey-pair) @@ -365,7 +365,7 @@ spec: name: "OperatedProduct 1" ``` -### Product Deployment Config: Apicast Hosted +### Product Deployment: Apicast Hosted Configure your product with *Apicast Hosted* deployment mode @@ -380,7 +380,7 @@ spec: apicastHosted: {} ``` -### Product Deployment Config:Apicast Self Managed +### Product Deployment: Apicast Self Managed Configure your product with *Apicast Self Managed* deployment mode diff --git a/doc/operator-user-guide.md b/doc/operator-user-guide.md index 56300ce47..9bc72ba3c 100644 --- a/doc/operator-user-guide.md +++ b/doc/operator-user-guide.md @@ -503,25 +503,25 @@ spec: Check [*APIManager DatabaseSpec*](apimanager-reference.md#DatabaseSpec) for reference. #### Enabling Pod Disruption Budgets -The 3scale API Management solution DeploymentConfigs deployed and managed by the +The 3scale API Management solution Deployments deployed and managed by the APIManager will be configured with Kubernetes Pod Disruption Budgets enabled. A Pod Disruption Budget limits the number of pods related to an application -(in this case, pods of a DeploymentConfig) that are down simultaneously +(in this case, pods of a Deployment) that are down simultaneously from **voluntary disruptions**. -When enabling the Pod Disruption Budgets for non-database DeploymentConfigs will +When enabling the Pod Disruption Budgets for non-database Deployments will be set with a setting of maximum of 1 unavailable pod at any given time. -Database-related DeploymentConfigs are excluded from this configuration. -Additionally, `system-sphinx` DeploymentConfig is also excluded. +Database-related Deployments are excluded from this configuration. +Additionally, `system-sphinx` Deployment is also excluded. For details about the behavior of Pod Disruption Budgets, what they perform and what constitutes a 'voluntary disruption' see the following [Kubernetes Documentation](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) Pods which are deleted or unavailable due to a rolling upgrade to an application -do count against the disruption budget, but the DeploymentConfigs are not +do count against the disruption budget, but the Deployments are not limited by Pod Disruption Budgets when doing rolling upgrades or they are scaled up/down. @@ -719,7 +719,7 @@ Check [Expanding persistent volumes](https://docs.openshift.com/container-platfo #### Setting custom PriorityClassName PriorityClassName specifies the Pod priority. See [here](https://docs.openshift.com/container-platform/4.13/nodes/pods/nodes-pods-priority.html) for more information. -It be can be customized through APIManager CR `priorityClassName` attribute for each DeploymentConfig. +It be can be customized through APIManager CR `priorityClassName` attribute for each Deployment. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -739,7 +739,7 @@ spec: #### Setting custom TopologySpreadConstraints TopologySpreadConstraints specifies how to spread matching pods among the given topology. See [here](https://docs.openshift.com/container-platform/4.13/nodes/scheduling/nodes-scheduler-pod-topology-spread-constraints.html) for more information. -It be can be customized through APIManager CR `topologySpreadConstraints` attribute for each DeploymentConfig. +It be can be customized through APIManager CR `topologySpreadConstraints` attribute for each Deployment. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -772,7 +772,7 @@ spec: ``` #### Setting custom labels -[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) can be customized through the APIManager CR `labels` attribute for each DeploymentConfig and are applied to their pods. +[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) can be customized through the APIManager CR `labels` attribute for each Deployment and are applied to their pods. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -912,7 +912,7 @@ spec: ``` #### Pod Disruption Budget -Whether Pod Disruption Budgets are enabled for non-database DeploymentConfigs +Whether Pod Disruption Budgets are enabled for non-database Deployments ```yaml apiVersion: apps.3scale.net/v1alpha1 diff --git a/pkg/3scale/amp/component/ampimages.go b/pkg/3scale/amp/component/ampimages.go index 24fe2d3a5..8f84bd49a 100644 --- a/pkg/3scale/amp/component/ampimages.go +++ b/pkg/3scale/amp/component/ampimages.go @@ -1,15 +1,10 @@ package component import ( - imagev1 "github.com/openshift/api/image/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - InsecureImportPolicy = false -) - type AmpImages struct { Options *AmpImagesOptions } @@ -18,237 +13,6 @@ func NewAmpImages(options *AmpImagesOptions) *AmpImages { return &AmpImages{Options: options} } -func (ampImages *AmpImages) BackendImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-backend", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "backend", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP backend", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "amp-backend " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.BackendImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) ZyncImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-zync", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "zync", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP Zync", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP Zync " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ZyncImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) APICastImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-apicast", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "apicast", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP APIcast", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP APIcast " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ApicastImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-system", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP System", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP system " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) ZyncDatabasePostgreSQLImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "zync-database-postgresql", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "Zync database PostgreSQL", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "Zync " + ampImages.Options.AmpRelease + " PostgreSQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ZyncDatabasePostgreSQLImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemMemcachedImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-memcached", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System Memcached", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + ampImages.Options.AmpRelease + " Memcached", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemMemcachedImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemSearchdImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-searchd", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System Searchd", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + ampImages.Options.AmpRelease + " Searchd", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemSearchdImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (ampImages *AmpImages) DeploymentsServiceAccount() *v1.ServiceAccount { return &v1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/3scale/amp/component/ampimages_options.go b/pkg/3scale/amp/component/ampimages_options.go index 8aba7a2a8..0d98921fe 100644 --- a/pkg/3scale/amp/component/ampimages_options.go +++ b/pkg/3scale/amp/component/ampimages_options.go @@ -7,16 +7,15 @@ import ( // AmpImagesOptions container object with all required to create components type AmpImagesOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - ApicastImage string `validate:"required"` - BackendImage string `validate:"required"` - SystemImage string `validate:"required"` - ZyncImage string `validate:"required"` - ZyncDatabasePostgreSQLImage string `validate:"required"` - SystemMemcachedImage string `validate:"required"` - SystemSearchdImage string `validate:"required"` - InsecureImportPolicy bool + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + ApicastImage string `validate:"required"` + BackendImage string `validate:"required"` + SystemImage string `validate:"required"` + ZyncImage string `validate:"required"` + ZyncDatabasePostgreSQLImage string `validate:"required"` + SystemMemcachedImage string `validate:"required"` + SystemSearchdImage string `validate:"required"` ImagePullSecrets []v1.LocalObjectReference `validate:"required"` } diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index d6b6b5d9d..196e96c3c 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -9,8 +9,9 @@ import ( "strings" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,8 +19,9 @@ import ( ) const ( - ApicastStagingName = "apicast-staging" - ApicastProductionName = "apicast-production" + ApicastStagingName = "apicast-staging" + ApicastProductionName = "apicast-production" + ApicastProductionInitContainerName = "system-master-svc" CustomPoliciesMountBasePath = "/opt/app-root/src/policies" CustomPoliciesAnnotationNameSegmentPrefix = "apicast-policy-volume" @@ -66,7 +68,7 @@ func (apicast *Apicast) StagingService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: apicast.stagingServicePorts(), - Selector: map[string]string{"deploymentConfig": ApicastStagingName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ApicastStagingName}, }, } } @@ -83,59 +85,40 @@ func (apicast *Apicast) ProductionService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: apicast.productionServicePorts(), - Selector: map[string]string{"deploymentConfig": ApicastProductionName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ApicastProductionName}, }, } } -func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps.openshift.io/v1", Kind: "DeploymentConfig"}, +func (apicast *Apicast) StagingDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ApicastStagingName, Labels: apicast.Options.CommonStagingLabels, - Annotations: apicast.stagingDeploymentConfigAnnotations(), + Annotations: apicast.stagingDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: apicast.Options.StagingReplicas, - Selector: map[string]string{ - "deploymentConfig": ApicastStagingName, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &apicast.Options.StagingReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ApicastStagingName, + }, }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - IntervalSeconds: &[]int64{1}[0], + Strategy: k8sappsv1.DeploymentStrategy{ + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - TimeoutSeconds: &[]int64{1800}[0], - UpdatePeriodSeconds: &[]int64{1}[0], - }, - Type: appsv1.DeploymentStrategyTypeRolling, - }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - ApicastStagingName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-apicast:%s", apicast.Options.ImageTag), - }, - }, }, + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.StagingPodTemplateLabels, Annotations: apicast.stagingPodAnnotations(), @@ -146,10 +129,10 @@ func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", Volumes: apicast.stagingVolumes(), Containers: []v1.Container{ - v1.Container{ + { Ports: apicast.stagingContainerPorts(), Env: apicast.buildApicastStagingEnv(), - Image: "amp-apicast:latest", + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, Name: ApicastStagingName, Resources: apicast.Options.StagingResourceRequirements, @@ -182,55 +165,35 @@ func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { } } -func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps.openshift.io/v1", Kind: "DeploymentConfig"}, +func (apicast *Apicast) ProductionDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ApicastProductionName, Labels: apicast.Options.CommonProductionLabels, - Annotations: apicast.productionDeploymentConfigAnnotations(), + Annotations: apicast.productionDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: apicast.Options.ProductionReplicas, - Selector: map[string]string{ - "deploymentConfig": ApicastProductionName, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &apicast.Options.ProductionReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ApicastProductionName, + }, }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - IntervalSeconds: &[]int64{1}[0], + Strategy: k8sappsv1.DeploymentStrategy{ + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - TimeoutSeconds: &[]int64{1800}[0], - UpdatePeriodSeconds: &[]int64{1}[0], }, - Type: appsv1.DeploymentStrategyTypeRolling, + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-master-svc", - ApicastProductionName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-apicast:%s", apicast.Options.ImageTag), - }, - }, - }, - }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.ProductionPodTemplateLabels, Annotations: apicast.productionPodAnnotations(), @@ -241,12 +204,12 @@ func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", Volumes: apicast.productionVolumes(), InitContainers: []v1.Container{ - v1.Container{ - Name: "system-master-svc", - Image: "amp-apicast:latest", + { + Name: ApicastProductionInitContainerName, + Image: containerImage, Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, Env: []v1.EnvVar{ - v1.EnvVar{ + { Name: "SLEEP_SECONDS", Value: "1", }, @@ -254,10 +217,10 @@ func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Ports: apicast.productionContainerPorts(), Env: apicast.buildApicastProductionEnv(), - Image: "amp-apicast:latest", + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, Name: ApicastProductionName, Resources: apicast.Options.ProductionResourceRequirements, @@ -495,7 +458,7 @@ func (apicast *Apicast) StagingPodDisruptionBudget() *policyv1.PodDisruptionBudg }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ApicastStagingName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ApicastStagingName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -514,7 +477,7 @@ func (apicast *Apicast) ProductionPodDisruptionBudget() *policyv1.PodDisruptionB }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ApicastProductionName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ApicastProductionName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -763,7 +726,7 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { return volumes } -func (apicast *Apicast) productionDeploymentConfigAnnotations() map[string]string { +func (apicast *Apicast) productionDeploymentAnnotations() map[string]string { annotations := map[string]string{} for _, customPolicy := range apicast.Options.ProductionCustomPolicies { @@ -779,15 +742,10 @@ func (apicast *Apicast) productionDeploymentConfigAnnotations() map[string]strin annotations[customEnvAnnotationKey(customEnvSecret)] = customEnvAnnotationValue(customEnvSecret) } - // keep backward compat - if len(annotations) == 0 { - return nil - } - return annotations } -func (apicast *Apicast) stagingDeploymentConfigAnnotations() map[string]string { +func (apicast *Apicast) stagingDeploymentAnnotations() map[string]string { annotations := map[string]string{} for _, customPolicy := range apicast.Options.StagingCustomPolicies { @@ -803,11 +761,6 @@ func (apicast *Apicast) stagingDeploymentConfigAnnotations() map[string]string { annotations[customEnvAnnotationKey(customEnvSecret)] = customEnvAnnotationValue(customEnvSecret) } - // keep backward compat - if len(annotations) == 0 { - return nil - } - return annotations } diff --git a/pkg/3scale/amp/component/backend.go b/pkg/3scale/amp/component/backend.go index 07fcc6114..d83eb9737 100644 --- a/pkg/3scale/amp/component/backend.go +++ b/pkg/3scale/amp/component/backend.go @@ -1,13 +1,13 @@ package component import ( - "fmt" "strconv" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,9 +15,10 @@ import ( ) const ( - BackendListenerName = "backend-listener" - BackendWorkerName = "backend-worker" - BackendCronName = "backend-cron" + BackendListenerName = "backend-listener" + BackendWorkerName = "backend-worker" + BackendCronName = "backend-cron" + BackendInitContainerName = "backend-redis-svc" ) const ( @@ -60,47 +61,35 @@ func NewBackend(options *BackendOptions) *Backend { return &Backend{Options: options} } -func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) WorkerDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: BackendWorkerName, Labels: backend.Options.CommonWorkerLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"backend-redis-svc", BackendWorkerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.WorkerReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendWorkerName, + }, }, - Replicas: backend.Options.WorkerReplicas, - Selector: map[string]string{"deploymentConfig": BackendWorkerName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.WorkerPodTemplateLabels, Annotations: backend.Options.WorkerPodTemplateAnnotations, @@ -109,21 +98,22 @@ func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.WorkerAffinity, Tolerations: backend.Options.WorkerTolerations, InitContainers: []v1.Container{ - v1.Container{ + { Name: "backend-redis-svc", - Image: "amp-backend:latest", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", - }, Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + }, + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ - v1.Container{ + { Name: BackendWorkerName, - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"bin/3scale_backend_worker", "run"}, Env: backend.buildBackendWorkerEnv(), Resources: backend.Options.WorkerResourceRequirements, @@ -147,52 +137,42 @@ func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { }, ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameWorker, - TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsWorker}}, + TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsWorker, + }, + }, }, } } -func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) CronDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: BackendCronName, Labels: backend.Options.CommonCronLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"backend-redis-svc", "backend-cron"}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.CronReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendCronName, + }, }, - Replicas: backend.Options.CronReplicas, - Selector: map[string]string{"deploymentConfig": BackendCronName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.CronPodTemplateLabels, Annotations: backend.Options.CronPodTemplateAnnotations, @@ -201,21 +181,22 @@ func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.CronAffinity, Tolerations: backend.Options.CronTolerations, InitContainers: []v1.Container{ - v1.Container{ + { Name: "backend-redis-svc", - Image: "amp-backend:latest", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", - }, Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + }, + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "backend-cron", - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"touch /tmp/healthy && backend-cron"}, Env: backend.buildBackendCronEnv(), Resources: backend.Options.CronResourceRequirements, @@ -234,52 +215,41 @@ func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameCron, TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsCron, - }}, + }, + }, }, } } -func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) ListenerDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: BackendListenerName, Labels: backend.Options.CommonListenerLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{BackendListenerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.ListenerReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendListenerName, + }, }, - Replicas: backend.Options.ListenerReplicas, - Selector: map[string]string{"deploymentConfig": BackendListenerName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.ListenerPodTemplateLabels, Annotations: backend.Options.ListenerPodTemplateAnnotations, @@ -288,9 +258,9 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.ListenerAffinity, Tolerations: backend.Options.ListenerTolerations, Containers: []v1.Container{ - v1.Container{ + { Name: BackendListenerName, - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"bin/3scale_backend", "start", "-e", "production", "-p", "3000", "-x", "/dev/stdout"}, Ports: backend.listenerPorts(), Env: backend.buildBackendListenerEnv(), @@ -298,7 +268,7 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000}}, }, InitialDelaySeconds: 30, @@ -311,7 +281,7 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000}}, }, InitialDelaySeconds: 30, @@ -326,7 +296,8 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameListener, TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsListener, - }}, + }, + }, }, } } @@ -343,17 +314,17 @@ func (backend *Backend) ListenerService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "http", Protocol: v1.ProtocolTCP, Port: 3000, TargetPort: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000, }, }, }, - Selector: map[string]string{"deploymentConfig": BackendListenerName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: BackendListenerName}, }, } } @@ -502,7 +473,7 @@ func (backend *Backend) WorkerPodDisruptionBudget() *policyv1.PodDisruptionBudge }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": BackendWorkerName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: BackendWorkerName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -521,7 +492,7 @@ func (backend *Backend) CronPodDisruptionBudget() *policyv1.PodDisruptionBudget }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "backend-cron"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "backend-cron"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -540,7 +511,7 @@ func (backend *Backend) ListenerPodDisruptionBudget() *policyv1.PodDisruptionBud }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": BackendListenerName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: BackendListenerName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -549,7 +520,7 @@ func (backend *Backend) ListenerPodDisruptionBudget() *policyv1.PodDisruptionBud func (backend *Backend) listenerPorts() []v1.ContainerPort { ports := []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, ContainerPort: 3000, Protocol: v1.ProtocolTCP}, + {HostPort: 0, ContainerPort: 3000, Protocol: v1.ProtocolTCP}, } if backend.Options.ListenerMetrics { diff --git a/pkg/3scale/amp/component/evaluation.go b/pkg/3scale/amp/component/evaluation.go index d6a88495d..e22422a9f 100644 --- a/pkg/3scale/amp/component/evaluation.go +++ b/pkg/3scale/amp/component/evaluation.go @@ -2,7 +2,7 @@ package component import ( "github.com/3scale/3scale-operator/pkg/common" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) @@ -15,10 +15,10 @@ func NewEvaluation() *Evaluation { func (evaluation *Evaluation) RemoveContainersResourceRequestsAndLimits(objects []common.KubernetesObject) { for _, obj := range objects { - dc, ok := obj.(*appsv1.DeploymentConfig) + deployment, ok := obj.(*k8sappsv1.Deployment) if ok { - for containerIdx := range dc.Spec.Template.Spec.Containers { - container := &dc.Spec.Template.Spec.Containers[containerIdx] + for containerIdx := range deployment.Spec.Template.Spec.Containers { + container := &deployment.Spec.Template.Spec.Containers[containerIdx] container.Resources = v1.ResourceRequirements{} } } diff --git a/pkg/3scale/amp/component/memcached.go b/pkg/3scale/amp/component/memcached.go index 7070da9fa..870a3b487 100644 --- a/pkg/3scale/amp/component/memcached.go +++ b/pkg/3scale/amp/component/memcached.go @@ -1,9 +1,9 @@ package component import ( - "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,52 +21,27 @@ func NewMemcached(options *MemcachedOptions) *Memcached { return &Memcached{Options: options} } -func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { + var memcachedReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemMemcachedDeploymentName, Labels: m.Options.DeploymentLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%", - }, - MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "memcache", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-memcached:%s", m.Options.ImageTag), - }, - }, + Replicas: &memcachedReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemMemcachedDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemMemcachedDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: m.Options.PodTemplateLabels, Annotations: m.Options.PodTemplateAnnotations, @@ -76,21 +51,24 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: m.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Containers: []v1.Container{ - v1.Container{ + { Name: "memcache", - Image: "system-memcached:latest", + Image: containerImage, Command: []string{"memcached", "-m", "64"}, Ports: []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, + {HostPort: 0, ContainerPort: 11211, Protocol: v1.ProtocolTCP}, }, Resources: m.Options.ResourceRequirements, LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 11211}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 11211, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 0, @@ -99,10 +77,13 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { FailureThreshold: 0, }, ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 11211}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 11211, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 5, @@ -115,7 +96,8 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { }, PriorityClassName: m.Options.PriorityClassName, TopologySpreadConstraints: m.Options.TopologySpreadConstraints, - }}, + }, + }, }, } } diff --git a/pkg/3scale/amp/component/redis.go b/pkg/3scale/amp/component/redis.go index 88ed07f52..a170057e8 100644 --- a/pkg/3scale/amp/component/redis.go +++ b/pkg/3scale/amp/component/redis.go @@ -1,12 +1,12 @@ package component import ( - "fmt" "path" "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" - imagev1 "github.com/openshift/api/image/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,22 +26,22 @@ func NewRedis(options *RedisOptions) *Redis { return &Redis{Options: options} } -func (redis *Redis) BackendDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: redis.buildDeploymentConfigTypeMeta(), - ObjectMeta: redis.buildDeploymentConfigObjectMeta(), - Spec: redis.buildDeploymentConfigSpec(), +func (redis *Redis) BackendDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: redis.buildDeploymentTypeMeta(), + ObjectMeta: redis.buildDeploymentObjectMeta(), + Spec: redis.buildDeploymentSpec(), } } -func (redis *Redis) buildDeploymentConfigTypeMeta() metav1.TypeMeta { +func (redis *Redis) buildDeploymentTypeMeta() metav1.TypeMeta { return metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", + Kind: reconcilers.DeploymentKind, + APIVersion: reconcilers.DeploymentAPIVersion, } } -func (redis *Redis) buildDeploymentConfigObjectMeta() metav1.ObjectMeta { +func (redis *Redis) buildDeploymentObjectMeta() metav1.ObjectMeta { return metav1.ObjectMeta{ Name: backendRedisObjectMetaName, Labels: redis.Options.BackendRedisLabels, @@ -51,63 +51,45 @@ func (redis *Redis) buildDeploymentConfigObjectMeta() metav1.ObjectMeta { const ( redisConfigVolumeName = "redis-config" - backendRedisObjectMetaName = "backend-redis" - backendRedisDCSelectorName = backendRedisObjectMetaName - backendRedisStorageVolumeName = "backend-redis-storage" - backendRedisConfigMapKey = "redis.conf" - backendRedisContainerName = "backend-redis" - backendRedisConfigPath = "/etc/redis.d/" + backendRedisObjectMetaName = "backend-redis" + backendRedisDeploymentSelectorName = backendRedisObjectMetaName + backendRedisStorageVolumeName = "backend-redis-storage" + backendRedisConfigMapKey = "redis.conf" + backendRedisContainerName = "backend-redis" + backendRedisConfigPath = "/etc/redis.d/" ) -func (redis *Redis) buildDeploymentConfigSpec() appsv1.DeploymentConfigSpec { - return appsv1.DeploymentConfigSpec{ +func (redis *Redis) buildDeploymentSpec() k8sappsv1.DeploymentSpec { + var redisReplicas int32 = 1 + + return k8sappsv1.DeploymentSpec{ Template: redis.buildPodTemplateSpec(), Strategy: redis.buildDeploymentStrategy(), - Selector: redis.buildDeploymentConfigSelector(), - Replicas: 1, - Triggers: redis.buildDeploymentConfigTriggers(), + Selector: redis.buildDeploymentSelector(), + Replicas: &redisReplicas, } } -func (redis *Redis) buildDeploymentStrategy() appsv1.DeploymentStrategy { - return appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, +func (redis *Redis) buildDeploymentStrategy() k8sappsv1.DeploymentStrategy { + return k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, } } func (redis *Redis) getSelectorLabels() map[string]string { return map[string]string{ - "deploymentConfig": backendRedisDCSelectorName, + reconcilers.DeploymentLabelSelector: backendRedisDeploymentSelectorName, } } -func (redis *Redis) buildDeploymentConfigSelector() map[string]string { - return redis.getSelectorLabels() -} - -func (redis *Redis) buildDeploymentConfigTriggers() appsv1.DeploymentTriggerPolicies { - return appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "backend-redis", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("backend-redis:%s", redis.Options.BackendImageTag), - }, - }, - }, +func (redis *Redis) buildDeploymentSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: redis.getSelectorLabels(), } } -func (redis *Redis) buildPodTemplateSpec() *v1.PodTemplateSpec { - return &v1.PodTemplateSpec{ +func (redis *Redis) buildPodTemplateSpec() v1.PodTemplateSpec { + return v1.PodTemplateSpec{ Spec: v1.PodSpec{ Affinity: redis.Options.BackendRedisAffinity, Tolerations: redis.Options.BackendRedisTolerations, @@ -126,7 +108,7 @@ func (redis *Redis) buildPodTemplateSpec() *v1.PodTemplateSpec { func (redis *Redis) buildPodVolumes() []v1.Volume { return []v1.Volume{ - v1.Volume{ + { Name: backendRedisStorageVolumeName, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -134,7 +116,7 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { }, }, }, - v1.Volume{ + { Name: redisConfigVolumeName, VolumeSource: v1.VolumeSource{ ConfigMap: &v1.ConfigMapVolumeSource{ @@ -142,7 +124,7 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { Name: redisConfigVolumeName, }, Items: []v1.KeyToPath{ - v1.KeyToPath{ + { Key: backendRedisConfigMapKey, Path: backendRedisConfigMapKey, }, @@ -155,8 +137,8 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { func (redis *Redis) buildPodContainers() []v1.Container { return []v1.Container{ - v1.Container{ - Image: "backend-redis:latest", + { + Image: redis.Options.BackendImage, ImagePullPolicy: v1.PullIfNotPresent, Name: backendRedisContainerName, Env: redis.buildEnv(), @@ -204,13 +186,13 @@ func (redis *Redis) buildPodContainerLivenessProbe() *v1.Probe { func (redis *Redis) buildPodContainerVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ - v1.VolumeMount{ + { Name: backendRedisStorageVolumeName, // https://github.com/sclorg/redis-container/ images have // redis data directory hardcoded on /var/lib/redis/data MountPath: "/var/lib/redis/data", }, - v1.VolumeMount{ + { Name: redisConfigVolumeName, MountPath: backendRedisConfigPath, }, @@ -248,7 +230,7 @@ func (redis *Redis) buildServiceSpec() v1.ServiceSpec { func (redis *Redis) buildServicePorts() []v1.ServicePort { return []v1.ServicePort{ - v1.ServicePort{ + { Port: 6379, TargetPort: intstr.FromInt(6379), Protocol: v1.ProtocolTCP, @@ -258,7 +240,7 @@ func (redis *Redis) buildServicePorts() []v1.ServicePort { func (redis *Redis) buildServiceSelector() map[string]string { return map[string]string{ - "deploymentConfig": backendRedisDCSelectorName, + reconcilers.DeploymentLabelSelector: backendRedisDeploymentSelectorName, } } @@ -376,36 +358,6 @@ func (redis *Redis) buildPVCSpec() v1.PersistentVolumeClaimSpec { } } -func (redis *Redis) BackendImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "backend-redis", - Labels: redis.Options.BackendCommonLabels, - Annotations: map[string]string{ - "openshift.io/display-name": "Backend Redis", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: redis.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "Backend " + redis.Options.AmpRelease + " Redis", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: redis.Options.BackendImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *redis.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (redis *Redis) BackendRedisSecret() *v1.Secret { return &v1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -430,41 +382,27 @@ func (redis *Redis) BackendRedisSecret() *v1.Secret { ////// Begin System Redis -func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (redis *Redis) SystemDeployment() *k8sappsv1.Deployment { + var redisReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemRedisDeploymentName, Labels: redis.Options.SystemRedisLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-redis", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-redis:%s", redis.Options.SystemImageTag), - }, - }, + Replicas: &redisReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemRedisDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemRedisDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: redis.Options.SystemRedisPodTemplateLabels, Annotations: redis.Options.SystemRedisPodTemplateAnnotations, @@ -474,43 +412,57 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: redis.Options.SystemRedisTolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "system-redis-storage", - VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "system-redis-storage", - ReadOnly: false}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "system-redis-storage", + ReadOnly: false, + }, + }, + }, { Name: "redis-config", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "redis-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "redis-config", + }, + Items: []v1.KeyToPath{ + { + Key: "redis.conf", + Path: "redis.conf", + }, + }, }, - Items: []v1.KeyToPath{ - v1.KeyToPath{ - Key: "redis.conf", - Path: "redis.conf"}}}}}, + }, + }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-redis", - Image: "system-redis:latest", + Image: redis.Options.SystemImage, Env: redis.buildEnv(), Resources: *redis.Options.SystemRedisContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "system-redis-storage", ReadOnly: false, MountPath: "/var/lib/redis/data", - }, v1.VolumeMount{ + }, + { Name: "redis-config", ReadOnly: false, - MountPath: "/etc/redis.d/"}, + MountPath: "/etc/redis.d/", + }, }, LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 6379}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 6379, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 0, @@ -521,7 +473,8 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"container-entrypoint", "bash", "-c", "redis-cli set liveness-probe \"`date`\" | grep OK"}}, + Command: []string{"container-entrypoint", "bash", "-c", "redis-cli set liveness-probe \"`date`\" | grep OK"}, + }, }, InitialDelaySeconds: 30, TimeoutSeconds: 5, @@ -535,7 +488,8 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { }, PriorityClassName: redis.Options.SystemRedisPriorityClassName, TopologySpreadConstraints: redis.Options.SystemRedisTopologySpreadConstraints, - }}, + }, + }, }, } } @@ -552,14 +506,14 @@ func (redis *Redis) SystemService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "redis", Protocol: v1.ProtocolTCP, Port: 6379, TargetPort: intstr.FromInt(6379), }, }, - Selector: map[string]string{"deploymentConfig": "system-redis"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-redis"}, }, } } @@ -576,7 +530,7 @@ func (redis *Redis) SystemPVC() *v1.PersistentVolumeClaim { }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ - v1.PersistentVolumeAccessMode("ReadWriteOnce"), + "ReadWriteOnce", }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")}, @@ -586,36 +540,6 @@ func (redis *Redis) SystemPVC() *v1.PersistentVolumeClaim { } } -func (redis *Redis) SystemImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-redis", - Labels: redis.Options.SystemCommonLabels, - Annotations: map[string]string{ - "openshift.io/display-name": "System Redis", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: redis.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + redis.Options.AmpRelease + " Redis", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: redis.Options.SystemImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *redis.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (redis *Redis) SystemRedisSecret() *v1.Secret { return &v1.Secret{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/3scale/amp/component/redis_options.go b/pkg/3scale/amp/component/redis_options.go index 0c510c2d3..962a56fb0 100644 --- a/pkg/3scale/amp/component/redis_options.go +++ b/pkg/3scale/amp/component/redis_options.go @@ -14,7 +14,6 @@ type RedisOptions struct { SystemImageTag string `validate:"required"` BackendRedisContainerResourceRequirements *v1.ResourceRequirements `validate:"required"` SystemRedisContainerResourceRequirements *v1.ResourceRequirements `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` BackendRedisPVCStorageClass *string SystemRedisPVCStorageClass *string diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index b8327c8bd..899497c91 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -1,11 +1,11 @@ package component import ( - "fmt" "sort" "strconv" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,6 +13,7 @@ import ( "github.com/3scale/3scale-operator/apis/apps" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( @@ -91,8 +92,11 @@ const ( ) const ( - SystemSidekiqName = "system-sidekiq" - SystemAppDeploymentName = "system-app" + SystemSidekiqName = "system-sidekiq" + SystemSideKiqInitContainerName = "check-svc" + SystemAppDeploymentName = "system-app" + SystemAppPreHookJobName = "system-app-pre" + SystemAppPostHookJobName = "system-app-post" SystemAppMasterContainerName = "system-master" SystemAppProviderContainerName = "system-provider" @@ -309,7 +313,9 @@ func (system *System) buildSystemAppPreHookEnv() []v1.EnvVar { } func (system *System) buildSystemAppPostHookEnv() []v1.EnvVar { - var result []v1.EnvVar + result := []v1.EnvVar{} + baseEnv := system.buildSystemBaseEnv() + result = append(result, baseEnv...) if system.Options.IncludeOracleOptionalSettings { result = append(result, helper.EnvVarFromSecretOptional("ORACLE_SYSTEM_PASSWORD", SystemSecretSystemDatabaseSecretName, "ORACLE_SYSTEM_PASSWORD")) } @@ -527,75 +533,50 @@ func (system *System) appPodVolumes() []v1.Volume { return res } -func (system *System) volumeNamesForSystemAppPreHookPod() []string { - res := []string{} +func (system *System) volumesForSystemAppLifecycleHookPods() []v1.VolumeMount { + res := []v1.VolumeMount{} if system.Options.PvcFileStorageOptions != nil { - res = append(res, SystemFileStoragePVCName) + res = append(res, v1.VolumeMount{ + Name: SystemFileStoragePVCName, + }) } if system.Options.S3FileStorageOptions != nil && system.Options.S3FileStorageOptions.STSEnabled { - res = append(res, S3StsCredentialsSecretName) + res = append(res, v1.VolumeMount{ + Name: S3StsCredentialsSecretName, + }) } return res } -func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (system *System) AppDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemAppDeploymentName, Labels: system.Options.CommonAppLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - Pre: &appsv1.LifecycleHook{ - FailurePolicy: appsv1.LifecycleHookFailurePolicyRetry, - ExecNewPod: &appsv1.ExecNewPodHook{ - // TODO the MASTER_ACCESS_TOKEN reference should be probably set as an envvar that gathers its value from the system-seed secret - // but changing that probably has some implications during an upgrade process of the product - Command: []string{"bash", "-c", "bundle exec rake boot openshift:deploy"}, - Env: system.buildSystemAppPreHookEnv(), - ContainerName: SystemAppMasterContainerName, - Volumes: system.volumeNamesForSystemAppPreHookPod()}, - }, - Post: &appsv1.LifecycleHook{ - FailurePolicy: appsv1.LifecycleHookFailurePolicyAbort, - ExecNewPod: &appsv1.ExecNewPodHook{ - Command: []string{"bash", "-c", "bundle exec rake boot openshift:post_deploy"}, - Env: system.buildSystemAppPostHookEnv(), - ContainerName: SystemAppMasterContainerName}}}, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{SystemAppProviderContainerName, SystemAppDeveloperContainerName, SystemAppMasterContainerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-system:%s", system.Options.ImageTag)}}}, + Replicas: &system.Options.AppReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemAppDeploymentName, + }, }, - Replicas: system.Options.AppReplicas, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: system.Options.AppPodTemplateLabels, Annotations: system.Options.AppPodTemplateAnnotations, @@ -605,9 +586,9 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: system.Options.AppTolerations, Volumes: system.appPodVolumes(), Containers: []v1.Container{ - v1.Container{ + { Name: SystemAppMasterContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appMasterPorts(), Env: system.buildAppMasterContainerEnv(), @@ -617,8 +598,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "master"}}, + Type: intstr.String, + StrVal: "master", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -631,14 +614,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "master", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -650,9 +636,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Stdin: false, StdinOnce: false, TTY: false, - }, v1.Container{ + }, + { Name: SystemAppProviderContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appProviderPorts(), Env: system.buildAppProviderContainerEnv(), @@ -662,8 +649,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "provider"}}, + Type: intstr.String, + StrVal: "provider", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -676,14 +665,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "provider", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -695,9 +687,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Stdin: false, StdinOnce: false, TTY: false, - }, v1.Container{ + }, + { Name: SystemAppDeveloperContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appDeveloperPorts(), Env: system.buildAppDeveloperContainerEnv(), @@ -707,8 +700,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "developer"}}, + Type: intstr.String, + StrVal: "developer", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -721,14 +716,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "developer", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -742,7 +740,88 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: system.Options.AppPriorityClassName, TopologySpreadConstraints: system.Options.AppTopologySpreadConstraints, - }}, + }, + }, + }, + } +} + +func (system *System) AppPreHookJob(containerImage string, currentSystemAppGeneration int64) *batchv1.Job { + var completions int32 = 1 + + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SystemAppPreHookJobName, + Labels: system.Options.CommonAppLabels, + Annotations: map[string]string{ + helper.SystemAppGenerationAnnotation: strconv.FormatInt(currentSystemAppGeneration, 10), + }, + }, + Spec: batchv1.JobSpec{ + Completions: &completions, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: system.appPodVolumes(), + Containers: []v1.Container{ + { + Name: SystemAppPreHookJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake boot openshift:deploy"}, + Env: system.buildSystemAppPreHookEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.volumesForSystemAppLifecycleHookPods(), + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: "amp", + PriorityClassName: system.Options.AppPriorityClassName, + }, + }, + }, + } +} + +func (system *System) AppPostHookJob(containerImage string, currentSystemAppGeneration int64) *batchv1.Job { + var completions int32 = 1 + + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SystemAppPostHookJobName, + Labels: system.Options.CommonAppLabels, + Annotations: map[string]string{ + helper.SystemAppGenerationAnnotation: strconv.FormatInt(currentSystemAppGeneration, 10), + }, + }, + Spec: batchv1.JobSpec{ + Completions: &completions, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: system.appPodVolumes(), + Containers: []v1.Container{ + { + Name: SystemAppPostHookJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake boot openshift:post_deploy"}, + Env: system.buildSystemAppPostHookEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.volumesForSystemAppLifecycleHookPods(), + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: "amp", + PriorityClassName: system.Options.AppPriorityClassName, + }, + }, }, } } @@ -819,47 +898,35 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { return res } -func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (system *System) SidekiqDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemSidekiqName, Labels: system.Options.CommonSidekiqLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"check-svc", SystemSidekiqName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-system:%s", system.Options.ImageTag)}}}, + Replicas: &system.Options.SidekiqReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemSidekiqName, + }, }, - Replicas: system.Options.SidekiqReplicas, - Selector: map[string]string{"deploymentConfig": SystemSidekiqName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: system.Options.SidekiqPodTemplateLabels, Annotations: system.Options.SideKiqPodTemplateAnnotations, @@ -869,9 +936,9 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: system.Options.SidekiqTolerations, Volumes: system.SidekiqPodVolumes(), InitContainers: []v1.Container{ - v1.Container{ - Name: "check-svc", - Image: "amp-system:latest", + { + Name: SystemSideKiqInitContainerName, + Image: containerImage, Command: []string{ "bash", "-c", @@ -881,9 +948,9 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: SystemSidekiqName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, Env: system.buildSystemSidekiqContainerEnv(), Resources: *system.Options.SidekiqContainerResourceRequirements, @@ -907,7 +974,8 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: system.Options.SideKiqPriorityClassName, TopologySpreadConstraints: system.Options.SideKiqTopologySpreadConstraints, - }}, + }, + }, }, } } @@ -1034,7 +1102,7 @@ func (system *System) ProviderService() *v1.Service { TargetPort: intstr.FromString("provider"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1058,7 +1126,7 @@ func (system *System) MasterService() *v1.Service { TargetPort: intstr.FromString("master"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1082,7 +1150,7 @@ func (system *System) DeveloperService() *v1.Service { TargetPort: intstr.FromString("developer"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1106,7 +1174,7 @@ func (system *System) MemcachedService() *v1.Service { TargetPort: intstr.FromInt(11211), }, }, - Selector: map[string]string{"deploymentConfig": "system-memcache"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-memcache"}, }, } } @@ -1204,7 +1272,7 @@ func (system *System) AppPodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -1223,7 +1291,7 @@ func (system *System) SidekiqPodDisruptionBudget() *policyv1.PodDisruptionBudget }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "system-sidekiq"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "system-sidekiq"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, diff --git a/pkg/3scale/amp/component/system_mysql.go b/pkg/3scale/amp/component/system_mysql.go index 0a22b73ef..f808410f3 100644 --- a/pkg/3scale/amp/component/system_mysql.go +++ b/pkg/3scale/amp/component/system_mysql.go @@ -1,10 +1,10 @@ package component import ( - "fmt" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -34,14 +34,14 @@ func (mysql *SystemMysql) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "system-mysql", Protocol: v1.ProtocolTCP, Port: 3306, TargetPort: intstr.FromInt(3306), }, }, - Selector: map[string]string{"deploymentConfig": "system-mysql"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-mysql"}, }, } } @@ -117,40 +117,26 @@ func (mysql *SystemMysql) PersistentVolumeClaim() *v1.PersistentVolumeClaim { } } -func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (mysql *SystemMysql) Deployment(containerImage string) *k8sappsv1.Deployment { + var mysqlReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemMySQLDeploymentName, Labels: mysql.Options.DeploymentLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-mysql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-mysql:%s", mysql.Options.ImageTag), - }, - }, + Replicas: &mysqlReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemMySQLDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemMySQLDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: mysql.Options.PodTemplateLabels, Annotations: mysql.Options.PodTemplateAnnotations, @@ -160,30 +146,46 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: mysql.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "mysql-storage", - VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "mysql-storage", - ReadOnly: false}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "mysql-storage", + ReadOnly: false, + }, + }, + }, + { Name: "mysql-extra-conf", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "mysql-extra-conf"}}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "mysql-extra-conf", + }, + }, + }, + }, + { Name: "mysql-main-conf", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "mysql-main-conf"}}}}, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "mysql-main-conf", + }, + }, + }, + }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-mysql", - Image: "system-mysql:latest", + Image: containerImage, Ports: []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, + { + HostPort: 0, ContainerPort: 3306, - Protocol: v1.ProtocolTCP}, + Protocol: v1.ProtocolTCP, + }, }, Env: []v1.EnvVar{ helper.EnvVarFromSecret("MYSQL_USER", SystemSecretSystemDatabaseSecretName, SystemSecretSystemDatabaseUserFieldName), @@ -198,25 +200,30 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { }, Resources: mysql.Options.ContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "mysql-storage", ReadOnly: false, MountPath: "/var/lib/mysql/data", - }, v1.VolumeMount{ + }, + { Name: "mysql-extra-conf", ReadOnly: false, MountPath: "/etc/my-extra.d", - }, v1.VolumeMount{ + }, + { Name: "mysql-main-conf", ReadOnly: false, - MountPath: "/etc/my-extra"}, + MountPath: "/etc/my-extra", + }, }, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 3306}}, + Type: intstr.Int, + IntVal: 3306, + }, + }, }, InitialDelaySeconds: 30, TimeoutSeconds: 0, @@ -227,7 +234,8 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-i", "-c", "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"}}, + Command: []string{"/bin/sh", "-i", "-c", "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"}, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 5, diff --git a/pkg/3scale/amp/component/system_mysql_image.go b/pkg/3scale/amp/component/system_mysql_image.go index a0875620e..2f85ef6f0 100644 --- a/pkg/3scale/amp/component/system_mysql_image.go +++ b/pkg/3scale/amp/component/system_mysql_image.go @@ -1,11 +1,5 @@ package component -import ( - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - type SystemMySQLImage struct { Options *SystemMySQLImageOptions } @@ -13,36 +7,3 @@ type SystemMySQLImage struct { func NewSystemMySQLImage(options *SystemMySQLImageOptions) *SystemMySQLImage { return &SystemMySQLImage{Options: options} } - -func (s *SystemMySQLImage) ImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-mysql", - Labels: map[string]string{ - "app": s.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System MySQL", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: s.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + s.Options.AmpRelease + " MySQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: s.Options.Image, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *s.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} diff --git a/pkg/3scale/amp/component/system_mysql_image_options.go b/pkg/3scale/amp/component/system_mysql_image_options.go index dbee8ed63..094d85262 100644 --- a/pkg/3scale/amp/component/system_mysql_image_options.go +++ b/pkg/3scale/amp/component/system_mysql_image_options.go @@ -3,10 +3,9 @@ package component import "github.com/go-playground/validator/v10" type SystemMySQLImageOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - Image string `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + Image string `validate:"required"` } func NewSystemMySQLImageOptions() *SystemMySQLImageOptions { diff --git a/pkg/3scale/amp/component/system_postgresql.go b/pkg/3scale/amp/component/system_postgresql.go index ca6462a90..33f44eed7 100644 --- a/pkg/3scale/amp/component/system_postgresql.go +++ b/pkg/3scale/amp/component/system_postgresql.go @@ -1,10 +1,10 @@ package component import ( - "fmt" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -34,14 +34,14 @@ func (p *SystemPostgreSQL) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "system-postgresql", Protocol: v1.ProtocolTCP, Port: 5432, TargetPort: intstr.FromInt(5432), }, }, - Selector: map[string]string{"deploymentConfig": "system-postgresql"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-postgresql"}, }, } } @@ -63,7 +63,7 @@ func (p *SystemPostgreSQL) DataPersistentVolumeClaim() *v1.PersistentVolumeClaim }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ - v1.PersistentVolumeAccessMode("ReadWriteOnce"), + "ReadWriteOnce", }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ @@ -76,40 +76,29 @@ func (p *SystemPostgreSQL) DataPersistentVolumeClaim() *v1.PersistentVolumeClaim } } -func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func (p *SystemPostgreSQL) Deployment(containerImage string) *k8sappsv1.Deployment { + var postgresReplicas int32 = 1 + + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: reconcilers.DeploymentLabelSelector, APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: SystemPostgreSQLDeploymentName, Labels: p.Options.DeploymentLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-postgresql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-postgresql:%s", p.Options.ImageTag), - }, - }, + Replicas: &postgresReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemPostgreSQLDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemPostgreSQLDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: p.Options.PodTemplateLabels, Annotations: p.Options.PodTemplateAnnotations, @@ -119,7 +108,7 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: p.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "postgresql-data", VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -129,11 +118,11 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-postgresql", - Image: "system-postgresql:latest", + Image: containerImage, Ports: []v1.ContainerPort{ - v1.ContainerPort{ + { ContainerPort: 5432, Protocol: v1.ProtocolTCP, }, @@ -148,7 +137,7 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { }, Resources: p.Options.ContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "postgresql-data", MountPath: "/var/lib/pgsql/data", }, @@ -171,7 +160,8 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1'"}}, + Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1'"}, + }, }, InitialDelaySeconds: 10, PeriodSeconds: 30, diff --git a/pkg/3scale/amp/component/system_postgresql_image.go b/pkg/3scale/amp/component/system_postgresql_image.go index 7e16ece4d..652657eb3 100644 --- a/pkg/3scale/amp/component/system_postgresql_image.go +++ b/pkg/3scale/amp/component/system_postgresql_image.go @@ -1,11 +1,5 @@ package component -import ( - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - type SystemPostgreSQLImage struct { Options *SystemPostgreSQLImageOptions } @@ -13,36 +7,3 @@ type SystemPostgreSQLImage struct { func NewSystemPostgreSQLImage(options *SystemPostgreSQLImageOptions) *SystemPostgreSQLImage { return &SystemPostgreSQLImage{Options: options} } - -func (s *SystemPostgreSQLImage) ImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-postgresql", - Labels: map[string]string{ - "app": s.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System database", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: s.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + s.Options.AmpRelease + " PostgreSQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: s.Options.Image, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *s.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} diff --git a/pkg/3scale/amp/component/system_postgresql_image_options.go b/pkg/3scale/amp/component/system_postgresql_image_options.go index 7384f97a4..53ca65196 100644 --- a/pkg/3scale/amp/component/system_postgresql_image_options.go +++ b/pkg/3scale/amp/component/system_postgresql_image_options.go @@ -3,10 +3,9 @@ package component import "github.com/go-playground/validator/v10" type SystemPostgreSQLImageOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - Image string `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + Image string `validate:"required"` } func NewSystemPostgreSQLImageOptions() *SystemPostgreSQLImageOptions { diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index bfc902996..f5b48fa93 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -1,9 +1,9 @@ package component import ( - "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -36,53 +36,38 @@ func (s *SystemSearchd) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "searchd", Protocol: v1.ProtocolTCP, Port: 9306, TargetPort: intstr.FromInt(9306), }, }, - Selector: map[string]string{"deploymentConfig": "system-searchd"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-searchd"}, }, } } -func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (s *SystemSearchd) Deployment(containerImage string) *k8sappsv1.Deployment { + var searchdReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemSearchdDeploymentName, Labels: s.Options.Labels, }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-searchd", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-searchd:%s", s.Options.ImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &searchdReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemSearchdDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemSearchdDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: s.Options.PodTemplateLabels, Annotations: s.Options.PodTemplateAnnotations, @@ -92,7 +77,7 @@ func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: s.Options.Tolerations, ServiceAccountName: "amp", Volumes: []v1.Volume{ - v1.Volume{ + { Name: SystemSearchdDBVolumeName, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -103,12 +88,12 @@ func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ - Name: "system-searchd", - Image: "system-searchd:latest", + { + Name: SystemSearchdDeploymentName, + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { ReadOnly: false, Name: SystemSearchdDBVolumeName, MountPath: "/var/lib/searchd", diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 5ac724b0c..30a5256be 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -1,13 +1,12 @@ package component import ( - "fmt" - - policyv1 "k8s.io/api/policy/v1" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -17,6 +16,7 @@ const ( ZyncName = "zync" ZyncQueDeploymentName = "zync-que" ZyncDatabaseDeploymentName = "zync-database" + ZyncInitContainerName = "zync-db-svc" ) const ( @@ -83,7 +83,7 @@ func (zync *Zync) QueRoleBinding() *rbacv1.RoleBinding { Name: "zync-que-rolebinding", }, Subjects: []rbacv1.Subject{ - rbacv1.Subject{ + { Kind: "ServiceAccount", Name: "zync-que-sa", }, @@ -106,17 +106,17 @@ func (zync *Zync) QueRole() *rbacv1.Role { Name: "zync-que-role", }, Rules: []rbacv1.PolicyRule{ - rbacv1.PolicyRule{ - APIGroups: []string{"apps.openshift.io"}, + { + APIGroups: []string{"apps"}, Resources: []string{ - "deploymentconfigs", + "deployments", }, Verbs: []string{ "get", "list", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{""}, Resources: []string{ "pods", @@ -127,7 +127,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "list", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes", @@ -141,7 +141,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "update", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes/status", @@ -150,7 +150,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "get", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes/custom-host", @@ -163,39 +163,21 @@ func (zync *Zync) QueRole() *rbacv1.Role { } } -func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) Deployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ZyncName, Labels: zync.Options.CommonZyncLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "zync-db-svc", - ZyncName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-zync:%s", zync.Options.ImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zync.Options.ZyncReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncName, }, }, - Replicas: zync.Options.ZyncReplicas, - Selector: map[string]string{"deploymentConfig": ZyncName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncPodTemplateLabels, Annotations: zync.Options.ZyncPodTemplateAnnotations, @@ -205,19 +187,20 @@ func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: zync.Options.ZyncTolerations, ServiceAccountName: "amp", InitContainers: []v1.Container{ - v1.Container{ - Name: "zync-db-svc", - Image: "amp-zync:latest", + { + Name: ZyncInitContainerName, + Image: containerImage, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", - }, Env: []v1.EnvVar{ - v1.EnvVar{ + }, + Env: []v1.EnvVar{ + { Name: "SLEEP_SECONDS", Value: "1", }, - v1.EnvVar{ + { Name: "DATABASE_URL", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -232,9 +215,9 @@ func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: ZyncName, - Image: "amp-zync:latest", + Image: containerImage, Ports: zync.zyncPorts(), Env: zync.commonZyncEnvVars(), LivenessProbe: &v1.Probe{ @@ -283,7 +266,7 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { helper.EnvVarFromSecret("DATABASE_URL", "zync", "DATABASE_URL"), helper.EnvVarFromSecret("SECRET_KEY_BASE", "zync", "SECRET_KEY_BASE"), helper.EnvVarFromSecret("ZYNC_AUTHENTICATION_TOKEN", "zync", "ZYNC_AUTHENTICATION_TOKEN"), - v1.EnvVar{ + { Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ @@ -292,7 +275,7 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { }, }, }, - v1.EnvVar{ + { Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ @@ -303,54 +286,34 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { }, } } -func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) QueDeployment(containerImage string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ZyncQueDeploymentName, Labels: zync.Options.CommonZyncQueLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: zync.Options.ZyncQueReplicas, - Selector: map[string]string{"deploymentConfig": ZyncQueDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zync.Options.ZyncQueReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncQueDeploymentName, + }, + }, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, }, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "que", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-zync:%s", zync.Options.ImageTag), - }, - }, - }, - }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncQuePodTemplateLabels, Annotations: zync.Options.ZyncQuePodTemplateAnnotations, @@ -362,11 +325,11 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { RestartPolicy: v1.RestartPolicyAlways, TerminationGracePeriodSeconds: &[]int64{30}[0], Containers: []v1.Container{ - v1.Container{ + { Name: "que", Command: []string{"/usr/bin/bash"}, Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, - Image: "amp-zync:latest", + Image: containerImage, ImagePullPolicy: v1.PullAlways, LivenessProbe: &v1.Probe{ FailureThreshold: 3, @@ -383,7 +346,11 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { }, }, Ports: []v1.ContainerPort{ - v1.ContainerPort{Name: "metrics", ContainerPort: ZyncQueMetricsPort, Protocol: v1.ProtocolTCP}, + { + Name: "metrics", + ContainerPort: ZyncQueMetricsPort, + Protocol: v1.ProtocolTCP, + }, }, Resources: zync.Options.QueContainerResourceRequirements, Env: zync.commonZyncEnvVars(), @@ -397,41 +364,26 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { } } -func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deployment { + var zyncDatabaseReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ZyncDatabaseDeploymentName, Labels: zync.Options.CommonZyncDatabaseLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "postgresql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("zync-database-postgresql:%s", zync.Options.DatabaseImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zyncDatabaseReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncDatabaseDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": ZyncDatabaseDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncDatabasePodTemplateLabels, Annotations: zync.Options.ZyncDatabasePodTemplateAnnotations, @@ -442,26 +394,28 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { RestartPolicy: v1.RestartPolicyAlways, ServiceAccountName: "amp", Containers: []v1.Container{ - v1.Container{ + { Name: "postgresql", - Image: " ", + Image: containerImage, Ports: []v1.ContainerPort{ - v1.ContainerPort{ + { ContainerPort: 5432, - Protocol: v1.ProtocolTCP}, + Protocol: v1.ProtocolTCP, + }, }, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "zync-database-data", MountPath: "/var/lib/pgsql/data", }, }, ImagePullPolicy: v1.PullIfNotPresent, Env: []v1.EnvVar{ - v1.EnvVar{ + { Name: "POSTGRESQL_USER", Value: "zync", - }, v1.EnvVar{ + }, + { Name: "POSTGRESQL_PASSWORD", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -471,7 +425,8 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { Key: "ZYNC_DATABASE_PASSWORD", }, }, - }, v1.EnvVar{ + }, + { Name: "POSTGRESQL_DATABASE", Value: "zync_production", }, @@ -500,7 +455,7 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { PriorityClassName: zync.Options.ZyncDatabasePriorityClassName, TopologySpreadConstraints: zync.Options.ZyncDatabaseTopologySpreadConstraints, Volumes: []v1.Volume{ - v1.Volume{ + { Name: "zync-database-data", VolumeSource: v1.VolumeSource{ EmptyDir: &v1.EmptyDirVolumeSource{ @@ -527,14 +482,14 @@ func (zync *Zync) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "8080-tcp", Protocol: v1.ProtocolTCP, Port: 8080, TargetPort: intstr.FromInt(8080), }, }, - Selector: map[string]string{"deploymentConfig": ZyncName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ZyncName}, }, } } @@ -551,14 +506,14 @@ func (zync *Zync) DatabaseService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "postgresql", Protocol: v1.ProtocolTCP, Port: 5432, TargetPort: intstr.FromInt(5432), }, }, - Selector: map[string]string{"deploymentConfig": "zync-database"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "zync-database"}, }, } } @@ -575,7 +530,7 @@ func (zync *Zync) ZyncPodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ZyncName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ZyncName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -594,7 +549,7 @@ func (zync *Zync) QuePodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "zync-que"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "zync-que"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -603,7 +558,7 @@ func (zync *Zync) QuePodDisruptionBudget() *policyv1.PodDisruptionBudget { func (zync *Zync) zyncPorts() []v1.ContainerPort { ports := []v1.ContainerPort{ - v1.ContainerPort{ContainerPort: 8080, Protocol: v1.ProtocolTCP}, + {ContainerPort: 8080, Protocol: v1.ProtocolTCP}, } if zync.Options.ZyncMetrics { diff --git a/pkg/3scale/amp/operator/ampimages_options_provider.go b/pkg/3scale/amp/operator/ampimages_options_provider.go index 5cb31ed83..bd753dc9b 100644 --- a/pkg/3scale/amp/operator/ampimages_options_provider.go +++ b/pkg/3scale/amp/operator/ampimages_options_provider.go @@ -21,7 +21,6 @@ func NewAmpImagesOptionsProvider(apimanager *appsv1alpha1.APIManager) *AmpImages func (a *AmpImagesOptionsProvider) GetAmpImagesOptions() (*component.AmpImagesOptions, error) { a.ampImagesOptions.AppLabel = *a.apimanager.Spec.AppLabel a.ampImagesOptions.AmpRelease = product.ThreescaleRelease - a.ampImagesOptions.InsecureImportPolicy = *a.apimanager.Spec.ImageStreamTagImportInsecure a.ampImagesOptions.ApicastImage = ApicastImageURL() if a.apimanager.Spec.Apicast != nil && a.apimanager.Spec.Apicast.Image != nil { diff --git a/pkg/3scale/amp/operator/ampimages_options_provider_test.go b/pkg/3scale/amp/operator/ampimages_options_provider_test.go index c088777d7..b6166af5c 100644 --- a/pkg/3scale/amp/operator/ampimages_options_provider_test.go +++ b/pkg/3scale/amp/operator/ampimages_options_provider_test.go @@ -34,7 +34,6 @@ func defaultAmpImageOptions() *component.AmpImagesOptions { ZyncDatabasePostgreSQLImage: component.ZyncPostgreSQLImageURL(), SystemMemcachedImage: SystemMemcachedImageURL(), SystemSearchdImage: SystemSearchdImageURL(), - InsecureImportPolicy: insecureImportPolicy, ImagePullSecrets: component.AmpImagesDefaultImagePullSecrets(), } } diff --git a/pkg/3scale/amp/operator/ampimages_reconciler.go b/pkg/3scale/amp/operator/ampimages_reconciler.go index 7fb33be03..215f78246 100644 --- a/pkg/3scale/amp/operator/ampimages_reconciler.go +++ b/pkg/3scale/amp/operator/ampimages_reconciler.go @@ -24,50 +24,6 @@ func (r *AMPImagesReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // backend IS - err = r.ReconcileImagestream(ampImages.BackendImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // zync IS - err = r.ReconcileImagestream(ampImages.ZyncImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // apicast IS - err = r.ReconcileImagestream(ampImages.APICastImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // system IS - err = r.ReconcileImagestream(ampImages.SystemImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - if !r.apiManager.IsExternal(appsv1alpha1.ZyncDatabase) { - // zync db postresql IS - err = r.ReconcileImagestream(ampImages.ZyncDatabasePostgreSQLImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - } - - // system memcached IS - err = r.ReconcileImagestream(ampImages.SystemMemcachedImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // system searchd IS - err = r.ReconcileImagestream(ampImages.SystemSearchdImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - err = r.ReconcileServiceAccount(ampImages.DeploymentsServiceAccount(), reconcilers.ServiceAccountImagePullPolicyMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/ampimages_reconciler_test.go b/pkg/3scale/amp/operator/ampimages_reconciler_test.go index 560cb7d56..9bc91b60e 100644 --- a/pkg/3scale/amp/operator/ampimages_reconciler_test.go +++ b/pkg/3scale/amp/operator/ampimages_reconciler_test.go @@ -11,7 +11,6 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -30,7 +29,6 @@ func TestAMPImagesReconciler(t *testing.T) { wildcardDomain = "test.3scale.net" log = logf.Log.WithName("operator_test") appLabel = "someLabel" - trueValue = true ) ctx := context.TODO() @@ -42,9 +40,8 @@ func TestAMPImagesReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, }, }, } @@ -85,13 +82,6 @@ func TestAMPImagesReconciler(t *testing.T) { objName string obj client.Object }{ - {"backendCreated", "amp-backend", &imagev1.ImageStream{}}, - {"zyncCreated", "amp-zync", &imagev1.ImageStream{}}, - {"apicastCreated", "amp-apicast", &imagev1.ImageStream{}}, - {"systemCreated", "amp-system", &imagev1.ImageStream{}}, - {"zyncPostgresqlCreated", "zync-database-postgresql", &imagev1.ImageStream{}}, - {"systemMemcachedCreated", "system-memcached", &imagev1.ImageStream{}}, - // TODO: service account created by AMPImagesReconciler. Should not be there. {"serviceAccountCreated", "amp", &v1.ServiceAccount{}}, } @@ -110,103 +100,3 @@ func TestAMPImagesReconciler(t *testing.T) { }) } } - -func TestAMPImagesReconcilerWithAllExternalDatabases(t *testing.T) { - var ( - name = "example-apimanager" - namespace = "operator-unittest" - wildcardDomain = "test.3scale.net" - log = logf.Log.WithName("operator_test") - appLabel = "someLabel" - trueValue = true - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - }, - HighAvailability: &appsv1alpha1.HighAvailabilitySpec{ - Enabled: true, - ExternalZyncDatabaseEnabled: &trueValue, - }, - ExternalComponents: appsv1alpha1.AllComponentsExternal(), - }, - } - // Objects to track in the fake client. - objs := []runtime.Object{apimanager} - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - err = imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - err = routev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - recorder := record.NewFakeRecorder(10000) - clientset := fakeclientset.NewSimpleClientset() - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - imagesReconciler := NewAMPImagesReconciler(baseAPIManagerLogicReconciler) - _, err = imagesReconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - cases := []struct { - testName string - objName string - obj client.Object - hasToExist bool - }{ - {"backendCreated", "amp-backend", &imagev1.ImageStream{}, true}, - {"zyncCreated", "amp-zync", &imagev1.ImageStream{}, true}, - {"apicastCreated", "amp-apicast", &imagev1.ImageStream{}, true}, - {"systemCreated", "amp-system", &imagev1.ImageStream{}, true}, - {"zyncPostgresqlCreated", "zync-database-postgresql", &imagev1.ImageStream{}, false}, - {"systemMemcachedCreated", "system-memcached", &imagev1.ImageStream{}, true}, - // TODO: service account created by AMPImagesReconciler. Should not be there. - {"serviceAccountCreated", "amp", &v1.ServiceAccount{}, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - obj := tc.obj - namespacedName := types.NamespacedName{ - Name: tc.objName, - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - if tc.hasToExist { - if err != nil { - subT.Errorf("error fetching object %s: %v", tc.objName, err) - } - } else { - if err == nil || !errors.IsNotFound(err) { - subT.Errorf("object %s that shouldn't exist exists or different error than NotFound returned: %v", tc.objName, err) - } - } - - }) - } -} diff --git a/pkg/3scale/amp/operator/apicast_options_provider.go b/pkg/3scale/amp/operator/apicast_options_provider.go index 6ea40c043..c37c47fc7 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider.go +++ b/pkg/3scale/amp/operator/apicast_options_provider.go @@ -19,6 +19,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type ApicastOptionsProvider struct { @@ -146,7 +147,7 @@ func (a *ApicastOptionsProvider) setResourceRequirementsOptions() { a.apicastOptions.StagingResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if a.apimanager.Spec.Apicast.ProductionSpec.Resources != nil { @@ -208,7 +209,7 @@ func (a *ApicastOptionsProvider) stagingPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "apicast-staging" + labels[reconcilers.DeploymentLabelSelector] = "apicast-staging" return labels } @@ -224,7 +225,7 @@ func (a *ApicastOptionsProvider) productionPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "apicast-production" + labels[reconcilers.DeploymentLabelSelector] = "apicast-production" return labels } @@ -245,7 +246,7 @@ func (a *ApicastOptionsProvider) setCustomPolicies() error { Child("apicast"). Child("productionSpec"). Child("customPolicies").Index(idx) - fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) + fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) return fldErr.ToAggregate() } @@ -272,7 +273,7 @@ func (a *ApicastOptionsProvider) setCustomPolicies() error { Child("apicast"). Child("stagingSpec"). Child("customPolicies").Index(idx) - fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) + fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) return fldErr.ToAggregate() } diff --git a/pkg/3scale/amp/operator/apicast_options_provider_test.go b/pkg/3scale/amp/operator/apicast_options_provider_test.go index c875a3f34..caab1d1c4 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider_test.go +++ b/pkg/3scale/amp/operator/apicast_options_provider_test.go @@ -18,6 +18,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( @@ -53,10 +54,10 @@ func testApicastProductionLabels() map[string]string { func testApicastStagingPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "apicast", - "threescale_component_element": "staging", - "deploymentConfig": "apicast-staging", + "app": appLabel, + "threescale_component": "apicast", + "threescale_component_element": "staging", + reconcilers.DeploymentLabelSelector: "apicast-staging", } addExpectedMeteringLabels(labels, "apicast-staging", helper.ApplicationType) @@ -65,10 +66,10 @@ func testApicastStagingPodLabels() map[string]string { func testApicastProductionPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "apicast", - "threescale_component_element": "production", - "deploymentConfig": "apicast-production", + "app": appLabel, + "threescale_component": "apicast", + "threescale_component_element": "production", + reconcilers.DeploymentLabelSelector: "apicast-production", } addExpectedMeteringLabels(labels, "apicast-production", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index 7a4b97e6f..6a496092b 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -6,7 +6,7 @@ import ( "reflect" "strings" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,6 +17,7 @@ import ( "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" ) func ApicastEnvCMMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { @@ -57,17 +58,22 @@ func NewApicastReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReco } func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + apicast, err := Apicast(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - stagingMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, + stagingMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAnnotationsMutator, + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, apicastLogLevelEnvVarMutator, apicastTracingConfigEnvVarsMutator, apicastOpentelemetryConfigEnvVarsMutator, @@ -83,28 +89,38 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { apicastCustomEnvAnnotationsMutator, // Should be always after volume mutator portsMutator, apicastPodTemplateEnvConfigMapAnnotationsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, } if r.apiManager.Spec.Apicast.StagingSpec.Replicas != nil { - stagingMutators = append(stagingMutators, reconcilers.DeploymentConfigReplicasMutator) + stagingMutators = append(stagingMutators, reconcilers.DeploymentReplicasMutator) } - // Staging DC - err = r.ReconcileDeploymentConfig(apicast.StagingDeploymentConfig(), reconcilers.DeploymentConfigMutator(stagingMutators...)) + // Staging Deployment + err = r.ReconcileDeployment(apicast.StagingDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(stagingMutators...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + // add apicast production env var mutator - productionMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, + productionMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAnnotationsMutator, + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, apicastProductionWorkersEnvVarMutator, apicastLogLevelEnvVarMutator, apicastTracingConfigEnvVarsMutator, @@ -121,33 +137,45 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { apicastCustomEnvAnnotationsMutator, // Should be always after volume portsMutator, apicastPodTemplateEnvConfigMapAnnotationsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, } if r.apiManager.Spec.Apicast.ProductionSpec.Replicas != nil { - productionMutators = append(productionMutators, reconcilers.DeploymentConfigReplicasMutator) + productionMutators = append(productionMutators, reconcilers.DeploymentReplicasMutator) } - // Production DC - productionDCMutator := reconcilers.DeploymentConfigMutator( - productionMutators..., - ) + // Production Deployment + err = r.ReconcileDeployment(apicast.ProductionDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(productionMutators...)) + if err != nil { + return reconcile.Result{}, err + } - err = r.ReconcileDeploymentConfig(apicast.ProductionDeploymentConfig(), productionDCMutator) + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations()), + reconcilers.ServiceSelectorMutator, + } // Staging Service - err = r.ReconcileService(apicast.StagingService(), getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations())) + err = r.ReconcileService(apicast.StagingService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } // Production Service - err = r.ReconcileService(apicast.ProductionService(), getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations())) + err = r.ReconcileService(apicast.ProductionService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -226,44 +254,44 @@ func getApiCastServiceMutator(apiManagerAnnotations map[string]string) reconcile return reconcilers.ServicePortMutator } -func apicastProductionWorkersEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastProductionWorkersEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_WORKERS" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_WORKERS"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_WORKERS"), nil } -func apicastLogLevelEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastLogLevelEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_LOG_LEVEL" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_LOG_LEVEL"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_LOG_LEVEL"), nil } -func apicastTracingConfigEnvVarsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastTracingConfigEnvVarsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTRACING_TRACER") + changed = reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTRACING_TRACER") - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTRACING_CONFIG") + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTRACING_CONFIG") changed = changed || tmpChanged return changed, nil } -func apicastOpentelemetryConfigEnvVarsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastOpentelemetryConfigEnvVarsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTELEMETRY") + changed = reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTELEMETRY") - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTELEMETRY_CONFIG") + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTELEMETRY_CONFIG") changed = changed || tmpChanged return changed, nil } -func apicastEnvironmentEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastEnvironmentEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_ENVIRONMENT" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_ENVIRONMENT"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_ENVIRONMENT"), nil } -func apicastHTTPSEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastHTTPSEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool @@ -273,14 +301,14 @@ func apicastHTTPSEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool "APICAST_HTTPS_CERTIFICATE", "APICAST_HTTPS_CERTIFICATE_KEY", } { - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, envVar) + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, envVar) changed = changed || tmpChanged } return changed, nil } -func apicastProxyConfigurationsEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastProxyConfigurationsEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to APIcast proxy-related configurations var changed bool @@ -290,18 +318,18 @@ func apicastProxyConfigurationsEnvVarMutator(desired, existing *appsv1.Deploymen "HTTPS_PROXY", "NO_PROXY", } { - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, envVar) + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, envVar) changed = changed || tmpChanged } return changed, nil } -func apicastServiceCacheSizeEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_SERVICE_CACHE_SIZE"), nil +func apicastServiceCacheSizeEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_SERVICE_CACHE_SIZE"), nil } -func portsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func portsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Ports, desired.Spec.Template.Spec.Containers[0].Ports) { @@ -312,11 +340,11 @@ func portsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { return changed, nil } -// volumeMountsMutator implements basic VolumeMount reconcilliation +// apicastVolumeMountsMutator implements basic VolumeMount reconciliation // Added when in desired and not in existing // Updated when in desired and in existing but not equal // Existing not in desired will NOT be removed. Allows manually added volumemounts -func apicastVolumeMountsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastVolumeMountsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false existingContainer := &existing.Spec.Template.Spec.Containers[0] desiredContainer := &desired.Spec.Template.Spec.Containers[0] @@ -405,11 +433,11 @@ func apicastVolumeMountsMutator(desired, existing *appsv1.DeploymentConfig) (boo return changed, nil } -// volumeMountsMutator implements basic VolumeMount reconcilliation +// apicastVolumesMutator implements basic VolumeMount reconciliation // Added when in desired and not in existing // Updated when in desired and in existing but not equal // Existing not in desired will NOT be removed. Allows manually added volumemounts -func apicastVolumesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastVolumesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false existingSpec := &existing.Spec.Template.Spec desiredSpec := &desired.Spec.Template.Spec @@ -498,7 +526,7 @@ func apicastVolumesMutator(desired, existing *appsv1.DeploymentConfig) (bool, er return changed, nil } -func apicastCustomPolicyAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastCustomPolicyAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing custom policy annotations not in desired and delete them updated := false @@ -522,7 +550,7 @@ func apicastCustomPolicyAnnotationsMutator(desired, existing *appsv1.DeploymentC return updated, nil } -func apicastTracingConfigAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastTracingConfigAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing tracing config volume annotations not in desired and delete them updated := false @@ -546,7 +574,7 @@ func apicastTracingConfigAnnotationsMutator(desired, existing *appsv1.Deployment return updated, nil } -func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing tracing config volume annotations not in desired and delete them updated := false @@ -570,7 +598,7 @@ func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *appsv1.Depl return updated, nil } -func apicastCustomEnvAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastCustomEnvAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing custom environments annotations not in desired and delete them updated := false @@ -594,7 +622,7 @@ func apicastCustomEnvAnnotationsMutator(desired, existing *appsv1.DeploymentConf return updated, nil } -func apicastPodTemplateEnvConfigMapAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastPodTemplateEnvConfigMapAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Only reconcile the pod annotation regarding apicast-environment hash desiredVal, ok := desired.Spec.Template.Annotations[APIcastEnvironmentCMAnnotation] if !ok { diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index 892bb5daa..a491641f6 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -5,20 +5,19 @@ import ( "fmt" "testing" - "github.com/3scale/3scale-operator/pkg/common" - "k8s.io/apimachinery/pkg/util/intstr" - grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" appsv1 "github.com/openshift/api/apps/v1" configv1 "github.com/openshift/api/config/v1" imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" fakeclientset "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -29,6 +28,7 @@ import ( "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/reconcilers" ) @@ -54,11 +54,10 @@ func TestApicastReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -120,8 +119,8 @@ func TestApicastReconciler(t *testing.T) { objName string obj k8sclient.Object }{ - {"stagingDeployment", "apicast-staging", &appsv1.DeploymentConfig{}}, - {"productionDeployment", "apicast-production", &appsv1.DeploymentConfig{}}, + {"stagingDeployment", "apicast-staging", &k8sappsv1.Deployment{}}, + {"productionDeployment", "apicast-production", &k8sappsv1.Deployment{}}, {"stagingService", "apicast-staging", &v1.Service{}}, {"productionService", "apicast-production", &v1.Service{}}, {"envConfigMap", "apicast-environment", &v1.ConfigMap{}}, @@ -191,11 +190,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -217,11 +215,15 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { }, }, } + ampImages, err := AmpImages(apimanager) + if err != nil { + t.Fatal(err) + } - // Existing DC has 1 custom policy defined: P1 - // Desired DC has 1 custom policy defined: P2 - // P2 should be added to existing DC - // P1 should be deleted from existing DC + // Existing Deployment has 1 custom policy defined: P1 + // Desired Deployment has 1 custom policy defined: P2 + // P2 should be added to existing Deployment + // P1 should be deleted from existing Deployment apicastOptions := &component.ApicastOptions{ ProductionCustomPolicies: []component.CustomPolicy{p1CustomPolicy}, @@ -229,12 +231,12 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { ProductionTracingConfig: &component.APIcastTracingConfig{}, } apicast := component.NewApicast(apicastOptions) - existingProdDC := apicast.ProductionDeploymentConfig() - existingProdDC.Namespace = namespace + existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) + existingProdDeployment.Namespace = namespace // - Policy annotation for P1 added p1Found := false - for key := range existingProdDC.Annotations { + for key := range existingProdDeployment.Annotations { if p1CustomPolicy.AnnotationKey() == key { p1Found = true } @@ -255,10 +257,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { } // Objects to track in the fake client. - objs := []runtime.Object{apimanager, existingProdDC, p2Secret} + objs := []runtime.Object{apimanager, existingProdDeployment, p2Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err = appsv1.AddToScheme(s) if err != nil { t.Fatal(err) } @@ -299,14 +301,14 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { Name: "apicast-production", Namespace: namespace, } - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} err = cl.Get(context.TODO(), namespacedName, existing) // object must exist, that is all required to be tested if err != nil { t.Fatal(err) } - // Assert existing DC: + // Assert existing Deployment: // - Volume for P1 deleted for idx := range existing.Spec.Template.Spec.Volumes { if existing.Spec.Template.Spec.Volumes[idx].Name == p1CustomPolicy.VolumeName() { @@ -392,44 +394,6 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { } ) - apicastOptions := &component.ApicastOptions{ - StagingTracingConfig: &component.APIcastTracingConfig{}, - ProductionTracingConfig: &existingTracingConfig1, - } - apicast := component.NewApicast(apicastOptions) - existingProdDC := apicast.ProductionDeploymentConfig() - existingProdDC.Namespace = namespace - - // - Tracing Configuration 1 added into the Production DC with the expected key - existingTracingConfig1Found := false - for key := range existingProdDC.Annotations { - if existingTracingConfig1.AnnotationKey() == key { - existingTracingConfig1Found = true - } - } - - if !existingTracingConfig1Found { - t.Fatal("tracing config 1 annotation not found. Should have been created") - } - - existingTc1Secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, - ObjectMeta: metav1.ObjectMeta{Name: *existingTracingConfig1.TracingConfigSecretName, Namespace: namespace}, - Data: map[string][]byte{ - "config": []byte("some existing tracing config"), - }, - Type: v1.SecretTypeOpaque, - } - - desiredTc1Secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, - ObjectMeta: metav1.ObjectMeta{Name: *desiredTracingConfig1.TracingConfigSecretName, Namespace: namespace}, - Data: map[string][]byte{ - "config": []byte("some desired tracing config"), - }, - Type: v1.SecretTypeOpaque, - } - apimanager := &appsv1alpha1.APIManager{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -437,11 +401,10 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -468,12 +431,54 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { }, }, } + ampImages, err := AmpImages(apimanager) + if err != nil { + t.Fatal(err) + } + + apicastOptions := &component.ApicastOptions{ + StagingTracingConfig: &component.APIcastTracingConfig{}, + ProductionTracingConfig: &existingTracingConfig1, + } + apicast := component.NewApicast(apicastOptions) + existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) + existingProdDeployment.Namespace = namespace + + // - Tracing Configuration 1 added into the Production Deployment with the expected key + existingTracingConfig1Found := false + for key := range existingProdDeployment.Annotations { + if existingTracingConfig1.AnnotationKey() == key { + existingTracingConfig1Found = true + } + } + + if !existingTracingConfig1Found { + t.Fatal("tracing config 1 annotation not found. Should have been created") + } + + existingTc1Secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: *existingTracingConfig1.TracingConfigSecretName, Namespace: namespace}, + Data: map[string][]byte{ + "config": []byte("some existing tracing config"), + }, + Type: v1.SecretTypeOpaque, + } + + desiredTc1Secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: *desiredTracingConfig1.TracingConfigSecretName, Namespace: namespace}, + Data: map[string][]byte{ + "config": []byte("some desired tracing config"), + }, + Type: v1.SecretTypeOpaque, + } // Objects to track in the fake client. - objs := []runtime.Object{apimanager, existingProdDC, existingTc1Secret, desiredTc1Secret} + objs := []runtime.Object{apimanager, existingProdDeployment, existingTc1Secret, desiredTc1Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err = appsv1.AddToScheme(s) if err != nil { t.Fatal(err) } @@ -515,14 +520,14 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { Name: "apicast-production", Namespace: namespace, } - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} err = cl.Get(context.TODO(), namespacedName, existing) // object must exist, that is all required to be tested if err != nil { t.Fatal(err) } - // // Assert existing DC: + // // Assert existing Deployment: // // - Volume for existingTracingConfig1 deleted for idx := range existing.Spec.Template.Spec.Volumes { if existing.Spec.Template.Spec.Volumes[idx].Name == existingTracingConfig1.VolumeName() { @@ -638,11 +643,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -670,11 +674,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -701,11 +704,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -838,22 +840,22 @@ func TestReplicaApicastReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -862,13 +864,13 @@ func TestReplicaApicastReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } @@ -892,11 +894,10 @@ func testApicastAPIManagerCreator(stagingReplicas, productionReplicas *int64) *a }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -912,14 +913,14 @@ func testApicastAPIManagerCreator(stagingReplicas, productionReplicas *int64) *a func TestReplicaApicastTelemtryReconciler(t *testing.T) { var ( - trueValue = true - log = logf.Log.WithName("operator_test") - opentelemtryEnabled bool = true - apicastManagementAPI = "enabled" - openSSLVerify = &trueValue - includeResponseCodes = &trueValue - customKey = "my-custom-key.json" - configJsonKey = "config.json" + trueValue = true + log = logf.Log.WithName("operator_test") + opentelemtryEnabled = true + apicastManagementAPI = "enabled" + openSSLVerify = &trueValue + includeResponseCodes = &trueValue + customKey = "my-custom-key.json" + configJsonKey = "config.json" ) ctx := context.TODO() @@ -933,6 +934,10 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { if err != nil { t.Fatal(err) } + err = k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } if err := configv1.AddToScheme(s); err != nil { t.Fatal(err) @@ -1181,7 +1186,7 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { }, }, ), - opentelemetryEnvExistsWithCustomKeysOnBothDCs, + opentelemetryEnvExistsWithCustomKeysOnBothDeployments, false, false, multiKeyOtlpSecret(), @@ -1292,19 +1297,19 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { } } -func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func validateOpentelemetryIsDisabled(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ - Name: dcType, + Name: dType, Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - envs := dc.Spec.Template.Spec.Containers[0].Env + envs := deployment.Spec.Template.Spec.Containers[0].Env var ( configEnvfound bool configEnvValueCorrect bool @@ -1330,22 +1335,22 @@ func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, c // Check if required environment variables are present and have correct values if configEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment %s", dType) } if opentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment %s", dType) } if configEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment %s", dType) } if opentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment %s", dType) } - volumeMounts := dc.Spec.Template.Spec.Containers[0].VolumeMounts + volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts var volumeMountFound bool - volumes := dc.Spec.Template.Spec.Volumes + volumes := deployment.Spec.Template.Spec.Volumes var volumeFound bool // Iterate over environment variables @@ -1362,16 +1367,16 @@ func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, c } if volumeMountFound { - return false, fmt.Errorf("Opentelemetry volume mount found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume mount found on deployment %s", dType) } if volumeFound { - return false, fmt.Errorf("Opentelemetry volume found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume found on deployment %s", dType) } return true, nil } -func disableOpentelemtry(dc string, client k8sclient.WithWatch) (error, *appsv1alpha1.APIManager) { +func disableOpentelemtry(deployment string, client k8sclient.WithWatch) (error, *appsv1alpha1.APIManager) { apim := &appsv1alpha1.APIManager{} namespacedName := types.NamespacedName{ Name: "example-apimanager", @@ -1383,29 +1388,29 @@ func disableOpentelemtry(dc string, client k8sclient.WithWatch) (error, *appsv1a return fmt.Errorf("error fetching APIM %s", err), nil } - if dc == "apicast-staging" { + if deployment == "apicast-staging" { *apim.Spec.Apicast.StagingSpec.OpenTelemetry = appsv1alpha1.OpenTelemetrySpec{} } - if dc == "apicast-production" { + if deployment == "apicast-production" { *apim.Spec.Apicast.ProductionSpec.OpenTelemetry = appsv1alpha1.OpenTelemetrySpec{} } return nil, apim } -func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func opentelemetryEnvExistsWithDefaultValues(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ - Name: dcType, + Name: dType, Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - envs := dc.Spec.Template.Spec.Containers[0].Env + envs := deployment.Spec.Template.Spec.Containers[0].Env var ( configEnvfound bool configEnvValueCorrect bool @@ -1431,22 +1436,22 @@ func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue s // Check if required environment variables are present and have correct values if !configEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment %s", dType) } if !opentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment %s", dType) } if !configEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment %s", dType) } if !opentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment %s", dType) } - volumeMounts := dc.Spec.Template.Spec.Containers[0].VolumeMounts + volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts var volumeMountFound bool - volumes := dc.Spec.Template.Spec.Volumes + volumes := deployment.Spec.Template.Spec.Volumes var volumeFound bool // Iterate over environment variables @@ -1463,29 +1468,29 @@ func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue s } if !volumeMountFound { - return false, fmt.Errorf("Opentelemetry volume mount not found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume mount not found on deployment %s", dType) } if !volumeFound { - return false, fmt.Errorf("Opentelemetry volume not found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume not found on deployment %s", dType) } // All environment variables are correctly set return true, nil } -func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func opentelemetryEnvExistsWithCustomKeysOnBothDeployments(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: "apicast-staging", Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - stageEnvs := dc.Spec.Template.Spec.Containers[0].Env + stageEnvs := deployment.Spec.Template.Spec.Containers[0].Env var ( stageConfigEnvfound bool stageConfigEnvValueCorrect bool @@ -1511,30 +1516,30 @@ func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigV // Check if required environment variables are present and have correct values if !stageConfigEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment stage %s", err) } if !stageOpentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment stage %s", err) } if !stageConfigEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment stage %s", err) } if !stageOpentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment stage %s", err) } - dc = &appsv1.DeploymentConfig{} + deployment = &k8sappsv1.Deployment{} namespacedName = types.NamespacedName{ Name: "apicast-production", Namespace: namespace, } - err = client.Get(context.TODO(), namespacedName, dc) + err = client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - prodEnvs := dc.Spec.Template.Spec.Containers[0].Env + prodEnvs := deployment.Spec.Template.Spec.Containers[0].Env var ( prodConfigEnvfound bool prodConfigEnvValueCorrect bool @@ -1560,16 +1565,16 @@ func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigV // Check if required environment variables are present and have correct values if !prodConfigEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment production %s", err) } if !prodOpentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment production %s", err) } if !prodConfigEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment production %s", err) } if !prodOpentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment production %s", err) } // All environment variables are correctly set @@ -1593,11 +1598,10 @@ func testApicastAPIManagerTelemetryCreator(apicastSpec *appsv1alpha1.ApicastSpec }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: apicastSpec, }, diff --git a/pkg/3scale/amp/operator/backend_options_provider.go b/pkg/3scale/amp/operator/backend_options_provider.go index 742050cd6..9224c856d 100644 --- a/pkg/3scale/amp/operator/backend_options_provider.go +++ b/pkg/3scale/amp/operator/backend_options_provider.go @@ -7,6 +7,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -121,7 +122,7 @@ func (o *OperatorBackendOptionsProvider) setResourceRequirementsOptions() { o.backendOptions.CronResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if o.apimanager.Spec.Backend.ListenerSpec.Resources != nil { @@ -197,7 +198,7 @@ func (o *OperatorBackendOptionsProvider) listenerPodTemplateLabels() map[string] labels[k] = v } - labels["deploymentConfig"] = "backend-listener" + labels[reconcilers.DeploymentLabelSelector] = "backend-listener" return labels } @@ -213,7 +214,7 @@ func (o *OperatorBackendOptionsProvider) workerPodTemplateLabels() map[string]st labels[k] = v } - labels["deploymentConfig"] = "backend-worker" + labels[reconcilers.DeploymentLabelSelector] = "backend-worker" return labels } @@ -229,7 +230,7 @@ func (o *OperatorBackendOptionsProvider) cronPodTemplateLabels() map[string]stri labels[k] = v } - labels["deploymentConfig"] = "backend-cron" + labels[reconcilers.DeploymentLabelSelector] = "backend-cron" return labels } diff --git a/pkg/3scale/amp/operator/backend_options_provider_test.go b/pkg/3scale/amp/operator/backend_options_provider_test.go index a350c1e71..e19207e50 100644 --- a/pkg/3scale/amp/operator/backend_options_provider_test.go +++ b/pkg/3scale/amp/operator/backend_options_provider_test.go @@ -9,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" @@ -56,10 +57,10 @@ func testBackendCommonCronLabels() map[string]string { func testBackendListenerPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "listener", - "deploymentConfig": "backend-listener", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "listener", + reconcilers.DeploymentLabelSelector: "backend-listener", } addExpectedMeteringLabels(labels, "backend-listener", helper.ApplicationType) @@ -68,10 +69,10 @@ func testBackendListenerPodLabels() map[string]string { func testBackendWorkerPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "worker", - "deploymentConfig": "backend-worker", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "worker", + reconcilers.DeploymentLabelSelector: "backend-worker", } addExpectedMeteringLabels(labels, "backend-worker", helper.ApplicationType) @@ -80,10 +81,10 @@ func testBackendWorkerPodLabels() map[string]string { func testBackendCronPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "cron", - "deploymentConfig": "backend-cron", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "cron", + reconcilers.DeploymentLabelSelector: "backend-cron", } addExpectedMeteringLabels(labels, "backend-cron", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/backend_reconciler.go b/pkg/3scale/amp/operator/backend_reconciler.go index e962aabc2..5586a1c8e 100644 --- a/pkg/3scale/amp/operator/backend_reconciler.go +++ b/pkg/3scale/amp/operator/backend_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -21,35 +22,63 @@ func NewBackendReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReco } func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + backend, err := Backend(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // Cron DC - cronConfigMutator := reconcilers.GenericBackendMutators() + // Cron Deployment + cronDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.CronSpec.Replicas != nil { - cronConfigMutator = append(cronConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + cronDeploymentMutator = append(cronDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(backend.CronDeploymentConfig(), reconcilers.DeploymentConfigMutator(cronConfigMutator...)) + err = r.ReconcileDeployment(backend.CronDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(cronDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } - // Listener DC - listenerConfigMutator := reconcilers.GenericBackendMutators() + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + // Listener Deployment + listenerDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.ListenerSpec.Replicas != nil { - listenerConfigMutator = append(listenerConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + listenerDeploymentMutator = append(listenerDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(backend.ListenerDeploymentConfig(), reconcilers.DeploymentConfigMutator(listenerConfigMutator...)) + err = r.ReconcileDeployment(backend.ListenerDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(listenerDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Listener Service - err = r.ReconcileService(backend.ListenerService(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(backend.ListenerService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -60,17 +89,26 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Worker DC - workerConfigMutator := reconcilers.GenericBackendMutators() + // Worker Deployment + workerDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.WorkerSpec.Replicas != nil { - workerConfigMutator = append(workerConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + workerDeploymentMutator = append(workerDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(backend.WorkerDeploymentConfig(), reconcilers.DeploymentConfigMutator(workerConfigMutator...)) + err = r.ReconcileDeployment(backend.WorkerDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(workerDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + // Environment ConfigMap err = r.ReconcileConfigMap(backend.EnvironmentConfigMap(), reconcilers.CreateOnlyMutator) if err != nil { diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index 6bd8f4355..eb74aa88b 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -12,6 +12,7 @@ import ( configv1 "github.com/openshift/api/config/v1" imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,11 +47,10 @@ func TestNewBackendReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: &oneValue}, @@ -60,19 +60,37 @@ func TestNewBackendReconciler(t *testing.T) { PodDisruptionBudget: &appsv1alpha1.PodDisruptionBudgetSpec{Enabled: true}, }, } + + backendRedisSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backend-redis", + Namespace: namespace, + }, + } + // Objects to track in the fake client. - objs := []runtime.Object{apimanager} + objs := []runtime.Object{apimanager, backendRedisSecret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = configv1.Install(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -97,11 +115,11 @@ func TestNewBackendReconciler(t *testing.T) { objName string obj k8sclient.Object }{ - {"cronDC", "backend-cron", &appsv1.DeploymentConfig{}}, - {"listenerDC", "backend-listener", &appsv1.DeploymentConfig{}}, + {"cronDeployment", "backend-cron", &k8sappsv1.Deployment{}}, + {"listenerDeployment", "backend-listener", &k8sappsv1.Deployment{}}, {"listenerService", "backend-listener", &v1.Service{}}, {"listenerRoute", "backend", &routev1.Route{}}, - {"workerDC", "backend-worker", &appsv1.DeploymentConfig{}}, + {"workerDeployment", "backend-worker", &k8sappsv1.Deployment{}}, {"environmentCM", "backend-environment", &v1.ConfigMap{}}, {"internalAPISecret", component.BackendSecretInternalApiSecretName, &v1.Secret{}}, {"listenerSecret", component.BackendSecretBackendListenerSecretName, &v1.Secret{}}, @@ -141,18 +159,25 @@ func TestReplicaBackendReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = appsv1.Install(s) if err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } + backendRedisSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backend-redis", + Namespace: namespace, + }, + } + cases := []struct { testName string objName string @@ -171,7 +196,7 @@ func TestReplicaBackendReconciler(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{tc.apimanager} + objs := []runtime.Object{tc.apimanager, backendRedisSecret} cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) clientset := fakeclientset.NewSimpleClientset() @@ -185,21 +210,21 @@ func TestReplicaBackendReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -208,13 +233,13 @@ func TestReplicaBackendReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } @@ -237,11 +262,10 @@ func backendApiManagerCreator(listenerReplicas, cronReplicas, workerReplicas *in }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: listenerReplicas}, diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index 732d54a05..e6e3abfd4 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -10,10 +10,10 @@ import ( "github.com/go-logr/logr" grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" - appsv1 "github.com/openshift/api/apps/v1" - imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -55,12 +55,8 @@ func (r *BaseAPIManagerLogicReconciler) ReconcilePodDisruptionBudget(desired *po return r.ReconcileResource(&policyv1.PodDisruptionBudget{}, desired, mutatefn) } -func (r *BaseAPIManagerLogicReconciler) ReconcileImagestream(desired *imagev1.ImageStream, mutatefn reconcilers.MutateFn) error { - return r.ReconcileResource(&imagev1.ImageStream{}, desired, mutatefn) -} - -func (r *BaseAPIManagerLogicReconciler) ReconcileDeploymentConfig(desired *appsv1.DeploymentConfig, mutatefn reconcilers.MutateFn) error { - return r.ReconcileResource(&appsv1.DeploymentConfig{}, desired, mutatefn) +func (r *BaseAPIManagerLogicReconciler) ReconcileDeployment(desired *k8sappsv1.Deployment, mutatefn reconcilers.MutateFn) error { + return r.ReconcileResource(&k8sappsv1.Deployment{}, desired, mutatefn) } func (r *BaseAPIManagerLogicReconciler) ReconcileService(desired *v1.Service, mutateFn reconcilers.MutateFn) error { @@ -95,6 +91,10 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileRoleBinding(desired *rbacv1.Rol return r.ReconcileResource(&rbacv1.RoleBinding{}, desired, mutateFn) } +func (r *BaseAPIManagerLogicReconciler) ReconcileJob(desired *batchv1.Job, mutateFn reconcilers.MutateFn) error { + return r.ReconcileResource(&batchv1.Job{}, desired, mutateFn) +} + func (r *BaseAPIManagerLogicReconciler) ReconcileGrafanaDashboard(desired *grafanav1alpha1.GrafanaDashboard, mutateFn reconcilers.MutateFn) error { kindExists, err := r.HasGrafanaDashboards() if err != nil { diff --git a/pkg/3scale/amp/operator/helper_test.go b/pkg/3scale/amp/operator/helper_test.go index f5b3a9208..30c79bbf0 100644 --- a/pkg/3scale/amp/operator/helper_test.go +++ b/pkg/3scale/amp/operator/helper_test.go @@ -13,13 +13,12 @@ import ( ) const ( - wildcardDomain = "test.3scale.net" - appLabel = "someLabel" - apimanagerName = "example-apimanager" - namespace = "someNS" - tenantName = "someTenant" - insecureImportPolicy = false - trueValue = true + wildcardDomain = "test.3scale.net" + appLabel = "someLabel" + apimanagerName = "example-apimanager" + namespace = "someNS" + tenantName = "someTenant" + trueValue = true ) func addExpectedMeteringLabels(src map[string]string, componentName string, componentType helper.ComponentType) { @@ -43,7 +42,6 @@ func addExpectedMeteringLabels(src map[string]string, componentName string, comp func basicApimanager() *appsv1alpha1.APIManager { tmpAppLabel := appLabel tmpTenantName := tenantName - tmpInsecureImportPolicy := insecureImportPolicy tmpTrueValue := trueValue apimanager := &appsv1alpha1.APIManager{ @@ -53,11 +51,10 @@ func basicApimanager() *appsv1alpha1.APIManager { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - WildcardDomain: wildcardDomain, - AppLabel: &tmpAppLabel, - ImageStreamTagImportInsecure: &tmpInsecureImportPolicy, - TenantName: &tmpTenantName, - ResourceRequirementsEnabled: &tmpTrueValue, + WildcardDomain: wildcardDomain, + AppLabel: &tmpAppLabel, + TenantName: &tmpTenantName, + ResourceRequirementsEnabled: &tmpTrueValue, }, System: &appsv1alpha1.SystemSpec{}, }, diff --git a/pkg/3scale/amp/operator/memcached_options_provider.go b/pkg/3scale/amp/operator/memcached_options_provider.go index ea74e4a72..948fd84ec 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider.go +++ b/pkg/3scale/amp/operator/memcached_options_provider.go @@ -7,6 +7,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" ) @@ -48,7 +49,7 @@ func (m *MemcachedOptionsProvider) setResourceRequirementsOptions() { m.memcachedOptions.ResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if m.apimanager.Spec.System.MemcachedResources != nil { @@ -80,7 +81,7 @@ func (m *MemcachedOptionsProvider) podTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-memcache" + labels[reconcilers.DeploymentLabelSelector] = "system-memcache" return labels } diff --git a/pkg/3scale/amp/operator/memcached_options_provider_test.go b/pkg/3scale/amp/operator/memcached_options_provider_test.go index bac6eb982..050606ab0 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider_test.go +++ b/pkg/3scale/amp/operator/memcached_options_provider_test.go @@ -8,6 +8,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" @@ -24,10 +26,10 @@ func testMemcachedDeploymentLabels() map[string]string { func testPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "memcache", - "deploymentConfig": "system-memcache", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "memcache", + reconcilers.DeploymentLabelSelector: "system-memcache", } addExpectedMeteringLabels(labels, "system-memcache", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 08ee2c6f7..c9d10422a 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -4,6 +4,8 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -18,27 +20,41 @@ func NewMemcachedReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicRe } func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + memcached, err := Memcached(r.apiManager) if err != nil { return reconcile.Result{}, err } - // DC - mutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Deployment + mutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(memcached.DeploymentConfig(), mutator) + err = r.ReconcileDeployment(memcached.Deployment(ampImages.Options.SystemMemcachedImage), mutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, nil } diff --git a/pkg/3scale/amp/operator/memcached_reconciler_test.go b/pkg/3scale/amp/operator/memcached_reconciler_test.go index 4465fb7c3..c1565e526 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler_test.go +++ b/pkg/3scale/amp/operator/memcached_reconciler_test.go @@ -8,6 +8,7 @@ import ( "github.com/3scale/3scale-operator/pkg/reconcilers" appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" fakeclientset "k8s.io/client-go/kubernetes/fake" @@ -18,13 +19,19 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -func TestMemcachedDCReconciler(t *testing.T) { +func TestMemcachedDeploymentReconciler(t *testing.T) { log := logf.Log.WithName("operator_test") ctx := context.TODO() apimanager := basicApimanager() s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -51,7 +58,7 @@ func TestMemcachedDCReconciler(t *testing.T) { objName string obj client.Object }{ - {"memcachedDC", "system-memcache", &appsv1.DeploymentConfig{}}, + {"memcachedDeployment", "system-memcache", &k8sappsv1.Deployment{}}, } for _, tc := range cases { diff --git a/pkg/3scale/amp/operator/redis_options_provider.go b/pkg/3scale/amp/operator/redis_options_provider.go index f9b55b2af..d6c4c6299 100644 --- a/pkg/3scale/amp/operator/redis_options_provider.go +++ b/pkg/3scale/amp/operator/redis_options_provider.go @@ -7,6 +7,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,7 +35,6 @@ func (r *RedisOptionsProvider) GetRedisOptions() (*component.RedisOptions, error r.options.AmpRelease = product.ThreescaleRelease r.options.BackendImageTag = product.ThreescaleRelease r.options.SystemImageTag = product.ThreescaleRelease - r.options.InsecureImportPolicy = r.apimanager.Spec.ImageStreamTagImportInsecure r.options.BackendImage = BackendRedisImageURL() if r.apimanager.Spec.Backend != nil && r.apimanager.Spec.Backend.RedisImage != nil { @@ -168,7 +168,7 @@ func (r *RedisOptionsProvider) setResourceRequirementsOptions() { r.options.SystemRedisContainerResourceRequirements = &v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if r.apimanager.Spec.Backend.RedisResources != nil { @@ -234,7 +234,7 @@ func (r *RedisOptionsProvider) systemRedisPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "system-redis" + labels[reconcilers.DeploymentLabelSelector] = "system-redis" return labels } @@ -250,7 +250,7 @@ func (r *RedisOptionsProvider) backendRedisPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "backend-redis" + labels[reconcilers.DeploymentLabelSelector] = "backend-redis" return labels } diff --git a/pkg/3scale/amp/operator/redis_options_provider_test.go b/pkg/3scale/amp/operator/redis_options_provider_test.go index 2901f3bb7..2798fbce3 100644 --- a/pkg/3scale/amp/operator/redis_options_provider_test.go +++ b/pkg/3scale/amp/operator/redis_options_provider_test.go @@ -8,6 +8,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -34,10 +35,10 @@ func testRedisSystemRedisLabels() map[string]string { func testRedisSystemRedisPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "redis", - "deploymentConfig": "system-redis", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "redis", + reconcilers.DeploymentLabelSelector: "system-redis", } addExpectedMeteringLabels(labels, "system-redis", helper.ApplicationType) @@ -61,10 +62,10 @@ func testRedisBackendRedisLabels() map[string]string { func testRedisBackendRedisPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "redis", - "deploymentConfig": "backend-redis", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "redis", + reconcilers.DeploymentLabelSelector: "backend-redis", } addExpectedMeteringLabels(labels, "backend-redis", helper.ApplicationType) @@ -136,7 +137,6 @@ func testSystemRedisSecret() *v1.Secret { } func defaultRedisOptions() *component.RedisOptions { - tmpInsecure := insecureImportPolicy return &component.RedisOptions{ AmpRelease: product.ThreescaleRelease, BackendImageTag: product.ThreescaleRelease, @@ -145,7 +145,6 @@ func defaultRedisOptions() *component.RedisOptions { SystemImage: component.SystemRedisImageURL(), BackendRedisContainerResourceRequirements: component.DefaultBackendRedisContainerResourceRequirements(), SystemRedisContainerResourceRequirements: component.DefaultSystemRedisContainerResourceRequirements(), - InsecureImportPolicy: &tmpInsecure, SystemCommonLabels: testRedisSystemCommonLabels(), SystemRedisLabels: testRedisSystemRedisLabels(), SystemRedisPodTemplateLabels: testRedisSystemRedisPodTemplateLabels(), diff --git a/pkg/3scale/amp/operator/redis_reconciler.go b/pkg/3scale/amp/operator/redis_reconciler.go index 556cab850..6896e7baa 100644 --- a/pkg/3scale/amp/operator/redis_reconciler.go +++ b/pkg/3scale/amp/operator/redis_reconciler.go @@ -1,8 +1,7 @@ package operator import ( - appsv1 "github.com/openshift/api/apps/v1" - imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -13,16 +12,15 @@ import ( "github.com/3scale/3scale-operator/pkg/upgrade" ) -// RedisDependencyReconciler is a generic DependencyReconciler that reconciles +// RedisReconciler is a generic DependencyReconciler that reconciles // an internal Redis instance using the Redis options type RedisReconciler struct { *BaseAPIManagerLogicReconciler - DeploymentConfig func(redis *component.Redis) *appsv1.DeploymentConfig + Deployment func(redis *component.Redis) *k8sappsv1.Deployment Service func(redis *component.Redis) *corev1.Service ConfigMap func(redis *component.Redis) *corev1.ConfigMap PersistentVolumeClaim func(redis *component.Redis) *corev1.PersistentVolumeClaim - ImageStream func(redis *component.Redis) *imagev1.ImageStream Secret func(redis *component.Redis) *corev1.Secret } @@ -32,11 +30,10 @@ func NewSystemRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIMa return &RedisReconciler{ BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - DeploymentConfig: (*component.Redis).SystemDeploymentConfig, + Deployment: (*component.Redis).SystemDeployment, Service: (*component.Redis).SystemService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).SystemPVC, - ImageStream: (*component.Redis).SystemImageStream, Secret: (*component.Redis).SystemRedisSecret, } } @@ -45,11 +42,10 @@ func NewBackendRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIM return &RedisReconciler{ BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - DeploymentConfig: (*component.Redis).BackendDeploymentConfig, + Deployment: (*component.Redis).BackendDeployment, Service: (*component.Redis).BackendService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).BackendPVC, - ImageStream: (*component.Redis).BackendImageStream, Secret: (*component.Redis).BackendRedisSecret, } } @@ -60,25 +56,39 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - // 3scale 2.13 -> 2.14 - upgrade.Redis6CommandArgsEnv, + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(r.DeploymentConfig(redis), dcMutator) + redisDeployment := r.Deployment(redis) + err = r.ReconcileDeployment(redisDeployment, deploymentMutator) if err != nil { return reconcile.Result{}, err } - // redis Service - err = r.ReconcileService(r.Service(redis), reconcilers.CreateOnlyMutator) + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the redis PVCs are ReadWriteOnce and so they can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(redisDeployment.Name, r.apiManager.GetNamespace(), true, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + + // Service + err = r.ReconcileService(r.Service(redis), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -97,13 +107,7 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // IS - err = r.ReconcileImagestream(r.ImageStream(redis), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // Redis Secret + // Secret err = r.ReconcileSecret(r.Secret(redis), reconcilers.DefaultsOnlySecretMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/redis_reconciler_test.go b/pkg/3scale/amp/operator/redis_reconciler_test.go index 7ddadda56..013079c7c 100644 --- a/pkg/3scale/amp/operator/redis_reconciler_test.go +++ b/pkg/3scale/amp/operator/redis_reconciler_test.go @@ -9,6 +9,7 @@ import ( appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -21,7 +22,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -func TestRedisBackendDCReconcilerCreate(t *testing.T) { +func TestRedisBackendDeploymentReconcilerCreate(t *testing.T) { var ( appLabel = "someLabel" name = "example-apimanager" @@ -41,11 +42,10 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - ResourceRequirementsEnabled: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, }, }, } @@ -60,7 +60,13 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -89,19 +95,17 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { objName string obj client.Object }{ - {"backend-redis", &appsv1.DeploymentConfig{}}, + {"backend-redis", &k8sappsv1.Deployment{}}, {"backend-redis", &v1.Service{}}, {"redis-config", &v1.ConfigMap{}}, {"backend-redis-storage", &v1.PersistentVolumeClaim{}}, - {"backend-redis", &imagev1.ImageStream{}}, }}, {"systemRedis", NewSystemRedisDependencyReconciler, []struct { objName string obj client.Object }{ - {"system-redis", &appsv1.DeploymentConfig{}}, + {"system-redis", &k8sappsv1.Deployment{}}, {"system-redis-storage", &v1.PersistentVolumeClaim{}}, - {"system-redis", &imagev1.ImageStream{}}, {"system-redis", &v1.Service{}}, }}, } diff --git a/pkg/3scale/amp/operator/system_mysql_image_options_provider.go b/pkg/3scale/amp/operator/system_mysql_image_options_provider.go index e0c46f348..5e5bd2b07 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_options_provider.go +++ b/pkg/3scale/amp/operator/system_mysql_image_options_provider.go @@ -21,7 +21,6 @@ func NewSystemMysqlImageOptionsProvider(apimanager *appsv1alpha1.APIManager) *Sy func (s *SystemMysqlImageOptionsProvider) GetSystemMySQLImageOptions() (*component.SystemMySQLImageOptions, error) { s.mysqlImageOptions.AppLabel = *s.apimanager.Spec.AppLabel s.mysqlImageOptions.AmpRelease = product.ThreescaleRelease - s.mysqlImageOptions.InsecureImportPolicy = s.apimanager.Spec.ImageStreamTagImportInsecure s.mysqlImageOptions.Image = SystemMySQLImageURL() if s.apimanager.Spec.System.DatabaseSpec != nil && diff --git a/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go b/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go index f6d554da2..82fd48a3c 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go @@ -15,12 +15,10 @@ const ( ) func defaultSystemMySQLImageOptions() *component.SystemMySQLImageOptions { - tmpInsecure := insecureImportPolicy return &component.SystemMySQLImageOptions{ - AppLabel: appLabel, - AmpRelease: product.ThreescaleRelease, - InsecureImportPolicy: &tmpInsecure, - Image: component.SystemMySQLImageURL(), + AppLabel: appLabel, + AmpRelease: product.ThreescaleRelease, + Image: component.SystemMySQLImageURL(), } } diff --git a/pkg/3scale/amp/operator/system_mysql_image_reconciler.go b/pkg/3scale/amp/operator/system_mysql_image_reconciler.go index 1d9827f1a..a723c81dd 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_image_reconciler.go @@ -3,34 +3,12 @@ package operator import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/reconcilers" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemMySQLImageReconciler struct { *BaseAPIManagerLogicReconciler } -func NewSystemMySQLImageReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconciler) DependencyReconciler { - return &SystemMySQLImageReconciler{ - BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - } -} - -func (r *SystemMySQLImageReconciler) Reconcile() (reconcile.Result, error) { - systemMySQLImage, err := SystemMySQLImage(r.apiManager) - if err != nil { - return reconcile.Result{}, err - } - - err = r.ReconcileImagestream(systemMySQLImage.ImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil -} - func SystemMySQLImage(apimanager *appsv1alpha1.APIManager) (*component.SystemMySQLImage, error) { optsProvider := NewSystemMysqlImageOptionsProvider(apimanager) opts, err := optsProvider.GetSystemMySQLImageOptions() diff --git a/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go b/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go deleted file mode 100644 index da0a76e9f..000000000 --- a/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package operator - -import ( - "context" - "testing" - - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - imagev1 "github.com/openshift/api/image/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - fakeclientset "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -func TestSystemMySQLImageReconciler(t *testing.T) { - var ( - appLabel = "someLabel" - name = "example-apimanager" - namespace = "operator-unittest" - trueValue = true - imageURL = "mysql:test" - log = logf.Log.WithName("operator_test") - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - }, - System: &appsv1alpha1.SystemSpec{ - DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ - MySQL: &appsv1alpha1.SystemMySQLSpec{ - Image: &imageURL, - }, - }, - }, - }, - } - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Objects to track in the fake client. - objs := []runtime.Object{} - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemMySQLImageReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - obj := &imagev1.ImageStream{} - - namespacedName := types.NamespacedName{ - Name: "system-mysql", - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - // object must exist, that is all required to be tested - if err != nil { - t.Fatal(err) - } -} diff --git a/pkg/3scale/amp/operator/system_mysql_options_provider.go b/pkg/3scale/amp/operator/system_mysql_options_provider.go index 4872bed20..e71ca57d9 100644 --- a/pkg/3scale/amp/operator/system_mysql_options_provider.go +++ b/pkg/3scale/amp/operator/system_mysql_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" "strings" @@ -145,7 +146,7 @@ func (s *SystemMysqlOptionsProvider) setResourceRequirementsOptions() { s.mysqlOptions.ContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.DatabaseSpec != nil && @@ -209,7 +210,7 @@ func (s *SystemMysqlOptionsProvider) podTemplateLabels() map[string]string { } } - labels["deploymentConfig"] = "system-mysql" + labels[reconcilers.DeploymentLabelSelector] = "system-mysql" return labels } diff --git a/pkg/3scale/amp/operator/system_mysql_options_provider_test.go b/pkg/3scale/amp/operator/system_mysql_options_provider_test.go index 823265592..b02b66bb7 100644 --- a/pkg/3scale/amp/operator/system_mysql_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_mysql_options_provider_test.go @@ -10,6 +10,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -41,10 +42,10 @@ func testSystemMysqlDeploymentLabels() map[string]string { func testSystemMysqlPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "mysql", - "deploymentConfig": "system-mysql", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "mysql", + reconcilers.DeploymentLabelSelector: "system-mysql", } addExpectedMeteringLabels(labels, "system-mysql", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler.go b/pkg/3scale/amp/operator/system_mysql_reconciler.go index 389e36e95..87e8c0317 100644 --- a/pkg/3scale/amp/operator/system_mysql_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_reconciler.go @@ -4,6 +4,8 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -19,29 +21,49 @@ func NewSystemMySQLReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogic } func (r *SystemMySQLReconciler) Reconcile() (reconcile.Result, error) { + systemMySQLImage, err := SystemMySQLImage(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + systemMySQL, err := SystemMySQL(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // DC - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // MySQL Deployment + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(systemMySQL.DeploymentConfig(), dcMutator) + err = r.ReconcileDeployment(systemMySQL.Deployment(systemMySQLImage.Options.Image), deploymentMutator) + if err != nil { + return reconcile.Result{}, err + } + + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the mysql-storage PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMySQLDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) if err != nil { return reconcile.Result{}, err } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } // Service - err = r.ReconcileService(systemMySQL.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(systemMySQL.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -58,7 +80,7 @@ func (r *SystemMySQLReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // PCV + // PVC err = r.ReconcilePersistentVolumeClaim(systemMySQL.PersistentVolumeClaim(), reconcilers.CreateOnlyMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler_test.go b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go new file mode 100644 index 000000000..8c89b96ad --- /dev/null +++ b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go @@ -0,0 +1,119 @@ +package operator + +import ( + "context" + "testing" + + appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + appsv1 "github.com/openshift/api/apps/v1" + imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakeclientset "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +func TestSystemMySQLReconcilerCreate(t *testing.T) { + var ( + appLabel = "someLabel" + name = "example-apimanager" + namespace = "operator-unittest" + trueValue = true + imageURL = "mysql:test" + wildcardDomain = "test.3scale.net" + tenantName = "someTenant" + log = logf.Log.WithName("operator_test") + ) + + ctx := context.TODO() + + apimanager := &appsv1alpha1.APIManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1alpha1.APIManagerSpec{ + APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + }, + System: &appsv1alpha1.SystemSpec{ + DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ + MySQL: &appsv1alpha1.SystemMySQLSpec{ + Image: &imageURL, + }, + }, + }, + }, + } + s := scheme.Scheme + s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) + err := imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + + // Objects to track in the fake client. + objs := []runtime.Object{} + + // Create a fake client to mock API calls. + cl := fake.NewFakeClient(objs...) + clientAPIReader := fake.NewFakeClient(objs...) + clientset := fakeclientset.NewSimpleClientset() + recorder := record.NewFakeRecorder(10000) + + baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) + baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) + + reconciler := NewSystemMySQLReconciler(baseAPIManagerLogicReconciler) + _, err = reconciler.Reconcile() + if err != nil { + t.Fatal(err) + } + + cases := []struct { + testName string + objName string + obj client.Object + }{ + {"systemMySQL_Deployment", "system-mysql", &k8sappsv1.Deployment{}}, + {"systemMySQL_Service", "system-mysql", &v1.Service{}}, + {"systemMySQL_Main_CM", "mysql-main-conf", &v1.ConfigMap{}}, + {"systemMySQL_Extra_CM", "mysql-extra-conf", &v1.ConfigMap{}}, + {"systemMySQL_PVC", "mysql-storage", &v1.PersistentVolumeClaim{}}, + {"systemDatabaseSecret", component.SystemSecretSystemDatabaseSecretName, &v1.Secret{}}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + obj := tc.obj + namespacedName := types.NamespacedName{ + Name: tc.objName, + Namespace: namespace, + } + err = cl.Get(context.TODO(), namespacedName, obj) + // object must exist, that is all required to be tested + if err != nil { + subT.Errorf("error fetching object %s: %v", tc.objName, err) + } + }) + } +} diff --git a/pkg/3scale/amp/operator/system_options_provider.go b/pkg/3scale/amp/operator/system_options_provider.go index 54bf71481..d02bfd70e 100644 --- a/pkg/3scale/amp/operator/system_options_provider.go +++ b/pkg/3scale/amp/operator/system_options_provider.go @@ -12,6 +12,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type SystemOptionsProvider struct { @@ -373,7 +374,7 @@ func (s *SystemOptionsProvider) setResourceRequirementsOptions() { s.options.SidekiqContainerResourceRequirements = &v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.AppSpec.MasterContainerResources != nil { @@ -546,7 +547,7 @@ func (s *SystemOptionsProvider) appPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-app" + labels[reconcilers.DeploymentLabelSelector] = "system-app" return labels } @@ -568,7 +569,7 @@ func (s *SystemOptionsProvider) sidekiqPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-sidekiq" + labels[reconcilers.DeploymentLabelSelector] = "system-sidekiq" return labels } diff --git a/pkg/3scale/amp/operator/system_options_provider_test.go b/pkg/3scale/amp/operator/system_options_provider_test.go index 5f3ca537d..c79a094c7 100644 --- a/pkg/3scale/amp/operator/system_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_options_provider_test.go @@ -9,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -39,10 +40,10 @@ func testSystemCommonAppLabels() map[string]string { func testSystemAppPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "app", - "deploymentConfig": "system-app", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "app", + reconcilers.DeploymentLabelSelector: "system-app", } addExpectedMeteringLabels(labels, "system-app", helper.ApplicationType) @@ -59,10 +60,10 @@ func testSystemCommonSidekiqLabels() map[string]string { func testSystemSidekiqPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "sidekiq", - "deploymentConfig": "system-sidekiq", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "sidekiq", + reconcilers.DeploymentLabelSelector: "system-sidekiq", } addExpectedMeteringLabels(labels, "system-sidekiq", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go b/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go index 322d2c382..9f877af15 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go @@ -21,7 +21,6 @@ func NewSystemPostgreSQLImageOptionsProvider(apimanager *appsv1alpha1.APIManager func (s *SystemPostgreSQLImageOptionsProvider) GetSystemPostgreSQLImageOptions() (*component.SystemPostgreSQLImageOptions, error) { s.options.AppLabel = *s.apimanager.Spec.AppLabel s.options.AmpRelease = product.ThreescaleRelease - s.options.InsecureImportPolicy = s.apimanager.Spec.ImageStreamTagImportInsecure s.options.Image = SystemPostgreSQLImageURL() if s.apimanager.Spec.System.DatabaseSpec != nil && diff --git a/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go b/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go index 1bed2a1ca..e839340dd 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go @@ -15,12 +15,10 @@ const ( ) func defaultSystemPostgreSQLImageOptions() *component.SystemPostgreSQLImageOptions { - tmpInsecure := insecureImportPolicy return &component.SystemPostgreSQLImageOptions{ - AppLabel: appLabel, - AmpRelease: product.ThreescaleRelease, - InsecureImportPolicy: &tmpInsecure, - Image: component.SystemPostgreSQLImageURL(), + AppLabel: appLabel, + AmpRelease: product.ThreescaleRelease, + Image: component.SystemPostgreSQLImageURL(), } } diff --git a/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go b/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go index 127ad2ef5..c0e439dc3 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go @@ -3,35 +3,12 @@ package operator import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemPostgreSQLImageReconciler struct { *BaseAPIManagerLogicReconciler } -func NewSystemPostgreSQLImageReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconciler) DependencyReconciler { - return &SystemPostgreSQLImageReconciler{ - BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - } -} - -func (r *SystemPostgreSQLImageReconciler) Reconcile() (reconcile.Result, error) { - systemPostgreSQLImage, err := SystemPostgreSQLImage(r.apiManager) - if err != nil { - return reconcile.Result{}, err - } - - err = r.ReconcileImagestream(systemPostgreSQLImage.ImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil -} - func SystemPostgreSQLImage(apimanager *appsv1alpha1.APIManager) (*component.SystemPostgreSQLImage, error) { optsProvider := NewSystemPostgreSQLImageOptionsProvider(apimanager) opts, err := optsProvider.GetSystemPostgreSQLImageOptions() diff --git a/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go b/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go deleted file mode 100644 index eb9155c43..000000000 --- a/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package operator - -import ( - "context" - "testing" - - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - imagev1 "github.com/openshift/api/image/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - fakeclientset "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -func TestSystemPostgreSQLImageReconcilerCreate(t *testing.T) { - var ( - appLabel = "someLabel" - name = "example-apimanager" - namespace = "operator-unittest" - trueValue = true - imageURL = "postgresql:test" - log = logf.Log.WithName("operator_test") - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - }, - System: &appsv1alpha1.SystemSpec{ - DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ - PostgreSQL: &appsv1alpha1.SystemPostgreSQLSpec{ - Image: &imageURL, - }, - }, - }, - }, - } - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Objects to track in the fake client. - objs := []runtime.Object{} - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemPostgreSQLImageReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - obj := &imagev1.ImageStream{} - - namespacedName := types.NamespacedName{ - Name: "system-postgresql", - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - // object must exist, that is all required to be tested - if err != nil { - t.Fatal(err) - } -} diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider.go b/pkg/3scale/amp/operator/system_postgresql_options_provider.go index 91c71aff3..764069657 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider.go @@ -9,6 +9,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -132,7 +134,7 @@ func (s *SystemPostgresqlOptionsProvider) setResourceRequirementsOptions() { s.options.ContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.DatabaseSpec != nil && @@ -194,7 +196,7 @@ func (s *SystemPostgresqlOptionsProvider) podTemplateLabels() map[string]string } } - labels["deploymentConfig"] = "system-postgresql" + labels[reconcilers.DeploymentLabelSelector] = "system-postgresql" return labels } diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go index 877ddb72e..72b21d508 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go @@ -10,6 +10,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" @@ -41,10 +43,10 @@ func testSystemPostgreSQLDeploymentLabels() map[string]string { func testSystemPostgreSQLPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "postgresql", - "deploymentConfig": "system-postgresql", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "postgresql", + reconcilers.DeploymentLabelSelector: "system-postgresql", } addExpectedMeteringLabels(labels, "system-postgresql", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler.go b/pkg/3scale/amp/operator/system_postgresql_reconciler.go index 8dbcc0e83..8d91316d1 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler.go @@ -4,6 +4,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -20,29 +21,49 @@ func NewSystemPostgreSQLReconciler(baseAPIManagerLogicReconciler *BaseAPIManager } func (r *SystemPostgreSQLReconciler) Reconcile() (reconcile.Result, error) { + systemPostgreSQLImage, err := SystemPostgreSQLImage(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + systemPostgreSQL, err := SystemPostgreSQL(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // DC - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // PostgreSQL Deployment + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(systemPostgreSQL.DeploymentConfig(), dcMutator) + err = r.ReconcileDeployment(systemPostgreSQL.Deployment(systemPostgreSQLImage.Options.Image), deploymentMutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the postgresql-data PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemPostgreSQLDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Service - err = r.ReconcileService(systemPostgreSQL.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(systemPostgreSQL.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go index 59f2e331b..6e9678633 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go @@ -10,6 +10,7 @@ import ( appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,11 +44,10 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - ResourceRequirementsEnabled: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, }, System: &appsv1alpha1.SystemSpec{ DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ @@ -60,7 +60,13 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { } s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) + err := imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -88,7 +94,7 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { objName string obj client.Object }{ - {"systemPostgreSQL_DC", "system-postgresql", &appsv1.DeploymentConfig{}}, + {"systemPostgreSQL_Deployment", "system-postgresql", &k8sappsv1.Deployment{}}, {"systemPostgreSQL_Service", "system-postgresql", &v1.Service{}}, {"systemPostgreSQL_PVC", "postgresql-data", &v1.PersistentVolumeClaim{}}, {"systemDatabaseSecret", component.SystemSecretSystemDatabaseSecretName, &v1.Secret{}}, diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index ab5583164..30a5793d4 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -1,7 +1,17 @@ package operator import ( + "context" "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + k8sappsv1 "k8s.io/api/apps/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" @@ -9,13 +19,6 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/pkg/upgrade" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" - "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemReconciler struct { @@ -42,88 +45,46 @@ func (r *SystemReconciler) reconcileFileStorage(system *component.System) error } func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { - system, err := System(r.apiManager, r.Client()) + ampImages, err := AmpImages(r.apiManager) if err != nil { return reconcile.Result{}, err } - err = r.reconcileFileStorage(system) + system, err := System(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // Provider Service - err = r.ReconcileService(system.ProviderService(), reconcilers.CreateOnlyMutator) + err = r.reconcileFileStorage(system) if err != nil { return reconcile.Result{}, err } - // Master Service - err = r.ReconcileService(system.MasterService(), reconcilers.CreateOnlyMutator) - if err != nil { - return reconcile.Result{}, err + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, } - // Developer Service - err = r.ReconcileService(system.DeveloperService(), reconcilers.CreateOnlyMutator) + // Provider Service + err = r.ReconcileService(system.ProviderService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // Memcached Service - err = r.ReconcileService(system.MemcachedService(), reconcilers.CreateOnlyMutator) + // Master Service + err = r.ReconcileService(system.MasterService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // SystemApp DC - systemAppDCMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - r.systemAppDCResourceMutator, - reconcilers.DeploymentConfigRemoveDuplicateEnvVarMutator, - // 3scale 2.13 -> 2.14 - upgrade.SphinxAddressReference, - // 3scale 2.13 -> 2.14 - upgrade.SystemBackendUrls, - } - - if r.apiManager.Spec.System.AppSpec.Replicas != nil { - systemAppDCMutators = append(systemAppDCMutators, reconcilers.DeploymentConfigReplicasMutator) - } - - err = r.ReconcileDeploymentConfig(system.AppDeploymentConfig(), reconcilers.DeploymentConfigMutator(systemAppDCMutators...)) + // Developer Service + err = r.ReconcileService(system.DeveloperService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // Sidekiq DC - sidekiqDCMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigRemoveDuplicateEnvVarMutator, - reconcilers.DeploymentConfigArgsMutator, - reconcilers.DeploymentConfigProbesMutator, - // 3scale 2.13 -> 2.14 - upgrade.SphinxAddressReference, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - } - - if r.apiManager.Spec.System.SidekiqSpec.Replicas != nil { - sidekiqDCMutators = append(sidekiqDCMutators, reconcilers.DeploymentConfigReplicasMutator) - } - - err = r.ReconcileDeploymentConfig(system.SidekiqDeploymentConfig(), reconcilers.DeploymentConfigMutator(sidekiqDCMutators...)) + // Memcached Service + err = r.ReconcileService(system.MemcachedService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -182,6 +143,134 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } + // Used to synchronize rollout of system Deployments + systemComponentsReady := true + + // If the system-app Deployment generation has changed, delete the PreHook/PostHook Jobs so they can be recreated + generationChanged, err := helper.HasAppGenerationChanged(component.SystemAppPreHookJobName, component.SystemAppDeploymentName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if generationChanged { + err = helper.DeleteJob(component.SystemAppPreHookJobName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + } + generationChanged, err = helper.HasAppGenerationChanged(component.SystemAppPostHookJobName, component.SystemAppDeploymentName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if generationChanged { + err = helper.DeleteJob(component.SystemAppPostHookJobName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + } + + // Used to synchronize the system-app Deployment with the PreHook/PostHook Jobs + currentAppDeploymentGeneration, err := getSystemAppDeploymentGeneration(r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + + // SystemApp PreHook Job + preHookJob := system.AppPreHookJob(ampImages.Options.SystemImage, currentAppDeploymentGeneration) + err = r.ReconcileJob(preHookJob, reconcilers.CreateOnlyMutator) + if err != nil { + return reconcile.Result{}, err + } + + // Block reconciling system-app Deployment until PreHook Job has completed + if !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { + systemComponentsReady = false + } + + if systemComponentsReady { + // SystemApp Deployment + systemAppDeploymentMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + r.systemAppDeploymentResourceMutator, + reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + reconcilers.DeploymentPodContainerImageMutator, + } + if r.apiManager.Spec.System.AppSpec.Replicas != nil { + systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) + } + err = r.ReconcileDeployment(system.AppDeployment(ampImages.Options.SystemImage), reconcilers.DeploymentMutator(systemAppDeploymentMutators...)) + if err != nil { + return reconcile.Result{}, err + } + } + + // Block reconciling PostHook Job unless BOTH the PreHook Job has completed and the system-app Deployment is ready and not in the process of updating + deployment := &k8sappsv1.Deployment{} + err = r.Client().Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: r.apiManager.GetNamespace(), + Name: component.SystemAppDeploymentName, + }, deployment) + if err != nil && !k8serr.IsNotFound(err) { + return reconcile.Result{}, err + } + if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) || helper.IsDeploymentProgressing(deployment) || !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { + systemComponentsReady = false + } + + // SystemApp PostHook Job + if systemComponentsReady { + err = r.ReconcileJob(system.AppPostHookJob(ampImages.Options.SystemImage, currentAppDeploymentGeneration), reconcilers.CreateOnlyMutator) + if err != nil { + return reconcile.Result{}, err + } + } + + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemAppDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + systemComponentsReady = false + } + + // Sidekiq Deployment + sidekiqDeploymentMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + reconcilers.DeploymentArgsMutator, + reconcilers.DeploymentProbesMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, + } + if r.apiManager.Spec.System.SidekiqSpec.Replicas != nil { + sidekiqDeploymentMutators = append(sidekiqDeploymentMutators, reconcilers.DeploymentReplicasMutator) + } + + err = r.ReconcileDeployment(system.SidekiqDeployment(ampImages.Options.SystemImage), reconcilers.DeploymentMutator(sidekiqDeploymentMutators...)) + if err != nil { + return reconcile.Result{}, err + } + + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.SystemSidekiqName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + systemComponentsReady = false + } + // SystemApp PDB err = r.ReconcilePodDisruptionBudget(system.AppPodDisruptionBudget(), reconcilers.GenericPDBMutator) if err != nil { @@ -224,16 +313,39 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } + // Requeue if any of the system-app Deployment's components aren't ready + if !systemComponentsReady { + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } + return reconcile.Result{}, nil } -func (r *SystemReconciler) systemAppDCResourceMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func getSystemAppDeploymentGeneration(namespace string, client k8sclient.Client) (int64, error) { + deployment := &k8sappsv1.Deployment{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: namespace, + Name: component.SystemAppDeploymentName, + }, deployment) + + // Return error if can't get Deployment + if err != nil && !k8serr.IsNotFound(err) { + return 0, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) + } + + // Return 1 if the Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + return 1, nil + } + + return deployment.Generation, nil +} + +func (r *SystemReconciler) systemAppDeploymentResourceMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { desiredName := common.ObjectInfo(desired) update := false - // // Check containers - // if len(desired.Spec.Template.Spec.Containers) != 3 { return false, fmt.Errorf(fmt.Sprintf("%s desired spec.template.spec.containers length changed to '%d', should be 3", desiredName, len(desired.Spec.Template.Spec.Containers))) } @@ -244,10 +356,7 @@ func (r *SystemReconciler) systemAppDCResourceMutator(desired, existing *appsv1. update = true } - // // Check containers resource requirements - // - for idx := 0; idx < 3; idx++ { if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[idx].Resources, &desired.Spec.Template.Spec.Containers[idx].Resources) { diff := cmp.Diff(existing.Spec.Template.Spec.Containers[idx].Resources, desired.Spec.Template.Spec.Containers[idx].Resources, cmpopts.IgnoreUnexported(resource.Quantity{})) @@ -260,7 +369,7 @@ func (r *SystemReconciler) systemAppDCResourceMutator(desired, existing *appsv1. return update, nil } -func System(cr *appsv1alpha1.APIManager, client client.Client) (*component.System, error) { +func System(cr *appsv1alpha1.APIManager, client k8sclient.Client) (*component.System, error) { optsProvider := NewSystemOptionsProvider(cr, cr.Namespace, client) opts, err := optsProvider.GetSystemOptions() if err != nil { diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index 3c6d515a6..a24eaa425 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -14,6 +14,8 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,19 +37,42 @@ func TestSystemReconcilerCreate(t *testing.T) { ctx := context.TODO() apimanager := basicApimanagerSpecTestSystemOptions() + appPreHookJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: apimanager.Namespace}, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: SystemImageURL(), + }, + }, + }, + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: v1.ConditionTrue, + }, + }, + }, + } + // Objects to track in the fake client. - objs := []runtime.Object{apimanager} + objs := []runtime.Object{apimanager, appPreHookJob} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -57,7 +82,13 @@ func TestSystemReconcilerCreate(t *testing.T) { if err := grafanav1alpha1.AddToScheme(s); err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -86,8 +117,8 @@ func TestSystemReconcilerCreate(t *testing.T) { {"systemMasterService", "system-master", &v1.Service{}}, {"systemDeveloperService", "system-developer", &v1.Service{}}, {"systemMemcacheService", "system-memcache", &v1.Service{}}, - {"systemAppDC", "system-app", &appsv1.DeploymentConfig{}}, - {"systemSideKiqDC", "system-sidekiq", &appsv1.DeploymentConfig{}}, + {"systemAppDeployment", "system-app", &k8sappsv1.Deployment{}}, + {"systemSideKiqDeployment", "system-sidekiq", &k8sappsv1.Deployment{}}, {"systemCM", "system", &v1.ConfigMap{}}, {"systemEnvironmentCM", "system-environment", &v1.ConfigMap{}}, {"systemSMTPSecret", "system-smtp", &v1.Secret{}}, @@ -126,6 +157,30 @@ func TestReplicaSystemReconciler(t *testing.T) { oneValue64 int64 = 1 twoValue int32 = 2 ) + + appPreHookJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: namespace}, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: SystemImageURL(), + }, + }, + }, + }, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: v1.ConditionTrue, + }, + }, + }, + } + ctx := context.TODO() s := scheme.Scheme @@ -133,12 +188,17 @@ func TestReplicaSystemReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } - if err := configv1.AddToScheme(s); err != nil { + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -157,7 +217,7 @@ func TestReplicaSystemReconciler(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{tc.apimanager} + objs := []runtime.Object{tc.apimanager, appPreHookJob} // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -172,22 +232,23 @@ func TestReplicaSystemReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + deployment.Generation = oneValue64 + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -196,13 +257,13 @@ func TestReplicaSystemReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } @@ -227,11 +288,10 @@ func testSystemAPIManagerCreator(appReplicas, sidekiqReplicas *int64) *appsv1alp Spec: appsv1alpha1.APIManagerSpec{ Apicast: &appsv1alpha1.ApicastSpec{RegistryURL: &tmpApicastRegistryURL}, APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, System: &appsv1alpha1.SystemSpec{ AppSpec: &appsv1alpha1.SystemAppSpec{Replicas: appReplicas}, diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider.go b/pkg/3scale/amp/operator/system_searchd_options_provider.go index 0f5407f33..e5a11ef89 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider.go @@ -10,6 +10,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type SystemSearchdOptionsProvider struct { @@ -48,7 +49,7 @@ func (s *SystemSearchdOptionsProvider) setResourceRequirementsOptions() { if *s.apimanager.Spec.ResourceRequirementsEnabled { s.options.ContainerResourceRequirements = component.DefaultSearchdContainerResourceRequirements() } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.SearchdSpec.Resources != nil { @@ -85,7 +86,7 @@ func (s *SystemSearchdOptionsProvider) podTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-searchd" + labels[reconcilers.DeploymentLabelSelector] = "system-searchd" return labels } diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go index 31802c929..8a5af71ed 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go @@ -9,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -49,10 +50,10 @@ func testSystemSearchdLabels() map[string]string { func testSystemSearchdPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "searchd", - "deploymentConfig": "system-searchd", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "searchd", + reconcilers.DeploymentLabelSelector: "system-searchd", } addExpectedMeteringLabels(labels, "system-searchd", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index 4b41439cd..04b8837a3 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -1,15 +1,12 @@ package operator import ( - appsv1 "github.com/openshift/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" ) type SystemSearchdReconciler struct { @@ -23,19 +20,23 @@ func NewSystemSearchdReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLog } func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { - searchd, err := SystemSearchd(r.apiManager) + ampImages, err := AmpImages(r.apiManager) if err != nil { return reconcile.Result{}, err } - // 3scale 2.13 -> 2.14 - err = r.upgradeFromSphinx() + searchd, err := SystemSearchd(r.apiManager) if err != nil { return reconcile.Result{}, err } + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Service - err = r.ReconcileService(searchd.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(searchd.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -46,52 +47,36 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // DC - searchdDCmutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigStrategyMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - reconcilers.DeploymentConfigProbesMutator, - reconcilers.DeploymentConfigArgsMutator, + // Deployment + searchdDeploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentStrategyMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentProbesMutator, + reconcilers.DeploymentArgsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(searchd.DeploymentConfig(), searchdDCmutator) + err = r.ReconcileDeployment(searchd.Deployment(ampImages.Options.SystemSearchdImage), searchdDeploymentMutator) if err != nil { return reconcile.Result{}, err } - return reconcile.Result{}, nil -} - -func (r *SystemSearchdReconciler) upgradeFromSphinx() error { - // The upgrade procedure is simply based on: - // * Delete "old" DC called system-sphinx (if found) - // * The regular reconciling logic will create a new DC called system-searchd - - oldService := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "system-sphinx", Namespace: r.apiManager.Namespace}, - } - common.TagObjectToDelete(oldService) - err := r.ReconcileResource(&v1.Service{}, oldService, reconcilers.CreateOnlyMutator) + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the system-searchd PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemSearchdDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) if err != nil { - return err - } - - oldDC := &appsv1.DeploymentConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "system-sphinx", Namespace: r.apiManager.Namespace}, + return reconcile.Result{}, err } - common.TagObjectToDelete(oldDC) - err = r.ReconcileResource(&appsv1.DeploymentConfig{}, oldDC, reconcilers.CreateOnlyMutator) - if err != nil { - return err + if !isMigrated { + return reconcile.Result{Requeue: true}, nil } - return nil + return reconcile.Result{}, nil } func SystemSearchd(cr *appsv1alpha1.APIManager) (*component.SystemSearchd, error) { diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go index 17c0dfeca..3e85db946 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go @@ -9,15 +9,13 @@ import ( "github.com/3scale/3scale-operator/pkg/reconcilers" appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" fakeclientset "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -35,7 +33,13 @@ func TestSystemSearchdReconciler(t *testing.T) { objs := []runtime.Object{apimanager} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -62,7 +66,7 @@ func TestSystemSearchdReconciler(t *testing.T) { }{ {"PVC", component.SystemSearchdPVCName, &v1.PersistentVolumeClaim{}}, {"Service", component.SystemSearchdServiceName, &v1.Service{}}, - {"DC", component.SystemSearchdDeploymentName, &appsv1.DeploymentConfig{}}, + {"Deployment", component.SystemSearchdDeploymentName, &k8sappsv1.Deployment{}}, } for _, tc := range cases { @@ -80,128 +84,3 @@ func TestSystemSearchdReconciler(t *testing.T) { }) } } - -func TestUpgradeFromSphinx(t *testing.T) { - var ( - log = logf.Log.WithName("upgrade_test") - ctx = context.TODO() - s = scheme.Scheme - apimanager = testSearchdBasicApimanager() - ) - - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - cases := []struct { - testName string - initialState []runtime.Object - }{ - {"Start from scratch", nil}, - {"only sphinx dc/service", []runtime.Object{ - testOldSphinxDC(namespace), testOldSphinxSvc(namespace), - }}, - {"only searchd dc/service", []runtime.Object{ - testsearchdDC(namespace), testsearchdSvc(namespace), - }}, - {"both sphinx and searchd dc/service", []runtime.Object{ - testOldSphinxDC(namespace), testOldSphinxSvc(namespace), - testsearchdDC(namespace), testsearchdSvc(namespace), - }}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{} - cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() - clientAPIReader := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemSearchdReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - subT.Fatal(err) - } - - // Sphinx Service should not be there - sphinxServiceKey := client.ObjectKey{Name: "system-sphinx", Namespace: namespace} - err = cl.Get(ctx, sphinxServiceKey, &v1.Service{}) - if err == nil { - subT.Fatalf("reading an object expected to be deleted: %s", sphinxServiceKey) - } - if !errors.IsNotFound(err) { - subT.Fatalf("unexpected error reading object %s: %v", sphinxServiceKey, err) - } - - // Sphinx DC should not be there - sphinxDCKey := client.ObjectKey{Name: "system-sphinx", Namespace: namespace} - err = cl.Get(ctx, sphinxDCKey, &appsv1.DeploymentConfig{}) - if err == nil { - subT.Fatalf("reading an object expected to be deleted: %s", sphinxDCKey) - } - if !errors.IsNotFound(err) { - subT.Fatalf("unexpected error reading object %s: %v", sphinxDCKey, err) - } - - opts := &component.SystemSearchdOptions{} - searchd := component.NewSystemSearchd(opts) - // Searchd Service should be there - searchdServiceKey := client.ObjectKey{Name: searchd.Service().Name, Namespace: namespace} - err = cl.Get(ctx, searchdServiceKey, &v1.Service{}) - if err != nil { - subT.Fatalf("error fetching object %s: %v", searchdServiceKey, err) - } - // Searchd DC should be there - searchDCKey := client.ObjectKey{Name: searchd.DeploymentConfig().Name, Namespace: namespace} - err = cl.Get(ctx, searchDCKey, &appsv1.DeploymentConfig{}) - if err != nil { - subT.Fatalf("error fetching object %s: %v", searchDCKey, err) - } - }) - } -} - -func testOldSphinxDC(namespace string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "system-sphinx", - Namespace: namespace, - }, - } -} - -func testOldSphinxSvc(namespace string) *v1.Service { - return &v1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "system-sphinx", - Namespace: namespace, - }, - } -} - -func testsearchdSvc(namespace string) *v1.Service { - opts := &component.SystemSearchdOptions{} - svc := component.NewSystemSearchd(opts).Service() - svc.Namespace = namespace - return svc -} - -func testsearchdDC(namespace string) *appsv1.DeploymentConfig { - opts := &component.SystemSearchdOptions{} - dc := component.NewSystemSearchd(opts).DeploymentConfig() - dc.Namespace = namespace - return dc -} diff --git a/pkg/3scale/amp/operator/zync_options_provider.go b/pkg/3scale/amp/operator/zync_options_provider.go index 035c384a3..3e9395a49 100644 --- a/pkg/3scale/amp/operator/zync_options_provider.go +++ b/pkg/3scale/amp/operator/zync_options_provider.go @@ -8,6 +8,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -178,7 +179,7 @@ func (z *ZyncOptionsProvider) setResourceRequirementsOptions() { z.zyncOptions.DatabaseContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if z.apimanager.Spec.Zync.AppSpec.Resources != nil { @@ -249,7 +250,7 @@ func (z *ZyncOptionsProvider) zyncPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "zync" + labels[reconcilers.DeploymentLabelSelector] = "zync" return labels } @@ -265,7 +266,7 @@ func (z *ZyncOptionsProvider) zyncQuePodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "zync-que" + labels[reconcilers.DeploymentLabelSelector] = "zync-que" return labels } @@ -281,7 +282,7 @@ func (z *ZyncOptionsProvider) zyncDatabasePodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "zync-database" + labels[reconcilers.DeploymentLabelSelector] = "zync-database" return labels } diff --git a/pkg/3scale/amp/operator/zync_options_provider_test.go b/pkg/3scale/amp/operator/zync_options_provider_test.go index 3d45b7cd6..d1a375476 100644 --- a/pkg/3scale/amp/operator/zync_options_provider_test.go +++ b/pkg/3scale/amp/operator/zync_options_provider_test.go @@ -16,6 +16,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( @@ -61,10 +62,10 @@ func testZyncDatabaseCommonLabels() map[string]string { func testZyncPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "zync", - "deploymentConfig": "zync", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "zync", + reconcilers.DeploymentLabelSelector: "zync", } addExpectedMeteringLabels(labels, "zync", helper.ApplicationType) @@ -73,10 +74,10 @@ func testZyncPodTemplateLabels() map[string]string { func testZyncQuePodTemplateCommonLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "zync-que", - "deploymentConfig": "zync-que", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "zync-que", + reconcilers.DeploymentLabelSelector: "zync-que", } addExpectedMeteringLabels(labels, "zync-que", helper.ApplicationType) @@ -85,10 +86,10 @@ func testZyncQuePodTemplateCommonLabels() map[string]string { func testZyncDatabasePodTemplateCommonLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "database", - "deploymentConfig": "zync-database", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "database", + reconcilers.DeploymentLabelSelector: "zync-database", } addExpectedMeteringLabels(labels, "zync-database", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index c650c2aa5..eaabbed81 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -21,13 +22,18 @@ func NewZyncReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconci } func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + zync, err := Zync(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } // Zync Que Role - err = r.ReconcileRole(zync.QueRole(), reconcilers.CreateOnlyMutator) + err = r.ReconcileRole(zync.QueRole(), reconcilers.RoleRuleMutator) if err != nil { return reconcile.Result{}, err } @@ -44,69 +50,102 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Zync DC - zyncMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Zync Deployment + zyncMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, } if r.apiManager.Spec.Zync.AppSpec.Replicas != nil { - zyncMutators = append(zyncMutators, reconcilers.DeploymentConfigReplicasMutator) + zyncMutators = append(zyncMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(zync.DeploymentConfig(), reconcilers.DeploymentConfigMutator(zyncMutators...)) + err = r.ReconcileDeployment(zync.Deployment(ampImages.Options.ZyncImage), reconcilers.DeploymentMutator(zyncMutators...)) if err != nil { return reconcile.Result{}, err } - // Zync Que DC - zyncQueMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + // Zync Que Deployment + zyncQueMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, } if r.apiManager.Spec.Zync.QueSpec.Replicas != nil { - zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentConfigReplicasMutator) + zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(zync.QueDeploymentConfig(), reconcilers.DeploymentConfigMutator(zyncQueMutators...)) + err = r.ReconcileDeployment(zync.QueDeployment(ampImages.Options.ZyncImage), reconcilers.DeploymentMutator(zyncQueMutators...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), r.BaseReconciler.Scheme()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Zync Service - err = r.ReconcileService(zync.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(zync.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } if !r.apiManager.IsExternal(appsv1alpha1.ZyncDatabase) { - // Zync DB DC - zyncDBDCMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Zync DB Deployment + zyncDBDMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) - err = r.ReconcileDeploymentConfig(zync.DatabaseDeploymentConfig(), zyncDBDCMutator) + err = r.ReconcileDeployment(zync.DatabaseDeployment(ampImages.Options.ZyncDatabasePostgreSQLImage), zyncDBDMutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + // Zync DB Service - err = r.ReconcileService(zync.DatabaseService(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(zync.DatabaseService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/zync_reconciler_test.go b/pkg/3scale/amp/operator/zync_reconciler_test.go index ef8259f72..936ea6821 100644 --- a/pkg/3scale/amp/operator/zync_reconciler_test.go +++ b/pkg/3scale/amp/operator/zync_reconciler_test.go @@ -4,8 +4,6 @@ import ( "context" "testing" - policyv1 "k8s.io/api/policy/v1" - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" @@ -16,7 +14,9 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,11 +51,10 @@ func TestNewZyncReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: &oneValue}, @@ -68,15 +67,19 @@ func TestNewZyncReconciler(t *testing.T) { objs := []runtime.Object{apimanager} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + err = configv1.Install(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -87,6 +90,12 @@ func TestNewZyncReconciler(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -110,9 +119,9 @@ func TestNewZyncReconciler(t *testing.T) { {"queRole", "zync-que-role", &rbacv1.Role{}}, {"queServiceAccount", "zync-que-sa", &v1.ServiceAccount{}}, {"queRoleBinding", "zync-que-rolebinding", &rbacv1.RoleBinding{}}, - {"zyncDC", "zync", &appsv1.DeploymentConfig{}}, - {"zyncQueDC", "zync-que", &appsv1.DeploymentConfig{}}, - {"zyncDatabaseDC", "zync-database", &appsv1.DeploymentConfig{}}, + {"zyncDeployment", "zync", &k8sappsv1.Deployment{}}, + {"zyncQueDeployment", "zync-que", &k8sappsv1.Deployment{}}, + {"zyncDatabaseDeployment", "zync-database", &k8sappsv1.Deployment{}}, {"zyncService", "zync", &v1.Service{}}, {"zyncDatabaseService", "zync-database", &v1.Service{}}, {"zyncSecret", component.ZyncSecretName, &v1.Secret{}}, @@ -157,11 +166,10 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: &oneValue}, @@ -182,15 +190,19 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { objs := []runtime.Object{apimanager, zyncExternalDatabaseSecret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = configv1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -201,6 +213,12 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -225,9 +243,9 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { {"queRole", "zync-que-role", &rbacv1.Role{}, true}, {"queServiceAccount", "zync-que-sa", &v1.ServiceAccount{}, true}, {"queRoleBinding", "zync-que-rolebinding", &rbacv1.RoleBinding{}, true}, - {"zyncDC", "zync", &appsv1.DeploymentConfig{}, true}, - {"zyncQueDC", "zync-que", &appsv1.DeploymentConfig{}, true}, - {"zyncDatabaseDC", "zync-database", &appsv1.DeploymentConfig{}, false}, + {"zyncDeployment", "zync", &k8sappsv1.Deployment{}, true}, + {"zyncQueDeployment", "zync-que", &k8sappsv1.Deployment{}, true}, + {"zyncDatabaseDeployment", "zync-database", &k8sappsv1.Deployment{}, false}, {"zyncService", "zync", &v1.Service{}, true}, {"zyncDatabaseService", "zync-database", &v1.Service{}, false}, {"zyncSecret", component.ZyncSecretName, &v1.Secret{}, true}, @@ -271,12 +289,18 @@ func TestReplicaZyncReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -310,20 +334,20 @@ func TestReplicaZyncReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { subT.Errorf("error updating dc of %s: %v", tc.objName, err) } @@ -334,13 +358,13 @@ func TestReplicaZyncReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } @@ -363,11 +387,10 @@ func testZyncAPIManagerCreator(zyncReplicas, zyncQueReplicas *int64) *appsv1alph }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: zyncReplicas}, diff --git a/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl index c048b73a6..1b88a6e83 100644 --- a/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl @@ -3621,15 +3621,15 @@ }, "id": 13, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -3689,7 +3689,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3704,7 +3704,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -3783,7 +3783,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3798,7 +3798,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -3876,7 +3876,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3891,7 +3891,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -3969,7 +3969,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3984,7 +3984,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4041,7 +4041,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -4053,7 +4053,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -4062,21 +4062,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -4161,7 +4161,7 @@ "renderer": "flot", "repeatedByRow": false, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -4173,7 +4173,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -4235,13 +4235,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -4304,7 +4304,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4319,7 +4319,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4401,7 +4401,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4416,7 +4416,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -4497,7 +4497,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4512,7 +4512,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4593,7 +4593,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4608,7 +4608,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4668,7 +4668,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4680,7 +4680,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -4689,21 +4689,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -4790,7 +4790,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4802,7 +4802,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -4864,13 +4864,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -4933,7 +4933,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -4948,7 +4948,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5030,7 +5030,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5045,7 +5045,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -5126,7 +5126,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5141,7 +5141,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5222,7 +5222,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5237,7 +5237,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5297,7 +5297,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5309,7 +5309,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -5318,21 +5318,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -5419,7 +5419,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5431,7 +5431,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -5493,13 +5493,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -5562,7 +5562,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5577,7 +5577,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5659,7 +5659,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5674,7 +5674,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -5755,7 +5755,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5770,7 +5770,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5851,7 +5851,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5866,7 +5866,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5926,7 +5926,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5938,7 +5938,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -5947,21 +5947,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -6048,7 +6048,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -6060,7 +6060,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -6118,15 +6118,15 @@ }, "id": 4, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6163,7 +6163,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -6175,7 +6175,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -6239,13 +6239,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6285,7 +6285,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -6297,7 +6297,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6361,13 +6361,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6407,7 +6407,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -6419,7 +6419,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6483,13 +6483,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6529,7 +6529,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -6541,7 +6541,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6601,15 +6601,15 @@ }, "id": 5, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -6648,7 +6648,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -6769,7 +6769,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6778,7 +6778,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6787,7 +6787,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6796,7 +6796,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6805,7 +6805,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6865,13 +6865,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -6913,7 +6913,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -7034,7 +7034,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7043,7 +7043,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7052,7 +7052,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7061,7 +7061,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7070,7 +7070,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7130,13 +7130,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -7178,7 +7178,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -7299,7 +7299,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7308,7 +7308,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7317,7 +7317,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7326,7 +7326,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7335,7 +7335,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7395,13 +7395,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -7443,7 +7443,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -7564,7 +7564,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7573,7 +7573,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7582,7 +7582,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7591,7 +7591,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7600,7 +7600,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7656,15 +7656,15 @@ }, "id": 6, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7700,7 +7700,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -7712,7 +7712,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -7776,13 +7776,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7821,7 +7821,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -7833,7 +7833,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -7897,13 +7897,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7942,7 +7942,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -7954,7 +7954,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -8018,13 +8018,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -8063,7 +8063,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -8075,7 +8075,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -8135,15 +8135,15 @@ }, "id": 7, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8182,7 +8182,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -8303,7 +8303,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8312,7 +8312,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8321,7 +8321,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8330,7 +8330,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8339,7 +8339,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8399,13 +8399,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8447,7 +8447,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -8568,7 +8568,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8577,7 +8577,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8586,7 +8586,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8595,7 +8595,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8604,7 +8604,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8664,13 +8664,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8712,7 +8712,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -8833,7 +8833,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8842,7 +8842,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8851,7 +8851,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8860,7 +8860,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8869,7 +8869,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8929,13 +8929,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8977,7 +8977,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9098,7 +9098,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9107,7 +9107,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9116,7 +9116,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9125,7 +9125,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9134,7 +9134,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9190,15 +9190,15 @@ }, "id": 15, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9234,7 +9234,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -9246,7 +9246,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -9328,7 +9328,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -9340,7 +9340,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9402,13 +9402,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9447,7 +9447,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -9459,7 +9459,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9544,7 +9544,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -9556,7 +9556,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9618,13 +9618,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9663,7 +9663,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -9675,7 +9675,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9760,7 +9760,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -9772,7 +9772,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9834,13 +9834,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9879,7 +9879,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9891,7 +9891,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9976,7 +9976,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9988,7 +9988,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -10096,9 +10096,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'backend-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'backend-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl index cd77a29bd..8cfadf8fa 100644 --- a/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl @@ -1464,8 +1464,8 @@ }, "id": 30, "panels": [], - "repeat": "deploymentConfig", - "title": "Pods ($deploymentConfig)", + "repeat": "deployment", + "title": "Pods ($deployment)", "type": "row" }, { @@ -1533,7 +1533,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1619,7 +1619,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1705,7 +1705,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1790,7 +1790,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -1855,7 +1855,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -1864,21 +1864,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -1970,7 +1970,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -2028,8 +2028,8 @@ }, "id": 37, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -2073,7 +2073,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2133,8 +2133,8 @@ }, "id": 39, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -2290,7 +2290,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2299,7 +2299,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2308,7 +2308,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2317,7 +2317,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2326,7 +2326,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2382,8 +2382,8 @@ }, "id": 41, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -2424,7 +2424,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2484,8 +2484,8 @@ }, "id": 43, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -2638,7 +2638,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2647,7 +2647,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2656,7 +2656,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2665,7 +2665,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2674,7 +2674,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2730,8 +2730,8 @@ }, "id": 45, "panels": [], - "repeat": "deploymentConfig", - "title": "Network Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -2772,7 +2772,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2859,7 +2859,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2917,8 +2917,8 @@ }, "id": 100, "panels": [], - "repeat": "deploymentConfig", - "title": "Active Record ($deploymentConfig)", + "repeat": "deployment", + "title": "Active Record ($deployment)", "type": "row" }, { @@ -2961,7 +2961,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(label_replace(rails_connection_pool_connections{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'},'connection', '$1', 'state', '(.*)')) by (connection)", + "expr": "sum(label_replace(rails_connection_pool_connections{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'},'connection', '$1', 'state', '(.*)')) by (connection)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -3049,7 +3049,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rails_connection_pool_size{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'})", + "expr": "sum (rails_connection_pool_size{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'})", "format": "time_series", "instant": false, "interval": "1m", @@ -3138,7 +3138,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rails_connection_pool_waiting{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'})", + "expr": "sum (rails_connection_pool_waiting{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'})", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -3240,9 +3240,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'system-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'system-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl index 5e4a4acbc..ebeb6b168 100644 --- a/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl @@ -1370,8 +1370,8 @@ }, "id": 13, "panels": [], - "repeat": "deploymentConfig", - "title": "Pods ($deploymentConfig)", + "repeat": "deployment", + "title": "Pods ($deployment)", "type": "row" }, { @@ -1439,7 +1439,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1525,7 +1525,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1611,7 +1611,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1696,7 +1696,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -1761,7 +1761,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -1770,21 +1770,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -1876,7 +1876,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -1934,8 +1934,8 @@ }, "id": 4, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -1979,7 +1979,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2039,8 +2039,8 @@ }, "id": 5, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -2196,7 +2196,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2205,7 +2205,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2214,7 +2214,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2223,7 +2223,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2232,7 +2232,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2288,8 +2288,8 @@ }, "id": 6, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -2330,7 +2330,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2390,8 +2390,8 @@ }, "id": 7, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -2544,7 +2544,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2553,7 +2553,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2562,7 +2562,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2571,7 +2571,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2580,7 +2580,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2636,8 +2636,8 @@ }, "id": 15, "panels": [], - "repeat": "deploymentConfig", - "title": "Network Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -2678,7 +2678,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2765,7 +2765,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2865,9 +2865,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'zync-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'zync-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/bindata.go b/pkg/assets/bindata.go index 209cbe8ea..cb4b73b71 100644 --- a/pkg/assets/bindata.go +++ b/pkg/assets/bindata.go @@ -124,7 +124,7 @@ func monitoringApicastGrafanaDashboard2JsonTpl() (*asset, error) { return a, nil } -var _monitoringBackendGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x93\x9b\x46\xba\xfe\x9e\x5f\xd1\x87\x64\x37\xe3\x9c\x19\x5b\xe8\x2e\x57\xb9\x4e\xf9\xba\xf1\xae\x9d\x9d\x75\x9c\x9c\xcd\x71\x5c\x5a\x46\xf4\x68\xd8\x41\x40\x00\x8d\x47\x9e\x9a\xfd\xed\xa7\x68\x10\x34\x97\xd6\xe8\x82\xa4\x06\x9e\x7c\x88\x47\x0d\x42\xcd\xdb\x97\xf7\x79\xde\x5b\xdf\x7d\x43\x88\xa2\x59\x96\xed\x6b\xbe\x61\x5b\x9e\xf2\x94\x04\x4d\x84\x28\xa6\xe1\xf9\xca\x53\xf2\x89\x7d\x22\x51\x2b\xbb\x72\x31\x37\x4c\xff\xad\xa5\x3c\x25\xea\x69\xd2\xaa\x6b\xbe\xe6\xd9\x73\x77\x42\x95\xa7\x44\x39\x3b\x23\x7f\x71\xb5\x4b\xcd\xd2\xc8\xd9\x99\xc2\xdd\x46\x2d\xed\xc2\x0c\x6e\xf1\xdd\x39\xe5\xda\xaf\x0c\xbd\xa0\xd5\x98\xd8\xd6\x4b\xdb\xb4\xdd\xe0\x99\xee\xf4\x42\x3b\x69\x9d\x92\xb6\xaa\x9e\x92\x76\xaf\x77\x4a\xd4\x47\xfc\xa3\x2d\x6d\xc6\x7e\xfb\x79\xf2\x3a\xe4\xcf\xe4\xb9\x49\x5d\xdf\xe3\xef\xf3\x17\x0e\xbb\x4f\xd7\xbc\xab\x0b\x5b\x73\x75\x25\xba\x76\xcf\xfe\xfd\xfc\x0d\x21\xf7\xc1\xed\x0a\xd5\x0d\x3f\xd3\x5b\x65\x6a\x51\xff\xad\xae\x3c\x25\xd6\xdc\x34\xc3\x16\x57\x73\xae\x3e\xda\xb6\xe9\x1b\x8e\xf2\x94\xb4\x58\xa3\x11\xdc\x32\x64\x7f\x9a\x86\x75\x1d\xc8\xf5\xd3\x67\xf6\xd1\xd1\x2c\x6a\x7a\xb1\x64\x97\x72\x55\x26\xb6\x69\x6a\x8e\x47\x83\x2f\x5e\x6a\xa6\x17\x8b\x41\x99\xba\x86\x7e\x6e\x27\x43\x13\xca\x2b\x23\xfe\x2f\xca\x53\xd2\xee\x72\x0d\xb7\xcb\xbe\x44\x9f\x17\xc1\xe7\xe5\x8b\xc6\xcf\x66\xfd\x6c\xc7\xf7\x71\xbd\xfb\x1c\xb7\xf9\x86\xcf\x64\xa0\xbc\x33\x3c\x9f\x5a\xd4\x4d\xa4\x19\xcb\xd2\xb5\xbf\x84\x52\x8c\x1e\x1d\xbf\x96\x66\x1a\x9a\xc7\x86\x90\xbd\x40\xf2\xcb\x17\x1a\x6b\x49\xbf\x6a\x30\x24\xef\xa8\x35\xf5\xd9\xeb\xb5\x52\xed\xb4\xe8\x76\x7e\xce\x7d\xc7\x7d\x8c\x6f\xb9\x34\x4c\x93\x17\x95\x58\x9a\xc3\x8c\x34\xd5\xf6\x03\xd2\x54\x8b\xa5\xd9\x19\xc5\x9f\x4d\x3a\xa5\x96\x9e\xfe\x29\xed\x66\x9a\x7d\x8f\x60\xf4\xe7\xae\x4b\x2d\xbf\xe0\xca\x4c\xbb\x2d\x6a\x35\xac\x82\x56\xef\xca\xfe\x92\x5f\x44\xbe\xed\x6b\x66\xc1\xdd\x37\x9a\x39\x4f\x84\x9a\x7b\x19\xd3\xb0\xd8\x55\xfe\x69\xac\xf1\x8b\xa1\xfb\xa9\xe9\x97\x99\xe2\xac\x29\x58\x1e\xe7\xb6\x61\xf9\xef\x6d\xb6\xb0\x59\x43\x32\x2c\xb6\x13\x6f\x37\xc9\x2f\x3a\xd4\x9d\x50\xcb\xd7\xa6\x34\x37\xd2\x4e\xf0\x28\x57\xd3\x8d\x79\xf0\x9d\x76\xba\x3d\x3f\x31\x5c\x6a\xe9\xd4\xa5\x6c\xdb\xb8\x34\x6d\x3f\xf9\x61\x8f\xba\x06\xf5\xfe\x7e\x43\x5d\xd7\xd0\x69\xa6\xd3\x9e\xa3\x4d\x68\xd1\xfc\xf3\x7c\x6d\x72\x9d\xfb\x15\xcf\xa7\x8e\x43\xf5\x77\x86\x95\xef\xb0\xaf\xb9\x53\xea\x7b\xdc\x0e\xca\xef\xa1\xc1\xe6\x72\xeb\xb0\xee\x79\xf3\xd9\x89\xab\xf9\xf4\x44\x73\x0c\xcf\xb6\x34\xdf\x76\xc7\x66\xb4\xd0\xc6\x2e\xf5\x1c\xdb\xf2\xe8\x78\x62\xeb\xd4\xbb\x0b\x76\x38\xd6\xc7\x67\xbf\x2b\xdf\xc5\x1f\x7e\x57\x4e\x5d\xfa\xc7\x9c\x7a\xfe\x38\x58\x8e\xcf\x7e\x57\xb4\xb9\x7f\x65\xbb\xc6\x57\xfa\xbb\x72\xff\x49\x9d\x7d\x7e\xc4\xef\x93\xc1\xa2\xb0\xdd\x99\x16\x4c\x36\xc5\x37\x66\x74\x1c\xca\x24\x7d\x8b\x61\xf9\xd4\xbd\x61\xf3\x46\x51\x67\xc5\xd7\xde\x68\x13\x9f\x6d\xcd\x6a\x2b\x75\x3d\x9c\xf6\x6f\xe2\x1f\x89\xbb\x93\x7e\x8c\x4b\x2f\xd9\x4e\xaa\x3c\x57\xe2\xe6\xfb\xd3\xe3\x48\xcb\xa5\x8e\x3c\xb2\x72\xa9\x23\x90\xd4\x8b\xe3\x4a\xca\xa5\x8e\xed\xfa\x72\x08\x2a\xec\x8b\x40\x4e\x2f\x8f\x3f\xa3\xd8\x84\x1f\xdb\xc1\x9f\x72\x08\x2c\xd3\x29\x81\xe4\x5e\x1d\x5f\x72\x2e\x75\x64\x93\x5b\xdc\x25\x81\xd4\x5e\x73\x52\x8b\xfe\xe2\xf0\xd3\x95\x4b\xbd\x2b\xdb\xd4\x73\xb8\x6a\x46\xdf\xb8\xf6\x8c\x03\x93\x71\xfb\x07\x3a\x8d\xf4\x63\xe6\x0b\x3f\x5f\x19\x97\x7e\xfe\x1b\x19\x84\x46\x22\xb1\x7a\xc4\xa1\x2e\xf1\xe8\xc4\xb6\x74\x72\x72\xb1\x58\xb6\x93\x40\xdc\x8f\x38\x18\x17\xc3\xd7\x3b\x1e\x4d\x68\x2e\x83\xa3\x19\x3c\xe1\x05\xeb\x8e\x53\xc2\x4b\x28\x31\x5e\x82\x41\xc3\xd2\x8d\x1b\x43\x9f\x6b\xa6\x92\x43\x15\xcb\x7b\x18\x6a\x4e\x3a\x70\xab\xdd\x1a\x19\x4c\x76\x31\x9f\x5c\x87\x1a\x94\x7f\xd7\x00\xfb\x44\x88\x22\x10\x47\x01\xfe\xcf\xdc\x5d\x8c\x89\x62\xec\xf3\xe9\x73\xae\x8b\x0b\xed\x96\xae\x50\xdc\x3a\x9d\x18\x33\x8d\x81\xe4\x96\x60\x5e\xba\xf4\x0f\x27\x33\x23\x4d\xed\x82\x9a\xb9\xde\x05\x17\xec\xe9\x0b\xcd\xa3\x69\x38\x1f\xe3\xbe\xdc\xed\x21\xf0\x4b\xff\x30\xf7\x8a\x0f\x2e\xde\xa4\x93\xde\x55\x6e\xff\x2c\xb7\x93\xb9\xe6\xc2\x7e\xe6\x96\xcb\x22\x3f\x15\x34\xd3\x98\x16\xe1\x5d\xd6\xfe\x8e\xde\xc4\x9d\xfe\x86\x7f\x68\x8d\xa9\x48\xaa\x61\x05\x17\xe9\xaa\xa5\x72\x91\x80\xa4\xbf\x9e\x39\xfe\xa2\x98\xbf\xff\x1f\x75\xed\xfc\x15\x10\x18\x10\x98\x35\x60\x80\xe7\xb0\x7b\x9e\xfd\xae\xb4\x6f\x6f\xe5\xd0\xfc\xed\x7f\xfe\x53\x1a\xc6\x92\x88\xa7\xdb\xea\xc8\x21\x9e\x6e\xab\x23\x0d\x4d\xe1\xc5\xd3\x95\x45\x3c\x5d\x69\xd8\x09\x2f\x9e\x91\x2c\xe2\x19\x49\x43\x41\x12\xf1\xf4\x64\xd9\x7b\x7a\xc2\xbd\xa7\x42\x5c\x23\x94\x3f\x09\x64\x0b\xb2\x01\xb2\x01\xb2\x51\x49\xb2\x91\xf5\x7b\x8c\x04\x5c\xa3\x53\x2a\xd7\x00\x6d\x00\x6d\xa0\xe3\x60\xdb\xf5\xc6\xa1\x52\xf1\xc6\xe1\x2e\xbd\x83\x1b\x84\x5c\x2c\xc8\x89\x49\x85\xaa\xfd\x8a\x6a\xfe\x4c\x73\xf6\xa7\xd6\xef\xee\xfe\x75\x77\x47\x4c\x4a\xee\xef\xff\x75\x7f\xbf\x06\xbb\x38\xa6\x86\x7f\xbe\x94\xdf\xc3\x2a\x3e\x78\x18\x09\x87\x87\x18\x56\x74\x8b\x07\xa5\x0f\xa5\xdf\x24\xa5\x3f\xd1\x5c\x3d\xf3\xe0\xa0\xe9\x5c\xd3\x75\xc3\x9a\xe6\x67\x4e\x70\xf1\x83\x3d\xb7\xf4\xcc\xc3\xe3\x9e\x4e\xa2\xd8\x97\xcc\x03\xe3\x90\x98\x6f\xdf\x3c\x7f\xf5\xba\xfd\x9c\x9f\xa3\xec\x2b\x3f\x4f\xb4\x70\x09\x7b\x7f\xa4\x46\x60\x79\xf5\x8a\xce\xa2\x75\xe4\x53\xd7\xb1\x4d\xcd\xa7\x7f\x77\x35\x6b\x9a\x62\x34\xc1\x56\x6d\x5b\xa1\x76\x6e\x3d\xee\x15\xac\x0f\xdb\xd1\x26\x86\xbf\xc8\xaf\xc1\x00\x91\x24\x9b\x9e\xef\x2d\x57\xda\x26\x08\xa6\x44\xeb\x68\x1e\xb1\x2c\x37\xfa\x94\x1e\x5e\xda\x30\x5f\xc4\xfb\x42\x5a\xb7\x5d\x19\xd3\x2b\xd3\x98\x5e\xf9\x2f\xa3\x71\x4e\x41\x84\x10\x04\xf5\x56\x82\xa0\x68\x7e\x0a\x81\x47\x16\x4d\x14\xc2\x05\x97\xde\x50\xd7\xa3\xbf\x89\xba\x09\x15\xbc\x6f\x15\xbc\x42\xd5\xee\xa2\x51\xa3\x57\x2f\x47\xb3\x16\xe9\xa4\xa0\xf5\x47\xc3\xf3\xed\xa9\xab\xcd\x84\xf3\x70\xa9\x40\xb3\x23\xa1\xdc\x3e\xcf\xed\x9b\xf9\x0d\x37\x79\xce\x6d\x38\x41\x7f\x9a\xcf\x2e\x18\x22\x4d\x89\x24\xba\xf8\xb3\xf1\x35\xab\x51\x95\x45\xfe\x67\x8a\x35\x22\xaf\x6a\xf8\x7d\xab\x50\x97\x14\x6a\x92\xbc\xb2\x13\x49\xce\x31\x0d\x3f\x9e\x61\x85\x7b\xf5\x22\x7c\xa3\x17\xd1\x7e\x1e\x4c\x7d\x5b\xc9\x5e\x2d\x16\xc6\x22\x27\x8c\x42\xf5\x52\x63\x4e\xa9\x0e\x72\xf2\x0c\xf7\xd3\x01\x48\x25\x48\xa5\x6c\x1a\x8d\x8f\x16\xab\x83\x3e\x3b\x2c\xa5\x74\xa9\x03\x42\x19\xb6\x83\x50\x82\x50\xf2\x0f\x3d\x3e\xa1\xec\x0d\x46\xdd\x37\x6d\x10\xca\xb8\xa1\x38\xdc\x26\x0f\x57\xf6\xc5\x28\x7b\xab\xd3\x09\x1a\xc9\x28\xeb\xa6\x7f\x77\xe2\x93\x22\x75\x0a\x36\x49\xc0\x26\x9b\xc9\x26\xdb\xbd\x9c\x3c\xc3\xbd\xb4\x05\x36\x09\x36\x29\x93\x36\xcb\x64\xd4\xd4\x41\x99\x1d\x8e\x4c\x7e\x60\xc2\x03\x97\x0c\xdb\xc1\x25\xc1\x25\xf9\x87\x1e\x9f\x4b\xbe\x69\x77\x47\xbd\x97\xe0\x92\x71\x43\x21\x97\x2c\x00\x2b\x7b\xe3\x92\x43\x70\xc9\x9a\x6b\xdf\xad\xa9\xe4\x0a\x65\x0a\x26\x49\xc0\x24\x9b\xc9\x24\x3b\x9d\x9c\x3c\xd9\x4e\xda\x07\x93\x04\x93\x94\x4a\x97\x89\x6a\x0e\xd4\x41\xa9\x1d\x23\xe4\xf5\x84\x89\xf1\x11\xd8\x65\xd8\x0e\x76\x09\x76\xc9\x3f\x54\x02\x76\xf9\x66\x34\xec\xb4\xc0\x2e\xe3\x86\x42\x76\x59\x00\x60\xf6\xc5\x2e\xfb\xab\x8b\x0d\x34\x8e\x5d\xd6\x59\x23\x97\x10\x01\xbb\x4a\xc1\x82\x71\x12\x30\xce\x66\x32\xce\xae\xa0\x94\x4b\xbf\x0d\xc6\x09\xc6\x29\x9b\x7e\xcb\xd7\x6a\xab\x83\x76\x3b\x7c\x3c\x2c\xd8\x26\xd8\x26\xd8\xa6\xc4\x6c\xf3\xc5\x70\x30\x78\x35\x02\xdb\x8c\x1b\x0a\xd9\x66\x01\x78\xd9\x1b\xdb\x5c\x5d\x6e\xa2\x91\x6c\xb3\x9e\xda\x78\xe7\xe8\x58\x30\xcd\xd4\x45\x30\xcd\x0d\x98\xe6\x21\x8e\x65\xe8\x8a\xc2\x55\x7b\x49\x36\xf9\x7a\x07\x33\x90\xb7\xc1\xf2\xb4\x34\x93\x3c\x3f\x7f\xab\xe4\x66\x57\x13\x4f\x69\xe8\x0a\xca\x15\xa9\x2a\xce\x69\x00\xa1\x2e\x49\x85\x1b\xd1\xb2\x1b\x6b\x8e\xb1\x75\xe9\x73\x76\x50\x8d\x1c\xd5\x07\xb5\xec\xa1\x39\xfb\x28\x7e\x5a\x8a\xd0\x1c\xc7\x34\x26\xec\xb0\x9f\xf1\x35\x5d\xc8\x22\xbe\x4c\xaf\x04\x82\x2c\xab\x4c\x6a\xd9\x82\x74\xe9\x25\x75\x5d\xea\x8e\x2f\x0d\xd3\xa7\xae\x84\x42\xcd\xf6\x50\x20\xe0\xb2\x0a\xad\x96\x2c\x60\xf9\x04\x2a\x12\x60\x59\xa5\x58\xcb\x10\x20\x75\x5d\x5b\x96\xb9\x18\xf6\x45\x20\xb4\xd7\x32\x09\xed\x86\x5a\xb2\x28\x95\xb0\x2f\x02\xa1\xbd\x91\x48\x68\x33\xea\xbb\xc6\x44\x12\xa9\x45\x9d\x11\x88\xed\x2f\x12\x89\xcd\xa3\xee\x8d\x31\xa1\x63\xdf\xbe\xa6\xb2\xec\x71\xe9\x3e\x09\x84\xf8\xa3\x7c\x42\x94\x4b\x7c\x22\xc1\xbd\x95\x49\x70\xbe\x26\xcb\x46\xc7\xba\x22\x10\xd9\x5f\x25\x12\xd9\xdc\xd3\xa6\x74\x6c\x1a\x33\x43\x16\xc9\xf1\x3d\x12\x08\xf0\x6f\x32\x09\xd0\x37\x4c\xe3\x2b\x43\x50\x92\xc8\x2f\xe9\x90\x40\x7c\xef\x24\x71\x7c\xc6\xd5\xe3\x79\x8b\x15\x8e\xad\xe2\xde\x02\xbe\xce\xc2\x4e\x36\xce\xd7\x59\x23\xdb\x6c\xde\x5f\x28\x32\xce\xb6\xcb\xcd\xaf\xc1\xc1\x55\xf1\x65\x58\x74\xf7\x09\x0f\x1a\x79\x8a\xd5\xce\xb2\x6a\xd0\x91\x56\x25\xc8\xaa\x31\xe7\x5b\x95\x20\xab\xc6\x1c\x76\xb5\xb3\xac\x70\xf2\x15\x77\xa1\x5c\xee\x82\x63\xb0\x08\xc8\x0b\xc8\x4b\xc5\xc9\x4b\x36\xb0\xa4\x27\x28\x59\xae\x0e\x56\x9f\x01\x81\xc0\x12\xd0\x90\x1d\xd5\xfa\x6e\x81\xa2\xe9\x38\x93\x3a\x44\x88\x1e\x30\x5f\x83\x09\x0f\x69\x1a\x61\x3b\xb4\x3f\xb4\x3f\xff\xd0\xe3\xa7\x69\xe0\x3c\xac\x75\xcc\xae\x05\xd0\x65\x5f\x69\x1a\xea\xa0\xbf\x12\x0d\x55\x28\x4f\x03\xba\x78\xbf\xd9\x1a\x62\xd5\x8a\x24\x0d\x76\x09\x49\x1a\xcd\x2b\x07\xd0\x17\xe4\x86\xa8\x03\x9c\x8c\x05\x96\x29\xb1\x66\x13\x05\xe6\xd7\x41\xc7\x1d\x90\x6f\x26\x62\x24\x7f\xa3\x0b\x30\xcf\xa8\x1d\xcc\x13\xcc\x93\x7f\x28\x98\x67\x35\x98\x67\x01\x9c\xd9\x1f\xf3\xac\x4d\xb5\x73\xe8\xe7\x58\xdc\x7b\xe6\xa0\xeb\xa8\x5b\xb0\x51\x02\x36\xda\x4c\x36\x3a\x10\x94\x43\x57\x07\x48\xa6\x07\x1b\xad\x86\xb6\x13\x66\x37\xd7\x41\xf3\x1d\xf2\xf0\xad\x50\x8c\xe4\x4d\x28\x46\x30\xd3\xb0\x1d\xcc\x14\xcc\x94\x7f\x28\x98\x69\x35\x98\x69\x01\xb4\xd9\x1b\x33\x1d\xae\xce\x6e\x01\x33\x6d\x84\xae\xde\xe1\xa8\xae\x35\x54\x2f\x58\x2a\x01\x4b\x6d\x26\x4b\x1d\x0a\x4a\xa8\xab\xc3\xd5\x27\x54\x80\xa5\x82\xa5\x4a\xa2\xf9\xea\xa5\xe9\x8e\xe2\x2f\x05\x23\x8d\xda\xc1\x48\xc1\x48\xf9\x87\x82\x91\x56\x83\x91\x16\xc0\x98\xfd\x31\xd2\xd5\xa7\xcb\x80\x91\xd6\x52\x2f\x97\xe1\x27\x05\xfb\x04\xfb\x04\xfb\x5c\xbe\x7d\xf0\x05\x51\x4d\x9b\xe1\xea\x13\x2b\xc0\x3e\xc1\x3e\x8f\xa9\xe5\x32\xf5\x75\xeb\xa0\xdf\x0e\xc7\x3b\xdf\x5a\x3e\x9d\xba\x61\xe0\xd0\x6b\x26\xc8\xf5\x6a\x42\x80\x81\x82\x81\xe6\x3a\x09\x06\x0a\x06\x2a\x07\x03\xcd\x43\x99\xfd\x31\xd0\x2e\x18\x68\x93\x74\xf3\xd6\xdc\x73\x03\x55\x0b\x16\x4a\xc0\x42\x9b\xc9\x42\x47\xa2\xea\x44\x43\x54\x27\x02\x0b\x95\x58\xd3\xa5\x0f\x2c\xa9\x83\xa6\x3b\x1c\x0b\x7d\xcd\x84\x07\xd6\x19\xb6\x83\x75\x82\x75\xf2\x0f\x05\xeb\xac\x06\xeb\x2c\x80\x2e\xfb\x63\x9d\xa8\x4e\xd4\x28\x5d\xbc\x35\xeb\x5c\xa1\x5a\xc1\x32\x09\x58\x66\x33\x59\xa6\xda\x12\x95\x27\x1a\xa2\x3c\x11\x68\xa6\xbc\xaa\x2d\x7b\xc4\x63\x1d\x74\xdb\xe1\x78\xe6\xfb\x50\x7a\x20\x9a\x61\x3b\x88\x26\x88\x26\xff\x50\x10\xcd\x6a\x10\xcd\x22\xf4\xb2\x3f\xa6\x89\x6a\x44\xcd\x52\xc7\x5b\x53\xcd\x55\xda\x15\x5c\x93\x80\x6b\x36\x94\x6b\xaa\xa2\xe2\x43\x43\x14\x1f\x02\xd7\x94\x57\xb9\x09\xce\xc5\xaf\x83\x8e\x3b\x1c\xe5\xfc\x39\x14\x22\xf9\xc8\x84\x08\xe6\x19\xb6\x83\x79\x82\x79\xf2\x0f\x05\xf3\xac\x08\xf3\x2c\xc0\x32\x7b\x63\x9e\x23\x54\x1b\x6a\xa4\x72\xde\x9a\x80\xae\xa1\x6b\xc1\x43\x09\x78\x68\x43\x79\x68\x5b\x54\x5e\x68\x84\xf2\x42\xe0\xa1\xd2\xab\xba\x7a\x29\xb9\x83\x33\x50\x70\xcf\xa8\x1d\xdc\x13\xdc\x93\x7f\x28\xb8\x67\x45\xb8\x67\x01\x7e\xd9\x1f\xf7\x44\x5d\xa1\x86\x29\xe4\x5d\x59\x27\xf8\x26\xf8\x26\xf8\xe6\xf2\xed\xd9\x0d\xa2\x82\x42\x23\x14\x14\x02\xdf\x94\x58\xbd\xf9\x5a\xcd\xb2\x47\x0e\x48\x36\x03\xd9\x81\x69\x86\xed\x60\x9a\x60\x9a\xfc\x43\xc1\x34\xab\xc2\x34\x0f\x58\x3f\x68\x84\xfa\x41\x4d\x52\xc5\xdb\xd3\x4c\xa1\x66\x05\xc7\x24\xe0\x98\x0d\xe5\x98\x1d\x51\xb9\xa0\x11\xca\x05\x81\x63\xca\xab\xd8\xe6\x9e\x36\xa5\x63\xd3\x98\x19\x35\xd3\x6f\x87\xa3\x9a\xbf\x04\x22\x24\xef\x98\x08\xc1\x38\xc3\x76\x30\x4e\x30\x4e\xfe\xa1\x60\x9c\x15\x61\x9c\x05\x38\x66\x7f\x8c\x13\xb5\x83\x1a\xa8\x98\xb7\x26\x9e\x0f\xea\x59\xf0\x4f\x02\xfe\xd9\x50\xfe\xd9\x15\xd5\x11\x1a\xa1\x8e\x10\xf8\xa7\xc4\x6a\xce\x37\x4c\xe3\x2b\x2b\x46\x5e\x2b\x2d\x77\x40\xfa\x99\x48\x10\xec\x33\x6c\x07\xfb\x04\xfb\xe4\x1f\x0a\xf6\x59\x11\xf6\x59\x80\x62\xf6\xc7\x3e\x51\x4f\xa8\x79\x6a\x79\x7b\xf2\xf9\x80\x96\x05\xf7\x24\xe0\x9e\x2b\xb8\xe7\xc4\x36\x4d\xcd\xf1\x18\x8a\x4a\xef\x01\xc2\xfd\x53\xcd\xec\x9f\xed\xee\x43\x24\xb0\x27\x28\xf0\xd3\xe3\x18\x8a\x66\x51\x33\x87\x31\xa3\x49\xfe\xbf\xb6\x7b\x4d\x5d\x6e\xfb\x5f\x4e\x27\xd7\xfe\xa2\x14\xbe\x56\x75\x29\xf5\x1a\xd2\xec\x16\x4b\xb3\x8b\x6a\x49\x60\xd4\xbb\xa9\xee\x2f\x6c\xa1\x8d\xff\x6d\x5f\x8c\x27\xf6\xdc\x12\xeb\xe5\x7b\xc2\xe9\xde\x60\x35\x0a\xb5\x6f\xa0\x6b\xc6\xe1\xcb\xef\x5b\x03\x07\xfd\xa8\x04\x35\xfe\xab\x7d\x91\xa3\xc2\xff\xb6\x2f\x48\x5a\x90\xe0\xbc\xe5\x72\x5e\x1b\x8c\x57\x3a\xc6\x5b\x5d\x3d\xfd\xb0\xe9\xbb\xdf\x16\xa0\x9e\xd5\x01\xad\xd0\xd3\xd0\xd3\x1b\xe8\x69\x77\x6e\x45\x2a\x76\x3d\x42\x1d\x11\xe9\x0f\xd4\xb1\x5d\xff\xaf\xf6\x45\xad\x68\xf4\xe1\x54\x78\x28\xbf\x40\x6b\xe7\x8d\xda\xe1\x88\xc0\x9c\x0d\xd5\xde\x64\xd5\x7e\x78\x63\x76\x6f\x30\xea\xbe\x69\xc3\x98\x1d\x37\x14\x1b\xb3\x0b\x70\xc9\xbe\x8c\xd9\xbd\xd5\x41\xe6\xf2\xdb\xb2\xa1\x68\x4b\xb6\x57\x0b\xf4\x66\xca\x4c\xbd\xbd\xfe\x84\x81\xba\x76\x06\xea\x3a\x33\xc4\x41\x2b\x27\xd0\x70\xdb\x5c\x1d\x80\x0a\x86\x08\x86\xb8\x7f\xc5\xf5\x93\xed\x1b\x97\x8b\xba\x29\xae\xc3\x31\xc4\x50\x7e\x60\x88\x60\x88\xe9\x97\x01\x43\x5c\x3e\xf8\xf0\x0c\xf1\xa2\x7b\x79\xd9\x6a\x81\x21\xc6\x0d\xc5\x0c\xb1\x00\x97\xec\x8d\x21\xae\x0e\x03\x6f\x02\x43\xac\xa7\xa2\xdd\x9a\x21\x0a\xf4\x26\x18\x22\x18\xe2\x11\x43\x98\x06\xc3\xdc\x9b\x85\xe1\x9a\x49\xa9\xbe\xa2\x10\x26\x97\x3a\x34\x1c\x0a\x9d\x3a\xa6\xbd\x98\x51\xcb\x7f\x69\x5b\x97\xc6\x94\xe3\x13\x13\xdb\xa1\xfa\xaf\x21\x9d\x4d\x8d\x6e\xe6\x1b\x4f\xd3\xbb\x92\x47\x4d\x3a\xf1\xf3\xaf\xcd\x2e\xfa\xf4\x96\xfd\xec\x85\x36\xb9\xa6\x96\x7e\x36\x71\x6d\x2b\xbd\x8e\x19\xde\xca\xdd\x92\x5b\xd2\xf7\xf9\x15\x7a\x6e\xeb\x1e\x39\xf9\x2e\xdb\xbf\x47\x1b\xc4\x67\x4d\xb4\xc9\x15\xfd\x68\xcc\xa8\x3d\xcf\xed\x04\x4c\xe9\xbe\xd0\x26\xd7\x53\x37\x9a\x34\x29\xb5\xc1\x2e\xff\x1a\x75\x3e\x3d\xde\x93\xa5\x99\x20\xd9\xc7\x95\x6f\xdf\xb4\xbb\xa3\xde\x4b\x7e\x21\xb8\xd3\x0b\xed\xa4\xdd\x19\x9c\x12\xb5\x3d\x3a\x25\xdd\xd6\x29\x69\x3d\x1e\x8e\xf8\xed\x56\xf9\xb6\x3d\x1a\x4d\xba\x7d\x25\xb7\xb1\xad\xa1\x8a\x8b\x96\x25\xb7\x28\x2d\xdb\xe2\xf5\xb6\x36\x67\x64\xf5\x2e\xb5\x24\x97\xef\xa7\xb6\x5a\xe9\x65\xb9\xbc\x50\xb0\x36\xb3\x60\x2c\xe6\x3a\xef\x02\x14\x99\x53\x7b\xfc\x1d\xef\xb5\x30\xc0\x4e\xb0\x65\x09\xd7\x51\x27\xb3\x8e\xfa\x0f\x2e\xa3\x82\xc2\x51\x86\xce\x66\xc2\x92\x50\x17\xc2\x84\x4e\x22\x48\x5e\xcf\x29\xab\xb0\xc0\x4c\x73\x1c\xc3\x9a\x7e\x0c\xe7\xa2\x5a\xd4\xbe\x42\xe7\x47\xd4\x25\x5c\x26\xc4\xb7\x09\x5b\x51\x85\x2b\x48\x4d\x16\x8d\x00\xe4\x2f\x1f\xc6\x40\xe3\xea\x87\xb5\x57\x28\xd5\x99\x76\xfb\x4a\xf3\xb5\xf3\xa5\xc5\x82\x9b\x1d\x79\x6b\xc9\xc4\xb6\xac\x70\x7f\x48\xdd\xf3\x31\xdc\x18\x52\x2b\xae\xd8\x94\x62\xce\xa7\x86\xf5\x2b\x75\x3d\xc3\x0e\xb4\x81\xd2\x7f\xdc\x7e\xdc\x4d\x1e\xe6\xd8\x9e\x7f\x69\xdc\xa6\x87\x21\x6a\x7c\x63\x5b\xcb\x6d\x5b\xe9\xb5\xfe\xc4\x5d\x77\x69\xfe\x3b\xac\x4d\xf8\x15\x26\xb3\xf7\x9a\xb3\x62\xac\x2e\x43\xa0\x91\x36\x10\x11\x7e\x1b\xfc\xe9\xc9\xf3\xcc\x05\x3b\xfe\xc2\x0a\x81\x57\x65\x6b\xf6\x1c\xcd\xbd\x36\x43\x23\x12\xd7\xcd\x4b\xc3\x34\x63\xf2\xc3\x76\xbd\x8e\x7a\x4a\x54\x75\x78\xca\xce\xd6\x24\xad\xc7\xea\x30\xb5\xeb\x5d\xce\xcd\x22\x33\x5f\xf0\x64\xfe\x39\xe1\x63\xda\xad\x53\xa2\x8e\x3a\xa9\x07\xac\xc4\xeb\xbe\x76\x61\x06\xcf\x99\xcf\xac\xf4\x0c\xd8\x08\x80\x5f\xcf\x2f\xe8\xd8\xa5\x8e\x69\x4c\x58\xd8\xfb\xc4\xb6\x7c\xd7\x36\x4d\xea\x8e\x3d\x5f\xf3\xe7\xde\xd8\xa5\x9a\xbe\x58\xde\xe2\x71\xf8\xfb\xfb\x04\x7e\x7f\x7f\x5a\xf8\x88\x67\xff\xf9\x3e\xa7\xd7\x1e\xff\xf0\xfd\xfd\xf6\xe1\x8d\x09\xe4\x5e\x89\xb8\x77\x36\x66\x29\xea\x69\x5b\x29\x82\xdf\x4a\xa7\xe5\x29\x85\xf8\x3b\x7b\x65\xe9\xa2\x99\x5b\x96\x61\x4d\x89\x63\xeb\x05\x21\xd7\x9e\x61\x4d\x4d\x1a\x48\x3a\xb9\xc6\xe6\x2b\xbf\x80\x87\xfc\x02\x66\x57\x57\x2f\x60\x3b\xc0\xe9\xca\xb3\xe2\xb5\xdb\x12\x2c\x8e\x87\x16\x2f\xbb\xf1\xa7\x68\xe7\xd5\x6e\xa6\xfb\x41\x22\xe7\xcb\x1d\xad\x00\x8a\x6c\x80\x52\x22\xb8\xb1\x29\x4a\x89\xc0\x0d\x50\xca\x2e\x28\xa5\x5f\x1a\x4a\x69\x17\xa1\x94\xd4\x94\x02\x4e\x29\x1d\xa7\x00\x87\x00\x87\x48\x86\x43\x1c\x3a\x29\x19\x7f\x90\x33\x52\x1f\xf0\x73\x54\x78\xf3\x8b\xa5\xdd\x68\x86\x19\xcc\x01\x40\x1c\x18\x5b\xea\x03\x63\x0a\xfc\x38\xdb\xe2\x98\x01\xac\x2d\xb0\xb6\x00\xe5\xa4\xfa\xd9\x3c\x94\xc3\x52\x40\x4f\x96\xff\xb7\x7c\xcd\xb0\xa8\x3b\x9e\xd1\x99\xed\x2e\xc6\x61\x1d\xc0\x8b\x85\x4f\x85\x20\xc3\xb1\xf5\x22\x48\x71\xf6\x49\x3b\xfb\xda\x3a\x1b\x7d\xfe\xef\xe4\xaf\x00\xe1\x5c\x2c\xc8\x89\x65\xeb\xf4\x51\x93\x8c\x2d\xcc\x97\xa2\x1b\x9e\xef\x1a\x17\x73\x9f\xea\xc4\xb6\xc8\x95\xed\xf9\x80\x25\x65\xc2\x92\x2d\xad\x2b\x7a\xb7\xab\x75\x34\xc0\x92\xdd\x60\xc9\xb0\x34\x58\xd2\x87\x79\xe5\xf0\xc0\x04\xc0\x03\xc0\xe3\xc0\xc0\x63\xa6\xdd\x9e\x78\xf3\xd9\x89\x4e\x4d\x5f\x0b\x6d\x1e\x8e\xad\x8f\x13\x0c\x12\xdb\x3a\x3c\x5f\x73\x7d\x6f\xcc\xe2\xd3\x4b\x81\x21\x9f\x7a\x71\x00\x96\x63\xeb\x8d\x82\x22\xef\xb5\x5b\x66\x10\x21\x4b\xb1\x92\x13\x53\xf3\x7c\xd2\x23\x33\xc3\x9a\xfb\xd4\x2b\x08\xf0\xa8\x1b\x26\xa9\x4e\xb6\xc9\x20\xa3\x76\x1f\x0e\x61\x1a\x0a\xea\x11\xa8\x5c\x16\x46\x09\xd9\x26\x81\x2a\x7f\x3d\x73\xfc\x45\x3e\x5c\x6c\x19\x2f\x9a\xbf\x52\xa7\x14\x15\xa2\x79\xe4\x6b\xf0\x96\x0f\x28\xd6\x75\x53\x55\x7a\xe9\xf6\x8d\x52\x55\xaa\xa2\xd8\x64\x4c\xa9\x39\xac\x81\x7f\x77\x45\xd3\x5e\xa9\x68\xd8\xfa\x38\x4b\x19\xbc\xb9\xfb\xde\x19\xd6\x75\x51\x86\x00\xa7\x90\x52\xed\x81\x90\xd9\x60\x3c\x88\x79\xeb\x1f\xba\xc1\xdc\x09\x05\x92\x8d\x65\xf7\x42\x29\x4d\x4a\x70\x2c\x6d\x34\x34\x73\xeb\xc1\xc1\x79\xb9\xfe\xe0\xd4\xd6\x1e\x36\xf7\xa8\x7e\x96\xb6\x3a\xa5\x85\xf4\x4a\x92\x9c\xba\x73\x5b\x27\x4c\xfe\xe4\x84\xed\x67\xa7\x84\x8d\xef\x29\x99\x5b\xc1\xbf\x8f\x88\x66\xe9\x21\x86\x65\x6f\x93\x18\xd6\x0c\x4e\x45\x21\xb5\xae\xdc\xd4\xba\xbd\xe7\xad\x55\x3a\xb9\x8e\x87\x9d\x28\xad\x56\x48\x65\xfa\x9b\x53\x99\xbc\x05\x91\x51\x99\x72\x2b\xa0\x36\x9e\xc9\x94\xc5\x60\x76\x48\xb6\x0f\x13\x68\xa8\xfe\x62\xf1\x21\x6f\xe0\x06\xbf\xd9\x85\xdf\x1c\xdf\xba\xb6\x6f\x5c\x13\xa6\x31\x3a\xb6\x5e\x89\x82\x01\xe7\xeb\xda\xde\xb6\x80\x2f\xad\x86\xc2\x17\xa0\x94\x74\x27\xa5\x2b\x01\x70\x88\xdc\xcd\x91\xe8\x0c\xb2\x76\xa2\x12\x57\x26\x6f\xa6\xd6\x6b\xd8\xfa\xd6\xa7\x2e\x63\xb9\xac\xa2\x5a\xab\x35\x18\xa9\xea\xa0\xc3\x7b\xe2\xc2\xfb\xce\x83\xe7\xbe\xcd\x64\x8a\x1e\x44\x6d\x2d\x8f\xe1\x78\x40\x75\xc5\xb7\xad\x50\x5f\xc8\xfc\x5c\x13\xf3\xc2\xeb\x2f\x5e\x84\xfd\xdc\xc3\xd7\x72\xfa\xab\xed\xc2\xa4\x0a\x04\x23\x22\x18\x91\xec\x33\x26\x60\xdb\x6d\xbe\x93\xbd\x94\x70\x97\xd4\xe4\xae\x9a\x0e\x68\x4a\xec\x41\x7d\x8d\xe1\x12\x85\x1a\x20\xc5\x74\x13\xc8\x83\x14\xd3\xca\xc3\xa1\x5c\x8a\xe9\xf6\x70\xa8\x53\x04\x87\x10\x04\x89\x20\xc8\x63\x01\x9e\x36\x00\x4f\x9d\x01\x0f\x42\x0e\x90\xcb\x5a\x1f\x2c\x05\xf3\x51\x25\xf0\x52\x3e\x97\x75\x7b\xc0\xd4\x2d\x02\x4c\xb0\x1f\xc1\x7e\x24\x27\x9c\x1a\x00\x4e\x55\x19\x4e\xd5\x36\x48\x50\x22\xf3\x11\x92\x66\x91\x34\x5b\x73\xfc\x93\x4b\x9a\xdd\x1e\xff\xf4\x60\x30\x3a\x3c\x02\x02\xc2\x11\x23\x9c\x3e\x10\x4e\x95\x11\x0e\xb2\x73\x91\x9d\x8b\xec\x5c\xb2\x52\xbf\x6f\x9e\x9d\x3b\x12\x84\xb4\xab\xed\x72\x0f\x83\x6b\x7c\x50\xbb\x44\xe9\xb9\x5b\x87\x12\xaa\x35\xd5\xa0\x32\x86\xc9\x23\x0d\x18\x69\xc0\x48\x03\xae\xf4\xd0\x20\x0d\x18\x69\xc0\x48\x03\x46\x1a\x30\xd2\x80\x65\xe6\x4c\x1b\xa7\x01\xb7\x5b\x79\x9b\x68\xc4\x99\x56\x9f\x2a\x09\xce\x94\xb4\x56\x2b\x11\x78\x73\xae\x34\x02\x55\x42\x46\x31\x32\x8a\x91\x51\x5c\xbd\x8c\xe2\xf5\x92\x75\xe5\xc5\x3c\x75\x4e\x2a\x6e\xab\xa2\x6a\x8a\xed\x61\xa2\xe4\x6a\x97\x54\xec\x52\xdd\xf0\x1e\x50\x5f\xe1\x3d\x2b\x74\x17\xd2\x89\xd7\xc4\xce\x88\x87\x10\x2f\xbf\x4e\xee\xe1\x6b\x86\x43\x24\x80\x10\xe1\xa0\x08\x07\x95\x3e\x58\x42\xa6\x74\xe2\x52\x76\xff\xa6\x84\x49\xd4\xd7\x9c\x2e\x51\x54\x04\x12\x89\x37\x01\x3b\x48\x24\xae\x3c\x10\xca\x26\x12\x6f\x0f\x84\x8a\x8f\xd4\x47\x5c\x28\xe2\x42\x91\x48\x0c\xa8\x53\x2a\xd4\x41\xb8\x02\x52\x88\xeb\x83\xa2\x60\x32\xaa\x04\x52\xca\xa5\x10\xef\x00\x95\xd4\x22\xa8\x04\x9b\x11\x6c\x46\x72\x02\x29\x99\x52\x88\x01\xa4\x90\x3c\x2c\x95\xc9\x08\xc9\xc3\x48\x1e\xae\x39\xf2\xc9\x26\x0f\xef\x80\x7c\x0a\x8b\xef\xc2\x48\x04\x23\x11\x92\x87\x81\x6d\x36\xc2\x36\x48\x1b\x46\xda\x30\xd2\x86\xc9\x4a\xcd\xbe\x71\xda\x70\x5b\x15\x85\xc0\x77\x92\xc8\x30\x84\xc0\xc7\x8d\x48\x1b\xe6\x2f\x55\x5f\x77\xca\x18\x05\x8f\x84\x61\x24\x0c\x23\x61\xb8\xd2\x43\x83\x84\x61\x24\x0c\x23\x61\x18\x09\xc3\x48\x18\x96\x99\x2d\x6d\x9e\x30\xdc\xce\xdb\x41\x23\xb6\xd4\x05\x5b\x42\xc2\xb0\x74\x09\xc3\xf5\x25\x49\xc7\xb7\x02\xee\x1b\x1c\x21\x55\x18\xa9\xc2\x47\x47\x11\x55\x41\x3b\xb5\x4e\x15\x6e\x8b\x4a\x3b\x76\x38\x23\x60\xed\x52\x85\xbf\xd8\xee\xf5\x83\xa5\x2e\xa2\x9b\x56\x68\x2f\x24\x0b\xaf\x89\x9b\x11\xff\x20\x5c\x80\x9d\x56\xee\xe1\x6b\x86\x3f\x24\xde\x66\x04\x7e\x22\xf0\x53\xfa\xe0\x08\x99\x92\x85\x4b\xda\xff\x9b\x12\x1e\x51\x5f\x63\xba\x44\xd1\x10\x48\x17\xde\x04\xee\x20\x5d\xb8\xf2\x50\x28\x97\x2e\xbc\x3d\x14\x1a\x14\x41\x21\x44\x82\x22\x12\x14\xe9\xc2\x00\x3b\xcb\x0b\xd2\x86\xc9\xd4\x2a\x5c\x01\x09\xc3\x99\x65\x28\x37\x8e\x82\xd9\xa8\x12\x58\x29\x9f\x30\xbc\x3d\x58\x1a\x16\x81\x25\xd8\x8d\x60\x37\x92\x13\x4a\xc9\x94\x30\x0c\x28\xb5\x11\x94\xaa\x6d\x70\xa1\x44\x66\x23\xa4\x0c\x23\x65\xb8\xe6\xd8\x27\x97\x32\xbc\x3d\xf6\x29\x2c\xb0\x0b\x43\x11\x0c\x45\x48\x19\x06\xba\x59\x5e\x40\xd2\xb0\xc4\x78\x07\x49\xc3\x55\x0a\x83\xdf\x3c\x69\xb8\x23\x0a\x83\xef\x26\x3d\x47\x18\x7c\xdc\x88\xa4\x61\xfe\x52\x1d\xb4\xa7\x8c\x11\xf1\x48\x1b\x46\xda\x30\xd2\x86\x2b\x3d\x34\x48\x1b\x46\xda\x30\xd2\x86\x91\x36\x8c\xb4\x61\x99\xf9\xd2\xe6\x69\xc3\xdd\xbc\x2d\x34\xe2\x4b\x2a\xf8\x12\xd2\x86\xa5\x4b\x1b\xae\x33\x4d\x3a\xbe\x25\x70\xdf\xf0\x88\x25\x0e\x3b\xb6\x8e\xb4\xe1\x66\x62\x20\x09\x50\x44\x55\xd0\x4e\xad\xd3\x86\xbb\x82\xd2\x8e\x49\xa9\x92\x95\x49\xc3\x79\x95\x72\x58\x25\x34\x71\x39\x12\x44\x0a\x55\x10\xbb\x65\x85\x02\x8a\xb7\x99\x97\xe7\xbf\x90\x5f\x02\xc6\xbb\x63\x02\x70\x75\x40\xea\xe6\x46\xfd\xee\xa0\x78\xba\xf4\x93\xf9\x22\x88\x45\x2c\x01\xb9\xd6\x09\x84\x4a\x64\xb4\xaf\xca\x42\x95\x12\x29\x5a\xb6\x4e\xc7\x31\xe6\x4b\xa3\xc5\xa7\x09\x6e\x9c\x38\xf3\xc8\x9a\xe6\xd1\x89\x6d\xe9\x11\x72\x7c\x7a\x77\x47\x1e\xff\x3c\x9f\x7d\xd0\x7c\x4a\xee\xef\x39\x20\xf9\x9f\x5d\xad\x6c\xe5\x80\xc8\xd5\xf6\xf7\x15\x20\xb2\x34\x13\xfc\x31\x81\x67\xac\x11\x60\x67\x93\x1c\x63\xc2\x9c\xc6\xb5\xcb\x85\x30\x7b\xdd\x62\xc8\xa0\x76\xdb\xeb\x61\xcc\x52\x0a\xd3\x74\x0f\xab\xee\x4c\xc3\xf3\xa9\xf5\xa0\x79\x24\xbe\x0d\xf8\xb4\x1c\x7c\xda\xeb\x89\x26\x5b\x07\x00\x95\x54\x14\xa0\x6e\xbb\xe6\x39\x52\x72\x7c\x73\x69\x89\x3b\x02\x80\xb0\x94\x40\x58\x54\x86\x11\x50\x18\x50\x18\x50\x38\xdd\xc9\x66\x42\xe1\x7e\x5b\x84\x4e\xd6\x34\xb7\x56\x11\x0a\x97\x52\x5e\x18\x20\x78\x93\x69\x26\x8c\xbc\xee\x01\x04\x13\x80\x60\x94\x1a\x07\xfc\x05\xfc\x05\xfc\x05\xfc\x05\xfc\x8d\xee\x3e\x08\xfc\x1d\x08\x23\x1c\xfb\xf5\x85\xbf\x65\x57\x28\x07\x00\x5e\x63\xa2\xa9\xa2\x89\x56\x58\x5f\x12\x00\x18\x00\xb8\x5a\xbb\x01\x20\x30\x20\x30\x01\x04\x06\x04\x06\x04\xae\x18\x04\x1e\x16\x23\x93\x35\xcf\xe8\xa9\x5b\xb8\xed\x3f\xe6\xb6\xaf\x1d\x14\xc8\x4e\x58\xa1\x98\x8c\x7c\x0f\x80\x6e\x2f\xb9\x02\x25\x6a\x8b\xaf\x50\x52\x26\xee\x15\x9d\x01\x05\x7c\xbb\x23\xbe\xd5\xa6\x34\x1a\xbd\x94\x72\x44\x78\x2e\x87\x49\x27\xae\xcd\x26\x7d\x1a\x55\xaf\x44\xaa\x57\xf6\x97\x1f\xa9\xa6\xb3\xd7\x4e\x7f\x2d\x04\x0b\xdc\xeb\x4e\x6c\x33\xb3\xe3\xea\xd4\x9b\x08\xe7\x4a\x89\x20\xd8\xf3\x17\xe6\x2a\xcd\xce\x36\xa0\x40\x52\x1f\xd3\xc8\x22\xdc\x29\x68\x82\x11\x7f\xfb\xed\xb7\xdf\xce\xde\xbf\x3f\x7b\xf5\x8a\xfc\xf8\xe3\xd3\xd9\xec\xa9\x97\xc1\x9c\x8e\xe6\xfb\xd4\xb5\x8a\x9f\xb5\xdc\x08\xaf\x0c\x5d\xa7\xd6\xc3\x19\xfe\x71\xb7\xf2\xc0\x6d\x29\x50\xdb\x8d\xd6\x42\x4e\x19\x27\xb5\x13\x3f\xef\xf2\x42\x5c\xa6\x75\x06\x3e\x87\x58\x38\x3f\x31\x83\x0b\x1f\x63\x58\xa9\xbc\x72\x0d\xd3\x24\xba\xfd\x25\x33\x3b\x83\xdb\x7e\x71\xd3\xe4\x3d\x2b\x42\x56\x1c\x91\x7c\x9b\x2d\x03\x57\x8c\x98\x53\x22\xb6\xe6\xb3\x8b\x2c\x4b\x9b\x5b\x06\x07\x8c\x36\x93\xfe\x07\xfa\xc7\x9c\xe6\x2a\x11\x34\x65\x00\x5e\xc8\x33\x00\xe4\x4f\xcd\x1c\x82\x97\xe5\x0e\x41\xa4\xf6\xd8\xc7\xcd\x06\xe2\x9d\x31\x33\x9a\xba\x0e\x5e\x1d\x7f\x1d\x84\xe2\x6f\xea\x2a\x78\x2d\xc3\x2a\x38\xb7\x75\xf9\xa4\x9f\xc6\xe7\x5b\x09\xff\x89\xfe\xe4\xee\x8e\x3c\xfe\x69\x69\xa9\x23\xf7\xf7\xb9\x86\xb3\x8e\x37\xd1\x4c\x7a\x76\x3d\xbf\xa0\xae\x45\x7d\xea\x9d\x4d\xec\x99\x33\xf7\xe9\x99\x4b\x43\x0a\xe5\x9d\x39\xb6\xfe\x3f\x37\x9a\x7b\x96\x18\x00\x13\xf3\xdf\x9f\x83\x0b\x8e\xad\x3f\xfb\x6e\x3c\x9e\xd0\x6c\xe9\x57\x6e\xc4\x9d\xac\x94\x0f\xbd\xda\x24\x1a\x63\x4e\x2c\x4f\x1e\xff\xf0\x64\x73\xb9\x78\xbe\x6b\x58\xd3\xf5\xe4\x12\xfd\xc5\x19\x16\xeb\x6f\x42\xd6\x2e\x4c\x9a\x35\x1e\x7b\xbe\xc6\xc8\x73\x6e\x65\x6d\x68\x57\x16\x96\x66\x48\xb5\xef\x50\xba\x2e\x5d\xdf\x62\xb9\x0c\xc7\x6e\x84\x98\x98\xac\x27\xb6\x9b\xaa\x00\x56\x3b\x71\xbe\x28\x43\x9c\x52\x4f\x5e\xf2\x84\x60\xc8\xd3\x35\xf2\xf6\xbc\x82\x4c\x86\xb5\x9a\x21\xcc\x57\x58\x3f\xcd\x1a\xf0\xd7\x87\x71\x55\x3e\xec\x91\x64\x1e\x84\xdd\x3c\x92\x5b\x96\x00\x72\x35\xcb\x0b\x06\x21\x3f\x04\x31\x70\xca\x34\xc3\x5b\xd9\x14\x6f\xe5\x37\xe4\x58\x0e\xc6\xa1\xa0\x9e\x8f\xda\x4d\x4e\x8b\xdb\x7f\x8c\x5d\xe2\xee\xa8\x5a\x6e\x25\xdc\x93\xfb\x73\x4f\x0e\x05\xc5\x83\xd4\x6e\x52\xd7\x10\x0e\xca\xca\x39\x28\xb7\x2e\xfa\x9f\xb9\x52\x97\xec\x6c\xf8\x41\xe1\x07\x95\xd0\xfa\x0d\x3f\x28\xfc\xa0\xf0\x83\x96\x3a\x04\xf0\x83\x6e\x33\x08\xf0\x83\xc2\x0f\x0a\x3f\x28\xfc\xa0\xf0\x83\x56\xd3\x14\x2e\x93\x1d\x1a\x7e\xd0\x52\xc5\x09\x3f\x68\xe3\x86\x1c\x7e\xd0\x12\x85\x09\x3f\x68\xc3\x06\x1c\x7e\x50\xf8\x41\xe1\x07\x95\xd0\x0f\x3a\x12\x55\x9d\xee\x25\x26\xf0\xda\xf9\x41\xcb\x2f\xb5\x07\x0f\x68\xa9\x1e\xd0\x91\xa8\x3c\x75\x0f\x29\x9a\xf0\x80\xd6\xa0\x34\x1f\x7c\x9f\xf0\x7d\x4a\x68\xf1\x86\xef\x13\xbe\x4f\xf8\x3e\x4b\x1d\x02\xf8\x3e\xb7\x19\x04\xf8\x3e\xe1\xfb\x84\xef\x13\xbe\x4f\xf8\x3e\xab\x69\xfe\x96\xc9\xf6\x0c\xdf\x67\xa9\xe2\x84\xef\xb3\x71\x43\x0e\xdf\x67\x89\xc2\x84\xef\xb3\x61\x03\x0e\xdf\x27\x7c\x9f\xf0\x7d\xca\xe7\xfb\xec\xb4\x44\xc7\x8c\xf5\x0e\x79\xe2\xee\x81\x7d\x9f\xfb\x38\x67\x01\xde\xcf\x32\xbd\x9f\x9d\x96\xe8\x5c\xb2\x5e\x72\x38\x2f\xbc\x9f\xf0\x7e\x56\xf8\x5c\x06\xf8\x3f\xe1\xff\x94\xd0\xea\x0d\xff\x27\xfc\x9f\xf0\x7f\x96\x3a\x04\xf0\x7f\x6e\x33\x08\xf0\x7f\xc2\xff\x09\xff\x27\xfc\x9f\xf0\x7f\x56\xd3\x04\x2e\x93\xfd\x19\xfe\xcf\x52\xc5\x09\xff\x67\xe3\x86\x1c\xfe\xcf\x12\x85\x09\xff\x67\xc3\x06\x1c\xfe\x4f\xf8\x3f\xe1\xff\x94\xd0\xff\xa9\x0a\xce\x99\x5f\xf3\x94\xf9\x3a\x1d\xb2\xf9\x9e\xce\x6c\x77\x81\x03\xe3\x1f\x9a\x31\x82\x03\xe3\x13\x7f\x39\xfc\x92\xd2\x1e\x0c\x5f\x95\x45\x29\xe5\xa1\xed\x09\x86\x9b\xb1\xad\x22\x82\x95\x17\x0b\x7f\x05\x78\x23\x9b\xa0\xb7\x53\x12\xff\xc6\x7f\x3d\xfb\x1e\x67\xb2\x97\x79\x26\x3b\xbf\xbd\xef\x06\x00\x71\x2c\x3b\x9b\xf2\x00\x79\x4d\x3c\x96\xbd\xa3\x0a\x8e\x65\x57\x7b\xdd\xf5\x30\x63\x29\x11\x73\xfd\xc3\xaa\xb4\x7d\x9c\x9a\x00\xbc\xb9\xde\x7c\x13\x1d\xd4\xde\xeb\x01\x71\xca\x8f\x38\xb7\x5d\xe0\xed\xcc\x95\xba\x9c\x78\x00\x60\x0b\x60\x0b\x60\x0b\x60\x9b\x7d\x13\x00\x5b\xee\xa1\x47\x01\xb6\x6d\xd1\x71\x60\xbd\x35\x8d\xa1\x55\x04\xb6\xe5\x96\xc1\x03\xa4\x5d\x6f\xa6\x89\x0e\xf7\xea\x0d\x00\x69\x01\x69\x2b\xb3\xee\x01\x66\x8f\x01\x66\x1d\x5b\x07\x94\x05\x94\x05\x94\x5d\xa7\x93\xcd\x84\xb2\x1d\x61\x45\xe7\x43\x9e\x6c\x7b\x60\x28\x5b\x76\x56\x33\xc0\xec\x7a\x73\x4d\x58\xa7\x19\x27\xd5\x02\xcc\x56\x69\xe5\x03\xce\xc2\x36\x0b\x40\x0b\x40\x9b\x7b\x13\x00\x5a\xee\xa1\x47\x01\xb4\x5d\x41\x99\x9e\xc1\x7a\x70\xb6\x86\x61\xaa\xa8\xb7\x53\x2a\x8a\xed\x0a\xea\xed\xa0\xda\x4e\x85\xab\xed\x54\x65\x6d\x1f\xab\x14\x0e\xa7\xdb\x50\x09\xa7\xa0\x5b\x85\xe8\x6d\x29\xd2\xa3\x27\xe1\x57\xb0\x18\x8e\x4e\x27\x21\x8a\xdb\x78\x0c\x50\x12\x47\xaa\x61\x68\x6a\x49\x10\x29\x0a\xe3\x44\x63\x81\xda\x38\xa5\x8d\xc3\xf6\x0b\x02\x15\x72\x4a\x1c\x06\x54\xc8\x41\x85\x1c\x54\xc8\x29\xb5\x42\xce\x56\xe6\xe3\x3d\x5b\x8f\x6b\x5d\xff\x26\x12\x73\x79\x02\x96\x54\xa2\xa5\x94\xc0\x39\xe2\xec\xdc\xa4\xc0\x4d\x53\xc6\xf4\x50\x35\x6e\x9a\x22\xcf\x52\xca\xdc\xc8\xbd\x46\x1a\x36\xa2\x52\xd4\xb1\xe1\x3d\x00\xbb\x39\x15\x51\xca\x86\xfd\x87\x52\x36\x75\x28\x65\xd3\x13\x94\xb2\x51\xfb\x89\x0d\x7b\xff\x41\x6f\x89\x47\xb2\x6a\xa9\x89\xf0\x30\xee\xd5\xc3\xd8\x13\xd4\xcd\x51\xfb\x9c\xef\x0d\x3e\xc6\xaa\xf9\x18\xb7\xdd\x27\x3a\x99\x2b\x75\xc9\x6f\x86\x2b\x13\xae\x4c\x29\x6d\xd5\x70\x65\x4a\x31\x0c\x70\x65\x4a\x32\x10\x70\x65\xca\x31\x0e\x70\x65\x4a\x31\x0c\x70\x65\xc2\x95\x09\x57\x26\x5c\x99\x70\x65\x36\xce\xed\x05\x57\x66\xfd\xc6\x14\xae\xcc\x72\xe5\x09\x57\x66\xdd\x46\x14\xae\x4c\xb8\x32\xe1\xca\x94\xd4\x95\x29\xaa\xb1\xdc\x4f\xd2\xee\x6b\xe7\xca\xdc\x4b\x29\x3a\x38\x31\xcb\x75\x62\x8a\x8a\x31\xf7\x91\x28\x09\x27\x66\x0d\x2a\xda\xc1\x7d\x09\xf7\xa5\x94\xf6\x69\xb8\x2f\xa5\x18\x06\xb8\x2f\x25\x19\x08\xb8\x2f\xe5\x18\x07\xb8\x2f\xa5\x18\x06\xb8\x2f\xe1\xbe\x84\xfb\x12\xee\x4b\xb8\x2f\x1b\xe7\xea\x82\xfb\xb2\x7e\x63\x0a\xf7\x65\xb9\xf2\x84\xfb\xb2\x6e\x23\x0a\xf7\x25\xdc\x97\x70\x5f\xca\xe9\xbe\xec\x8b\x4e\xd2\xea\x1f\xf2\x88\xd8\x03\xbb\x2f\xf7\x74\xfc\x00\x1c\x98\xa5\x3a\x30\xfb\xa2\xa3\xb7\xfa\x38\x4d\x16\x0e\xcc\x5a\x9c\x62\x00\x17\x26\x5c\x98\x52\xda\xa8\xe1\xc2\x94\x62\x18\xe0\xc2\x94\x64\x20\xe0\xc2\x94\x63\x1c\xe0\xc2\x94\x62\x18\xe0\xc2\x84\x0b\x13\x2e\x4c\xb8\x30\xe1\xc2\x6c\x9c\xbb\x0b\x2e\xcc\xfa\x8d\x29\x5c\x98\xe5\xca\x13\x2e\xcc\xba\x8d\x28\x5c\x98\x70\x61\xc2\x85\x29\xa7\x0b\x73\x20\x3c\x41\x3d\xfe\xdc\x98\x13\x27\x7f\xa2\xfe\x17\xdb\xbd\x6e\xd8\x49\xe8\xfd\xcd\xe7\x8c\xe8\x24\xf4\xc4\x13\x0d\xd7\x62\xda\xb5\xf8\x90\x4b\x71\x5d\xd7\x61\x3b\xdd\xbe\x89\xeb\xb0\x2a\xcb\x51\xca\xd3\xc9\x0d\x57\xf3\x29\x87\x2a\xad\x70\xab\x18\xbb\x74\x42\x8d\x9b\x08\x58\x8e\xd9\x7c\x2b\x07\xc9\x7d\xea\xcd\x3e\x3f\x3a\xf0\x09\xe5\x8e\xad\x17\x9c\x4f\xce\x99\x03\x4a\x06\x6c\x5b\x9e\x35\xfe\x21\x94\x39\x79\xa1\x59\x7a\xb8\x22\x77\x82\x73\x4d\x3d\x70\x9c\x33\xd4\xe5\xd0\x4a\x32\xc1\x5e\x38\x7b\x3d\x8a\x5c\x69\x29\x35\x84\x73\xfc\x10\xec\xe7\x30\xf2\x3a\xc3\x8b\xa1\xa8\x80\xfc\x10\xf0\x02\xf0\xa2\x51\xf0\x82\x19\x0d\x66\x86\x5f\x0f\x7c\x41\x1c\x5b\x27\x55\x40\x18\x1f\x23\xb1\x03\x62\x94\x65\x15\x02\x8c\x90\x12\x46\x1c\xc4\xc4\x34\x14\x86\x22\xf7\x13\x75\xb3\xf7\x28\x79\xce\xa0\x55\xb5\xb3\x46\x60\xa0\xda\x74\xc6\x09\xcb\xca\xc1\x42\x25\x31\x84\xdc\x7a\x69\x0f\x32\x97\xea\x72\xc6\x50\x95\xa0\x6a\xad\x2c\x61\x95\x41\xaa\xb0\x85\x95\x02\x54\x61\x0b\xab\x30\x88\xad\x33\x92\x19\x89\xdc\xb3\x7d\x18\xc3\xea\x88\x64\x86\x40\x32\x47\x47\x32\x30\xba\xc1\xe8\x56\x61\x2c\x03\xbc\x92\xee\xa6\x74\x78\xe5\x10\x46\xb7\x6e\x4b\x78\x48\xf8\x28\x51\x6b\xb5\x33\xba\x95\x5b\x5a\x1f\xe6\xb6\x35\xe7\x9a\xc8\x61\x3b\x68\x01\xa4\xd6\x10\xa4\xca\x64\x6e\x2b\xa7\x1a\x7e\x95\xe0\x29\x0c\x6d\xc7\x40\xa7\x30\xb4\x95\x02\x4e\x61\x68\xab\x30\x70\xad\x35\x86\x11\x39\xa9\x07\x2a\x30\x4c\x0d\x31\x8c\x4c\x86\xb6\xe6\x61\x18\x98\xd8\x60\x62\xab\x30\x8a\x01\x52\x49\x77\x53\x3a\xa4\x72\x10\x13\x9b\xda\x11\x41\x86\x43\x1e\x5e\x79\x68\x13\x5b\xd9\xe5\x5f\x61\x64\x5b\x73\xb6\x89\x3c\xc1\x03\x9c\x48\x59\x47\x80\x2a\x93\x91\xad\xac\x8a\xad\x55\x82\xa8\x30\xb3\x1d\x03\xa1\xc2\xcc\x56\x0a\x40\x85\x99\xad\xc2\xe0\xb5\xce\x28\xa6\x2d\x72\x4b\x0f\xba\x40\x31\x35\x44\x31\x32\x99\xd9\x9a\x88\x62\xea\x65\x68\xab\x48\x81\x0a\x98\xd9\xca\x41\x31\x40\x2a\xe9\x6e\x1e\x1f\xa9\x7c\x13\x3d\x36\x58\x73\xc1\x72\x52\xd8\x11\x36\xe1\xd8\x28\xde\xe4\x8a\xce\xb4\x5f\xa9\xeb\x45\x2a\x62\x18\x36\xfb\x8b\x70\x59\xe8\x9a\x7b\x1d\xde\xe9\x6b\xd3\x64\xd8\x95\xb0\x1a\x73\x24\xb8\xe5\x46\xac\xc4\x3f\xe5\xd3\x99\x63\x6a\xbe\x61\x25\xbb\xbf\x62\x1a\x9e\xcf\xcd\x9b\xbb\x22\x48\x70\x57\xa4\x0b\xce\x5d\x7b\x46\xfd\x2b\x3a\x17\xf9\x5b\xb8\x1b\x8a\x86\x5e\xb9\x32\xd8\x92\xe0\x81\x8d\x61\x4d\xcc\xb9\x4e\x9f\x9b\x45\x70\xa1\x78\x06\x28\xb3\xb9\xe9\x1b\x05\xb7\x47\x2b\x4b\x29\xc0\x65\x29\xfd\xcf\x97\x48\x56\xfe\x98\x53\x37\x40\x57\x8a\x53\xf4\x76\xdc\x58\xa9\xa9\xd6\x29\x0d\xe6\x1a\x5f\xb4\x51\xf1\xae\x0d\xe7\x17\xd7\xfc\x79\x61\x4d\x0a\x3a\xb7\xdc\x51\xb8\xce\x65\x97\x74\x6a\x8a\x99\xbf\x46\x52\xcd\xbc\xbc\x68\x8c\xa2\x59\xf1\xb9\x50\x8b\x67\x8b\x78\x0b\xc6\x2f\x77\x5b\xa9\xa3\xa8\xc4\xda\x52\xd9\x60\x30\x0b\xbf\xc4\x8d\x25\xf7\x22\xbc\x40\xd2\xf8\x26\x5b\x9b\x73\x5d\xd1\xac\x27\x9c\x64\xdb\xe0\x36\x0e\x7e\x6a\xad\xf8\x8d\x35\x67\xcd\x64\xee\xf9\xf6\xac\xdc\x19\x13\x09\xe0\x79\xb6\x04\x7b\xfc\xc6\xdf\x8d\xc7\x9a\x69\x16\x4f\x82\x87\xa9\x10\x43\xa0\x97\x86\x65\x44\xa0\x37\x9c\x07\xe3\x50\xa5\x25\x15\x5e\x0d\xeb\xd2\xe6\x40\x55\x01\xa6\x5a\xa2\xcb\xc7\x3f\x7c\x7f\x7f\x4a\x32\x78\xe9\xc1\xd9\x98\xd1\xa8\xf1\x64\x14\x96\x5e\x5c\x67\x83\x59\xf1\xdd\x87\xb6\x99\xfd\x48\xe1\x81\x6d\xea\xc9\xc9\xe3\x1f\x1e\x15\x41\xd1\x27\xeb\x4f\xc5\x08\x4f\xf1\xcf\xf7\xb5\x29\x9b\x73\xde\x3f\x96\x6f\xa7\xa4\xaf\xe6\xc4\x10\xb4\x15\xdf\x1c\x4d\xf4\x50\x4e\xdc\x85\xb9\x47\x3f\x86\x0f\x4a\x71\x46\xf6\x6f\x00\x8a\xee\x43\x3d\x67\xb0\x91\x89\x34\xdc\x65\x08\x49\x15\xcb\xfe\x72\xa6\x2e\xe1\x9b\xe2\xdb\x51\x9b\x92\xfa\x9a\x63\x4c\xae\x19\x6d\x8b\xbe\x1c\x89\x72\xbc\xc4\xd8\xfc\x2e\xa3\xf4\x12\xe5\x10\xeb\x6e\xf6\xa1\xc3\x7f\x50\x67\xc9\xdf\x3d\xee\x6f\x95\xff\xd0\x69\xf1\x57\x38\x90\xd9\xe6\xfe\x56\xf5\x70\xf9\x7d\x5e\xbe\x43\xc0\x08\xf2\xbb\x9f\xf8\x57\xf8\x07\xf7\xf9\x07\xf3\xbf\xd2\xee\xf2\x1f\x92\x53\x17\x94\x81\xce\xf7\x77\xd9\x97\x94\xf8\xbe\xda\x8c\x7e\x29\x11\x38\x59\xc2\xf8\x10\x97\x90\x27\xe4\x45\x04\x49\xd8\xe5\x9b\x04\xdf\x7c\x73\xff\xcd\xff\x07\x00\x00\xff\xff\x7f\x05\xdb\x96\xee\xd8\x03\x00") +var _monitoringBackendGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x93\x9b\x46\xba\xfe\x9e\x5f\xd1\x87\x64\x37\xe3\x9c\x19\x5b\xe8\x2e\x57\xb9\x4e\xf9\xba\xf1\xae\x9d\x9d\x75\x9c\x9c\xcd\x71\x5c\x5a\x46\xf4\x68\xd8\x41\x40\x00\x8d\x47\x9e\x9a\xfd\xed\xa7\x68\x10\x34\x97\xd6\x15\x69\x1a\x78\xf2\x21\x1e\x35\x08\x35\x6f\x5f\xde\xe7\x79\x6f\x7d\xf7\x0d\x21\x8a\x66\x59\xb6\xaf\xf9\x86\x6d\x79\xca\x53\x12\x34\x11\xa2\x98\x86\xe7\x2b\x4f\xc9\x27\xf6\x89\x44\xad\xec\xca\xc5\xdc\x30\xfd\xb7\x96\xf2\x94\xa8\xa7\x49\xab\xae\xf9\x9a\x67\xcf\xdd\x09\x55\x9e\x12\xe5\xec\x8c\xfc\xc5\xd5\x2e\x35\x4b\x23\x67\x67\x0a\x77\x1b\xb5\xb4\x0b\x33\xb8\xc5\x77\xe7\x94\x6b\xbf\x32\xf4\x82\x56\x63\x62\x5b\x2f\x6d\xd3\x76\x83\x67\xba\xd3\x0b\xed\xa4\x75\x4a\xda\xaa\x7a\x4a\xda\xbd\xde\x29\x51\x1f\xf1\x8f\xb6\xb4\x19\xfb\xed\xe7\xc9\xeb\x90\x3f\x93\xe7\x26\x75\x7d\x8f\xbf\xcf\x5f\x38\xec\x3e\x5d\xf3\xae\x2e\x6c\xcd\xd5\x95\xe8\xda\x3d\xfb\xf7\xf3\x37\x84\xdc\x07\xb7\x2b\x54\x37\xfc\x4c\x6f\x95\xa9\x45\xfd\xb7\xba\xf2\x94\x58\x73\xd3\x0c\x5b\x5c\xcd\xb9\xfa\x68\xdb\xa6\x6f\x38\xca\x53\xd2\x62\x8d\x46\x70\xcb\x90\xfd\x69\x1a\xd6\x75\x20\xd7\x4f\x9f\xd9\x47\x47\xb3\xa8\xe9\xc5\x92\x5d\xca\x55\x99\xd8\xa6\xa9\x39\x1e\x0d\xbe\x78\xa9\x99\x5e\x2c\x06\x65\xea\x1a\xfa\xb9\x9d\x0c\x4d\x28\xaf\x8c\xf8\xbf\x28\x4f\x49\xbb\xcb\x35\xdc\x2e\xfb\x12\x7d\x5e\x04\x9f\x97\x2f\x1a\x3f\x9b\xf5\xb3\x1d\xdf\xc7\xf5\xee\x73\xdc\xe6\x1b\x3e\x93\x81\xf2\xce\xf0\x7c\x6a\x51\x37\x91\x66\x2c\x4b\xd7\xfe\x12\x4a\x31\x7a\x74\xfc\x5a\x9a\x69\x68\x1e\x1b\x42\xf6\x02\xc9\x2f\x5f\x68\xac\x25\xfd\xaa\xc1\x90\xbc\xa3\xd6\xd4\x67\xaf\xd7\x4a\xb5\xd3\xa2\xdb\xf9\x39\xf7\x1d\xf7\x31\xbe\xe5\xd2\x30\x4d\x5e\x54\x62\x69\x0e\x33\xd2\x54\xdb\x6b\xa4\xa9\x16\x4b\xb3\x33\x8a\x3f\x9b\x74\x4a\x2d\x3d\xfd\x53\xda\xcd\x34\xfb\x1e\xc1\xe8\xcf\x5d\x97\x5a\x7e\xc1\x95\x99\x76\x5b\xd4\x6a\x58\x05\xad\xde\x95\xfd\x25\xbf\x88\x7c\xdb\xd7\xcc\x82\xbb\x6f\x34\x73\x9e\x08\x35\xf7\x32\xa6\x61\xb1\xab\xfc\xd3\x58\xe3\x17\x43\xf7\x53\xd3\x2f\x33\xc5\x59\x53\xb0\x3c\xce\x6d\xc3\xf2\xdf\xdb\x6c\x61\xb3\x86\x64\x58\x6c\x27\xde\x6e\x92\x5f\x74\xa8\x3b\xa1\x96\xaf\x4d\x69\x6e\xa4\x9d\xe0\x51\xae\xa6\x1b\xf3\xe0\x3b\xed\x74\x7b\x7e\x62\xb8\xd4\xd2\xa9\x4b\xd9\xb6\x71\x69\xda\x7e\xf2\xc3\x1e\x75\x0d\xea\xfd\xfd\x86\xba\xae\xa1\xd3\x4c\xa7\x3d\x47\x9b\xd0\xa2\xf9\xe7\xf9\xda\xe4\x3a\xf7\x2b\x9e\x4f\x1d\x87\xea\xef\x0c\x2b\xdf\x61\x5f\x73\xa7\xd4\xf7\xb8\x1d\x94\xdf\x43\x83\xcd\xe5\xd6\x61\xdd\xf3\xe6\xb3\x13\x57\xf3\xe9\x89\xe6\x18\x9e\x6d\x69\xbe\xed\x8e\xcd\x68\xa1\x8d\x5d\xea\x39\xb6\xe5\xd1\xf1\xc4\xd6\xa9\x77\x17\xec\x70\xac\x8f\xcf\x7e\x57\xbe\x8b\x3f\xfc\xae\x9c\xba\xf4\x8f\x39\xf5\xfc\x71\xb0\x1c\x9f\xfd\xae\x68\x73\xff\xca\x76\x8d\xaf\xf4\x77\xe5\xfe\x93\x3a\xfb\xfc\x88\xdf\x27\x83\x45\x61\xbb\x33\x2d\x98\x6c\x8a\x6f\xcc\xe8\x38\x94\x49\xfa\x16\xc3\xf2\xa9\x7b\xc3\xe6\x8d\xa2\xce\x8a\xaf\xbd\xd1\x26\x3e\xdb\x9a\xd5\x56\xea\x7a\x38\xed\xdf\xc4\x3f\x12\x77\x27\xfd\x18\x97\x5e\xb2\x9d\x54\x79\xae\xc4\xcd\xf7\xa7\x0f\x23\x2d\x97\x3a\xf2\xc8\xca\xa5\x8e\x40\x52\x2f\x1e\x56\x52\x2e\x75\x6c\xd7\x97\x43\x50\x61\x5f\x04\x72\x7a\xf9\xf0\x33\x8a\x4d\xf8\xb1\x1d\xfc\x29\x87\xc0\x32\x9d\x12\x48\xee\xd5\xc3\x4b\xce\xa5\x8e\x6c\x72\x8b\xbb\x24\x90\xda\x6b\x4e\x6a\xd1\x5f\x1c\x7e\xba\x72\xa9\x77\x65\x9b\x7a\x0e\x57\xcd\xe8\x1b\xd7\x9e\x71\x60\x32\x6e\xff\x40\xa7\x91\x7e\xcc\x7c\xe1\xe7\x2b\xe3\xd2\xcf\x7f\x23\x83\xd0\x48\x24\x56\x8f\x38\xd4\x25\x1e\x9d\xd8\x96\x4e\x4e\x2e\x16\xcb\x76\x12\x88\xfb\x11\x07\xe3\x62\xf8\x7a\xc7\xa3\x09\xcd\x65\x70\x34\x83\x27\xbc\x60\xdd\x71\x4a\x78\x09\x25\xc6\x4b\x30\x68\x58\xba\x71\x63\xe8\x73\xcd\x54\x72\xa8\x62\x79\x0f\x43\xcd\x49\x07\x6e\xb5\x5b\x23\x83\xc9\x2e\xe6\x93\xeb\x50\x83\xf2\xef\x1a\x60\x9f\x08\x51\x04\xe2\x28\xc0\xff\x99\xbb\x8b\x31\x51\x8c\x7d\x3e\x7d\xce\x75\x71\xa1\xdd\xd2\x15\x8a\x5b\xa7\x13\x63\xa6\x31\x90\xdc\x12\xcc\x4b\x97\xfe\xe1\x64\x66\xa4\xa9\x5d\x50\x33\xd7\xbb\xe0\x82\x3d\x7d\xa1\x79\x34\x0d\xe7\x63\xdc\x97\xbb\x3d\x04\x7e\xe9\x1f\xe6\x5e\x71\xed\xe2\x4d\x3a\xe9\x5d\xe5\xf6\xcf\x72\x3b\x99\x6b\x2e\xec\x67\x6e\xb9\x2c\xf2\x53\x41\x33\x8d\x69\x11\xde\x65\xed\xef\xe8\x4d\xdc\xe9\x6f\xf8\x87\xd6\x98\x8a\xa4\x1a\x56\x70\x91\xae\x5a\x2a\x17\x09\x48\xfa\xeb\x99\xe3\x2f\x8a\xf9\xfb\xff\x51\xd7\xce\x5f\x01\x81\x01\x81\xd9\x00\x06\x78\x0e\xbb\xe7\xd9\xef\x4a\xfb\xf6\x56\x0e\xcd\xdf\xfe\xe7\x3f\xa5\x61\x2c\x89\x78\xba\xad\x8e\x1c\xe2\xe9\xb6\x3a\xd2\xd0\x14\x5e\x3c\x5d\x59\xc4\xd3\x95\x86\x9d\xf0\xe2\x19\xc9\x22\x9e\x91\x34\x14\x24\x11\x4f\x4f\x96\xbd\xa7\x27\xdc\x7b\x2a\xc4\x35\x42\xf9\x93\x40\xb6\x20\x1b\x20\x1b\x20\x1b\x95\x24\x1b\x59\xbf\xc7\x48\xc0\x35\x3a\xa5\x72\x0d\xd0\x06\xd0\x06\x3a\x0e\xb6\x5d\x6f\x1c\x2a\x15\x6f\x1c\xee\xd2\x7b\xb8\x41\xc8\xc5\x82\x9c\x98\x54\xa8\xda\xaf\xa8\xe6\xcf\x34\xe7\x70\x6a\xfd\xee\xee\x5f\x77\x77\xc4\xa4\xe4\xfe\xfe\x5f\xf7\xf7\x1b\xb0\x8b\x87\xd4\xf0\xcf\x97\xf2\x5b\xaf\xe2\x83\x87\x91\x70\x78\x88\x61\x45\xb7\x78\x50\xfa\x50\xfa\x4d\x52\xfa\x13\xcd\xd5\x33\x0f\x0e\x9a\xce\x35\x5d\x37\xac\x69\x7e\xe6\x04\x17\x3f\xd8\x73\x4b\xcf\x3c\x3c\xee\xe9\x24\x8a\x7d\xc9\x3c\x30\x0e\x89\xf9\xf6\xcd\xf3\x57\xaf\xdb\xcf\xf9\x39\xca\xbe\xf2\xf3\x44\x0b\x97\xb0\xf7\x47\x6a\x04\x96\x57\xaf\xe8\x2c\x5a\x47\x3e\x75\x1d\xdb\xd4\x7c\xfa\x77\x57\xb3\xa6\x29\x46\x13\x6c\xd5\xb6\x15\x6a\xe7\xd6\xe3\x5e\xc1\xfa\xb0\x1d\x6d\x62\xf8\x8b\xfc\x1a\x0c\x10\x49\xb2\xe9\xf9\xde\x72\xa5\x6d\x83\x60\x4a\xb4\x8e\xe6\x11\xcb\x72\xa3\x4f\xe9\xe1\xa5\x0d\xf3\x45\xbc\x2f\xa4\x75\xdb\x95\x31\xbd\x32\x8d\xe9\x95\xff\x32\x1a\xe7\x14\x44\x08\x41\x50\x6f\x25\x08\x8a\xe6\xa7\x10\x78\x64\xd1\x44\x21\x5c\x70\xe9\x0d\x75\x3d\xfa\x9b\xa8\x9b\x50\xc1\x87\x56\xc1\x2b\x54\xed\x3e\x1a\x35\x7a\xf5\x72\x34\x6b\x91\x4e\x0a\x5a\x7f\x34\x3c\xdf\x9e\xba\xda\x4c\x38\x0f\x97\x0a\x34\x3b\x12\xca\xed\xf3\xdc\xbe\x99\xdf\x70\x93\xe7\xdc\x86\x13\xf4\xa7\xf9\xec\x82\x21\xd2\x94\x48\xa2\x8b\x3f\x1b\x5f\xb3\x1a\x55\x59\xe4\x7f\xa6\x58\x23\xf2\xaa\x86\xdf\xb7\x0a\x75\x49\xa1\x26\xc9\x2b\x3b\x91\xe4\x1c\xd3\xf0\xe3\x19\x56\xb8\x57\x2f\xc2\x37\x7a\x11\xed\xe7\xc1\xd4\xb7\x95\xec\xd5\x62\x61\x2c\x72\xc2\x28\x54\x2f\x35\xe6\x94\xea\x20\x27\xcf\x70\x3f\x1d\x80\x54\x82\x54\xca\xa6\xd1\xf8\x68\xb1\x3a\xe8\xb3\xe3\x52\x4a\x97\x3a\x20\x94\x61\x3b\x08\x25\x08\x25\xff\xd0\x87\x27\x94\xbd\xc1\xa8\xfb\xa6\x0d\x42\x19\x37\x14\x87\xdb\xe4\xe1\xca\xa1\x18\x65\x6f\x75\x3a\x41\x23\x19\x65\xdd\xf4\xef\x5e\x7c\x52\xa4\x4e\xc1\x26\x09\xd8\x64\x33\xd9\x64\xbb\x97\x93\x67\xb8\x97\xb6\xc0\x26\xc1\x26\x65\xd2\x66\x99\x8c\x9a\x3a\x28\xb3\xe3\x91\xc9\x0f\x4c\x78\xe0\x92\x61\x3b\xb8\x24\xb8\x24\xff\xd0\x87\xe7\x92\x6f\xda\xdd\x51\xef\x25\xb8\x64\xdc\x50\xc8\x25\x0b\xc0\xca\xc1\xb8\xe4\x10\x5c\xb2\xe6\xda\x77\x67\x2a\xb9\x42\x99\x82\x49\x12\x30\xc9\x66\x32\xc9\x4e\x27\x27\x4f\xb6\x93\xf6\xc1\x24\xc1\x24\xa5\xd2\x65\xa2\x9a\x03\x75\x50\x6a\x0f\x11\xf2\x7a\xc2\xc4\xf8\x08\xec\x32\x6c\x07\xbb\x04\xbb\xe4\x1f\x2a\x01\xbb\x7c\x33\x1a\x76\x5a\x60\x97\x71\x43\x21\xbb\x2c\x00\x30\x87\x62\x97\xfd\xd5\xc5\x06\x1a\xc7\x2e\xeb\xac\x91\x4b\x88\x80\x5d\xa5\x60\xc1\x38\x09\x18\x67\x33\x19\x67\x57\x50\xca\xa5\xdf\x06\xe3\x04\xe3\x94\x4d\xbf\xe5\x6b\xb5\xd5\x41\xbb\x1d\x3f\x1e\x16\x6c\x13\x6c\x13\x6c\x53\x62\xb6\xf9\x62\x38\x18\xbc\x1a\x81\x6d\xc6\x0d\x85\x6c\xb3\x00\xbc\x1c\x8c\x6d\xae\x2e\x37\xd1\x48\xb6\x59\x4f\x6d\xbc\x77\x74\x2c\x98\x66\xea\x22\x98\xe6\x16\x4c\xf3\x18\xc7\x32\x74\x45\xe1\xaa\xbd\x24\x9b\x7c\xb3\x83\x19\xc8\xdb\x60\x79\x5a\x9a\x49\x9e\x9f\xbf\x55\x72\xb3\xab\x89\xa7\x34\x74\x05\xe5\x8a\x54\x15\xe7\x34\x80\x50\x97\xa4\xc2\x8d\x68\xd9\x8d\x35\xc7\xd8\xb9\xf4\x39\x3b\xa8\x46\x8e\xea\x83\x5a\xf6\xd0\x9c\x43\x14\x3f\x2d\x45\x68\x8e\x63\x1a\x13\x76\xd8\xcf\xf8\x9a\x2e\x64\x11\x5f\xa6\x57\x02\x41\x96\x55\x26\xb5\x6c\x41\xba\xf4\x92\xba\x2e\x75\xc7\x97\x86\xe9\x53\x57\x42\xa1\x66\x7b\x28\x10\x70\x59\x85\x56\x4b\x16\xb0\x7c\x02\x15\x09\xb0\xac\x52\xac\x65\x08\x90\xba\xae\x2d\xcb\x5c\x0c\xfb\x22\x10\xda\x6b\x99\x84\x76\x43\x2d\x59\x94\x4a\xd8\x17\x81\xd0\xde\x48\x24\xb4\x19\xf5\x5d\x63\x22\x89\xd4\xa2\xce\x08\xc4\xf6\x17\x89\xc4\xe6\x51\xf7\xc6\x98\xd0\xb1\x6f\x5f\x53\x59\xf6\xb8\x74\x9f\x04\x42\xfc\x51\x3e\x21\xca\x25\x3e\x91\xe0\xde\xca\x24\x38\x5f\x93\x65\xa3\x63\x5d\x11\x88\xec\xaf\x12\x89\x6c\xee\x69\x53\x3a\x36\x8d\x99\x21\x8b\xe4\xf8\x1e\x09\x04\xf8\x37\x99\x04\xe8\x1b\xa6\xf1\x95\x21\x28\x49\xe4\x97\x74\x48\x20\xbe\x77\x92\x38\x3e\xe3\xea\xf1\xbc\xc5\x0a\xc7\x56\x71\x6f\x01\x5f\x67\x61\x27\x1b\xe7\xeb\xac\x91\x6d\x36\xef\x2f\x14\x19\x67\xdb\xe5\xe6\xd7\xe0\xe0\xaa\xf8\x32\x2c\xba\x87\x84\x07\x8d\x3c\xc5\x6a\x6f\x59\x35\xe8\x48\xab\x12\x64\xd5\x98\xf3\xad\x4a\x90\x55\x63\x0e\xbb\xda\x5b\x56\x38\xf9\x8a\xbb\x50\x2e\x77\xc1\x31\x58\x04\xe4\x05\xe4\xa5\xe2\xe4\x25\x1b\x58\xd2\x13\x94\x2c\x57\x07\xab\xcf\x80\x40\x60\x09\x68\xc8\x9e\x6a\x7d\xbf\x40\xd1\x74\x9c\x49\x1d\x22\x44\x8f\x98\xaf\xc1\x84\x87\x34\x8d\xb0\x1d\xda\x1f\xda\x9f\x7f\xe8\xc3\xa7\x69\xe0\x3c\xac\x4d\xcc\xae\x05\xd0\xe5\x50\x69\x1a\xea\xa0\xbf\x12\x0d\x55\x28\x4f\x03\xba\xf8\xb0\xd9\x1a\x62\xd5\x8a\x24\x0d\x76\x09\x49\x1a\xcd\x2b\x07\xd0\x17\xe4\x86\xa8\x03\x9c\x8c\x05\x96\x29\xb1\x66\x13\x05\xe6\xd7\x41\xc7\x1d\x91\x6f\x26\x62\x24\x7f\xa3\x0b\x30\xcf\xa8\x1d\xcc\x13\xcc\x93\x7f\x28\x98\x67\x35\x98\x67\x01\x9c\x39\x1c\xf3\xac\x4d\xb5\x73\xe8\xe7\x58\xdc\x07\xe6\xa0\x9b\xa8\x5b\xb0\x51\x02\x36\xda\x4c\x36\x3a\x10\x94\x43\x57\x07\x48\xa6\x07\x1b\xad\x86\xb6\x13\x66\x37\xd7\x41\xf3\x1d\xf3\xf0\xad\x50\x8c\xe4\x4d\x28\x46\x30\xd3\xb0\x1d\xcc\x14\xcc\x94\x7f\x28\x98\x69\x35\x98\x69\x01\xb4\x39\x18\x33\x1d\xae\xce\x6e\x01\x33\x6d\x84\xae\xde\xe3\xa8\xae\x0d\x54\x2f\x58\x2a\x01\x4b\x6d\x26\x4b\x1d\x0a\x4a\xa8\xab\xc3\xd5\x27\x54\x80\xa5\x82\xa5\x4a\xa2\xf9\xea\xa5\xe9\x1e\xc4\x5f\x0a\x46\x1a\xb5\x83\x91\x82\x91\xf2\x0f\x05\x23\xad\x06\x23\x2d\x80\x31\x87\x63\xa4\xab\x4f\x97\x01\x23\xad\xa5\x5e\x2e\xc3\x4f\x0a\xf6\x09\xf6\x09\xf6\xb9\x7c\xfb\xe0\x0b\xa2\x9a\x36\xc3\xd5\x27\x56\x80\x7d\x82\x7d\x3e\xa4\x96\xcb\xd4\xd7\xad\x83\x7e\x3b\x1e\xef\x7c\x6b\xf9\x74\xea\x86\x81\x43\xaf\x99\x20\x37\xab\x09\x01\x06\x0a\x06\x9a\xeb\x24\x18\x28\x18\xa8\x1c\x0c\x34\x0f\x65\x0e\xc7\x40\xbb\x60\xa0\x4d\xd2\xcd\x3b\x73\xcf\x2d\x54\x2d\x58\x28\x01\x0b\x6d\x26\x0b\x1d\x89\xaa\x13\x0d\x51\x9d\x08\x2c\x54\x62\x4d\x97\x3e\xb0\xa4\x0e\x9a\xee\x78\x2c\xf4\x35\x13\x1e\x58\x67\xd8\x0e\xd6\x09\xd6\xc9\x3f\x14\xac\xb3\x1a\xac\xb3\x00\xba\x1c\x8e\x75\xa2\x3a\x51\xa3\x74\xf1\xce\xac\x73\x85\x6a\x05\xcb\x24\x60\x99\xcd\x64\x99\x6a\x4b\x54\x9e\x68\x88\xf2\x44\xa0\x99\xf2\xaa\xb6\xec\x11\x8f\x75\xd0\x6d\xc7\xe3\x99\xef\x43\xe9\x81\x68\x86\xed\x20\x9a\x20\x9a\xfc\x43\x41\x34\xab\x41\x34\x8b\xd0\xcb\xe1\x98\x26\xaa\x11\x35\x4b\x1d\xef\x4c\x35\x57\x69\x57\x70\x4d\x02\xae\xd9\x50\xae\xa9\x8a\x8a\x0f\x0d\x51\x7c\x08\x5c\x53\x5e\xe5\x26\x38\x17\xbf\x0e\x3a\xee\x78\x94\xf3\xe7\x50\x88\xe4\x23\x13\x22\x98\x67\xd8\x0e\xe6\x09\xe6\xc9\x3f\x14\xcc\xb3\x22\xcc\xb3\x00\xcb\x1c\x8c\x79\x8e\x50\x6d\xa8\x91\xca\x79\x67\x02\xba\x81\xae\x05\x0f\x25\xe0\xa1\x0d\xe5\xa1\x6d\x51\x79\xa1\x11\xca\x0b\x81\x87\x4a\xaf\xea\xea\xa5\xe4\x8e\xce\x40\xc1\x3d\xa3\x76\x70\x4f\x70\x4f\xfe\xa1\xe0\x9e\x15\xe1\x9e\x05\xf8\xe5\x70\xdc\x13\x75\x85\x1a\xa6\x90\xf7\x65\x9d\xe0\x9b\xe0\x9b\xe0\x9b\xcb\xb7\x67\x37\x88\x0a\x0a\x8d\x50\x50\x08\x7c\x53\x62\xf5\xe6\x6b\x35\xcb\x1e\x39\x22\xd9\x0c\x64\x07\xa6\x19\xb6\x83\x69\x82\x69\xf2\x0f\x05\xd3\xac\x0a\xd3\x3c\x62\xfd\xa0\x11\xea\x07\x35\x49\x15\xef\x4e\x33\x85\x9a\x15\x1c\x93\x80\x63\x36\x94\x63\x76\x44\xe5\x82\x46\x28\x17\x04\x8e\x29\xaf\x62\x9b\x7b\xda\x94\x8e\x4d\x63\x66\xd4\x4c\xbf\x1d\x8f\x6a\xfe\x12\x88\x90\xbc\x63\x22\x04\xe3\x0c\xdb\xc1\x38\xc1\x38\xf9\x87\x82\x71\x56\x84\x71\x16\xe0\x98\xc3\x31\x4e\xd4\x0e\x6a\xa0\x62\xde\x99\x78\xae\xd5\xb3\xe0\x9f\x04\xfc\xb3\xa1\xfc\xb3\x2b\xaa\x23\x34\x42\x1d\x21\xf0\x4f\x89\xd5\x9c\x6f\x98\xc6\x57\x56\x8c\xbc\x56\x5a\xee\x88\xf4\x33\x91\x20\xd8\x67\xd8\x0e\xf6\x09\xf6\xc9\x3f\x14\xec\xb3\x22\xec\xb3\x00\xc5\x1c\x8e\x7d\xa2\x9e\x50\xf3\xd4\xf2\xee\xe4\x73\x8d\x96\x05\xf7\x24\xe0\x9e\x2b\xb8\xe7\xc4\x36\x4d\xcd\xf1\x18\x8a\x4a\xef\x01\xc2\xfd\x53\xcd\xec\x9f\xed\xee\x3a\x12\xd8\x13\x14\xf8\xe9\x71\x0c\x45\xb3\xa8\x99\xc3\x98\xd1\x24\xff\x5f\xdb\xbd\xa6\x2e\xb7\xfd\x2f\xa7\x93\x6b\x7f\x51\x0a\x5f\xab\xba\x94\x7a\x03\x69\x76\x8b\xa5\xd9\x45\xb5\x24\x30\xea\xfd\x54\xf7\x17\xb6\xd0\xc6\xff\xb6\x2f\xc6\x13\x7b\x6e\x89\xf5\xf2\x3d\xe1\x74\x6f\xb0\x1a\x85\xda\x37\xd0\x35\xe3\xf0\xe5\x0f\xad\x81\x83\x7e\x54\x82\x1a\xff\xd5\xbe\xc8\x51\xe1\x7f\xdb\x17\x24\x2d\x48\x70\xde\x72\x39\xaf\x0d\xc6\x2b\x1d\xe3\xad\xae\x9e\x5e\x6f\xfa\xee\xb7\x05\xa8\x67\x75\x40\x2b\xf4\x34\xf4\xf4\x16\x7a\xda\x9d\x5b\x91\x8a\xdd\x8c\x50\x47\x44\xfa\x03\x75\x6c\xd7\xff\xab\x7d\x51\x2b\x1a\x7d\x3c\x15\x1e\xca\x2f\xd0\xda\x79\xa3\x76\x38\x22\x30\x67\x43\xb5\x37\x59\xb5\x1f\xdf\x98\xdd\x1b\x8c\xba\x6f\xda\x30\x66\xc7\x0d\xc5\xc6\xec\x02\x5c\x72\x28\x63\x76\x6f\x75\x90\xb9\xfc\xb6\x6c\x28\xda\x92\xed\xd5\x02\xbd\x99\x32\x53\xef\xae\x3f\x61\xa0\xae\x9d\x81\xba\xce\x0c\x71\xd0\xca\x09\x34\xdc\x36\x57\x07\xa0\x82\x21\x82\x21\x1e\x5e\x71\xfd\x64\xfb\xc6\xe5\xa2\x6e\x8a\xeb\x78\x0c\x31\x94\x1f\x18\x22\x18\x62\xfa\x65\xc0\x10\x97\x0f\x3e\x3e\x43\xbc\xe8\x5e\x5e\xb6\x5a\x60\x88\x71\x43\x31\x43\x2c\xc0\x25\x07\x63\x88\xab\xc3\xc0\x9b\xc0\x10\xeb\xa9\x68\x77\x66\x88\x02\xbd\x09\x86\x08\x86\xf8\x80\x21\x4c\x83\x61\xee\xcd\xc2\x70\xcd\xa4\x54\x5f\x51\x08\x93\x4b\x1d\x1a\x0e\x85\x4e\x1d\xd3\x5e\xcc\x02\xfd\x90\xa0\xff\x89\xed\x50\xfd\xd7\x90\xc8\xa6\xc6\x35\xbe\xf7\x69\x7a\x27\xf2\xa8\x49\x27\x7e\xfe\x55\xd9\x45\x9f\xde\xb2\x9f\xba\xd0\x26\xd7\xd4\xd2\xcf\x26\xae\x6d\xa5\xd7\x2e\xc3\x58\xb9\x5b\x72\xcb\xf8\x3e\xbf\x2a\xcf\x6d\xdd\x23\x27\xdf\x25\x3d\x7b\xb4\x45\x34\xd6\x44\x9b\x5c\xd1\x8f\xc6\x8c\xda\xf3\xdc\xba\x67\x2a\xf6\x85\x36\xb9\x9e\xba\xd1\x14\x49\x29\x09\x76\xf9\xd7\xa8\xdb\xe9\xd1\x9d\x2c\x8d\x02\xc9\xae\xad\x7c\xfb\xa6\xdd\x1d\xf5\x5e\xf2\xd3\xde\x9d\x5e\x68\x27\xed\xce\xe0\x94\xa8\xed\xd1\x29\xe9\xb6\x4e\x49\xeb\xf1\x70\xc4\x6f\xae\xca\xb7\xed\xd1\x68\xd2\xed\x2b\xb9\x6d\x6c\x03\xc5\x5b\xb4\x08\xb9\x25\x68\xd9\x16\xaf\xa5\xb5\x39\xa3\xa6\x77\xa9\x05\xb8\x7c\x3f\xb5\xd5\x4a\x2f\xc2\xe5\x85\x82\x95\x98\x85\x5e\x31\xb3\x79\x17\x60\xc6\x9c\x92\xe3\xef\x78\xaf\x85\xe1\x74\x82\x0d\x4a\xb8\x6a\x3a\x99\x55\xd3\x5f\xbb\x68\x0a\xca\x44\x19\x3a\x9b\x09\x4b\xfa\x5c\x08\x0a\x3a\x89\x20\x79\xad\xa6\xac\xd2\xfc\x33\xcd\x71\x0c\x6b\xfa\x31\x9c\x8b\x6a\x51\xfb\x0a\x0d\x1f\x11\x95\x70\x81\x10\xdf\x26\x6c\x2d\x15\xae\x1d\x35\x59\x2e\x02\x48\xbf\x7c\x18\x83\x88\xab\x1f\xd6\x5e\xa1\x42\x67\xda\xed\x2b\xcd\xd7\xce\x97\xf6\x09\x6e\x76\xe4\x6d\x23\x13\xdb\xb2\xc2\x9d\x21\x75\xcf\xc7\x70\x4b\x48\xad\xb8\x62\xc3\x89\x39\x9f\x1a\xd6\xaf\xd4\xf5\x0c\x3b\xd8\xfb\x95\xfe\xe3\xf6\xe3\x6e\xf2\x30\xc7\xf6\xfc\x4b\xe3\x36\x3d\x0c\x51\xe3\x1b\xdb\x5a\x6e\xd2\x4a\xaf\xf5\x27\xee\xba\x4b\xf3\xdf\x61\x6d\xc2\xaf\x30\x99\xbd\xd7\x9c\x15\x63\x75\x19\xc2\x8a\xb4\x39\x88\xf0\x1b\xe0\x4f\x4f\x9e\x67\x2e\xd8\xf1\x17\x56\x08\x5c\xfe\xed\xd8\x73\x34\xf7\xda\x0c\x8d\x45\x5c\x07\x2f\x0d\xd3\x8c\x49\x0e\xdb\xef\x3a\xea\x29\x51\xd5\xe1\x29\x3b\x43\x93\xb4\x1e\xab\xc3\xd4\x7e\x77\x39\x37\x8b\xcc\x79\xc1\x93\xf9\xe7\x84\x8f\x69\xb7\x4e\x89\x3a\xea\xa4\x1e\xb0\x12\x97\xfb\xda\x85\x19\x3c\x67\x3e\xb3\xd2\x63\xbf\x15\xd0\xbe\x9e\x5f\xd0\xb1\x4b\x1d\xd3\x98\xb0\xf0\xf6\x89\x6d\xf9\xae\x6d\x9a\xd4\x1d\x7b\xbe\xe6\xcf\xbd\xb1\x4b\x35\x7d\xb1\xbc\xc5\xe3\x70\xf6\xf7\x09\xcc\xfe\xfe\xb4\xf0\x11\xcf\xfe\xf3\x3d\xa7\xcb\x1e\xff\xf0\xfd\xfd\xee\x01\x8c\x09\xa8\x5e\x89\xa9\xf7\x36\x57\x29\xea\x69\x5b\x29\x02\xd8\x4a\xa7\xe5\x29\x85\x08\x3b\x7b\x65\xe9\x84\x99\x5b\x96\x61\x4d\x89\x63\xeb\x05\x41\xd5\x9e\x61\x4d\x4d\x1a\xc8\x38\xb9\xc6\x66\x2a\xbf\x68\x87\xfc\xa2\x65\x57\x57\x2f\x5a\x3b\x40\xe2\xca\xb3\xe2\xf5\xda\x12\x2c\x8b\x75\x0b\x96\xdd\xf8\x53\xb4\xdb\x6a\x37\xd3\xc3\xa0\x8f\xf3\xe5\x2e\x56\x00\x3f\xb6\x40\x26\x11\xc4\xd8\x16\x99\x44\x80\x06\xc8\x64\x1f\x64\xd2\x2f\x0d\x99\xb4\x8b\x90\x49\x6a\x4a\x01\x9b\x94\x8e\x4d\x80\x3d\x80\x3d\xa4\xc1\x1e\x0e\x9d\x94\x86\x39\xc8\x19\xa9\x03\xd4\x79\x50\x30\xf3\x8b\xa5\xdd\x68\x86\x19\x8c\x3b\x00\x0d\xcc\x29\xf5\x01\x2d\x05\x7e\x99\x5d\x51\xcb\x00\xf6\x14\xd8\x53\x80\x69\x1a\x8b\x69\x58\x32\xe7\xc9\xf2\xff\x96\xaf\x19\x16\x75\xc7\x33\x3a\xb3\xdd\xc5\x38\xac\xe8\x77\xb1\xf0\xa9\x10\x58\x38\xb6\x9e\x86\x11\x67\x9f\xb4\xb3\xaf\xad\xb3\xd1\xe7\xff\x4e\xfe\x0a\xf0\xcc\xc5\x82\x9c\x58\xb6\x4e\x1f\x35\xc9\x9c\xc2\x7c\x23\xba\xe1\xf9\xae\x71\x31\xf7\xa9\x4e\x6c\x8b\x5c\xd9\x9e\x0f\x28\x52\x26\x14\xd9\xd1\x7e\xa2\x77\xbb\x5a\x47\x03\x14\xd9\x0f\x8a\x0c\x4b\x83\x22\x7d\x18\x50\x8e\x0f\x46\x00\x36\x00\x36\x8e\x06\x36\x66\xda\xed\x89\x37\x9f\x9d\xe8\xd4\xf4\xb5\xd0\xb6\xe1\xd8\xfa\x38\xc1\x1d\xb1\x4d\xc3\xf3\x35\xd7\xf7\xc6\x2c\xba\x7c\x4f\xe8\xf1\xa9\x17\x07\x4e\x39\xb6\xde\x28\xf8\xf1\x5e\xbb\x65\x86\x0f\xb2\x14\x28\x39\x31\x35\xcf\x27\x3d\x32\x33\xac\xb9\x4f\xbd\x82\x50\x8d\xba\xe1\x90\xea\x64\x89\x0c\x32\xaa\x76\x7d\xe8\xd1\x50\x50\x47\x40\xe5\xb2\x27\x4a\xc8\x12\x09\xd4\xf7\xeb\x99\xe3\x2f\xf2\x61\x5e\xcb\x38\xcf\xfc\x95\x3a\xa5\x96\x10\xcd\x23\x5f\x83\xb7\x5c\xa3\x4c\x37\x4d\x31\xe9\xa5\xdb\xb7\x4a\x31\x91\x5f\x99\xc9\x98\x04\x73\x2c\xb3\xfd\xfe\xca\xa5\xbd\x52\xb9\xb0\x35\x71\x96\x32\x66\x73\xf7\xbd\x33\xac\xeb\xa2\x68\x7e\x4e\x09\xa5\xda\x03\xf1\xb2\x61\x58\x8b\x6d\xeb\x1c\x7e\xc1\x9c\x04\x05\x32\x8d\xa5\xf6\x42\x29\x4d\x3e\x70\x11\xad\x1d\x8e\xb9\xb5\x76\x40\x5e\x6e\x3e\x20\x35\xb4\x6f\xcd\x3d\xaa\x9f\xa5\xad\x48\x69\xf1\xbc\x92\x24\xcf\xed\xdc\xd6\x09\x93\x3c\x39\x61\xfb\xd6\x29\x61\x23\x7b\x4a\xe6\x56\xf0\xef\x23\xa2\x59\x7a\x88\x4f\xd9\xdb\x24\x86\x32\x83\x53\x42\x48\x77\x2b\x37\xdd\xed\xe0\xb9\x64\x95\x4e\x78\xe3\x21\x25\xca\x9d\x15\xd2\x94\xfe\xf6\x34\x25\x6f\x11\x64\x34\xa5\xdc\xaa\xa4\x8d\x67\x29\x65\xb1\x93\x3d\x12\xe0\xc3\xa4\x16\xaa\xbf\x58\x7c\xc8\x1b\xac\xc1\x5d\x76\xe3\x2e\x0f\x69\x2d\x3b\x34\x96\x09\xd3\x09\x1d\x5b\xaf\x44\xe2\xfe\xf9\xa6\xb6\xb4\x1d\x20\x4b\xab\xa1\x90\x05\xc8\x24\xdd\x49\xe9\x52\xf1\x8f\x91\x43\x39\x12\x9d\x05\xd6\x4e\xd4\xe0\xca\x24\xca\xd4\x7a\x0d\x5b\xdf\xfa\xd4\x65\x6c\x96\x55\x36\x6b\xb5\x06\x23\x55\x1d\x74\x78\x6f\x5a\x78\xdf\x79\xf0\xdc\xb7\x99\x8c\xcd\x03\xab\xaa\xe5\x41\x18\x6b\xd4\x55\x7c\xdb\x0a\x95\x85\x3c\x4c\x78\xeb\x09\xd9\x35\x0f\x73\xd4\xcf\x3d\x7c\x23\x67\xbd\xda\x2e\x4c\x77\x40\xe0\x20\x02\x07\xc9\x21\x7d\xf9\xbb\x6e\xed\x9d\xec\xa5\x84\xa3\xa4\x26\x77\x75\xf6\xfd\xa6\xc4\x0c\xd4\xd1\xc4\x2d\x51\x88\x00\x12\x3e\xb7\x81\x39\x48\xf8\xac\x3c\x04\xca\x25\x7c\xee\x0e\x81\x3a\x45\x10\x08\x01\x8b\x08\x58\x7c\x28\x90\xd3\x06\xc8\xa9\x27\xc8\x41\xd8\x00\x32\x4b\xab\x8d\x9c\x60\x20\xaa\x04\x3a\xca\x67\x96\xee\x0e\x8f\xba\x45\xf0\x08\x16\x22\x58\x88\xe4\x04\x4f\x03\x80\xa7\x6a\x82\xa7\x1a\x86\xf8\x49\x64\x20\x42\x0a\x2b\x52\x58\x6b\x8e\x79\x72\x29\xac\xbb\x63\x9e\x1e\x4c\x42\xc7\x47\x3d\x40\x35\x62\x54\xd3\x07\xaa\xa9\x26\xaa\x41\xae\xac\x60\x29\x20\x57\x36\x69\x46\xae\xec\xe6\x21\x66\x82\x20\x74\xb5\x5d\xee\x91\x6a\x8d\x0f\x43\x97\x28\x59\x76\xe7\x40\x40\xb5\x76\x5a\x53\xc6\xc0\x76\x24\xe5\x22\x29\x57\x24\x53\x24\xe5\x4a\x35\x1c\x48\xca\x45\x52\x2e\x92\x72\x91\x94\x8b\xa4\x5c\x22\x25\x1f\xda\x3a\x29\xb7\xdd\xca\xdb\x38\x23\x3e\xb4\xfa\xdc\x45\xf0\xa1\xa4\xb5\x5a\x69\xb9\xdb\xf3\xa0\x11\x68\xd0\x11\x68\x10\xf2\x7b\x91\xdf\x5b\x5f\xf4\x23\x01\x7e\xa8\x0a\xce\xa9\x73\x8a\x6f\x5b\x15\xd5\x2a\x6c\x0f\x13\xc5\x56\xa3\x14\x5f\x97\xea\x86\xb7\x46\x65\x85\xf7\xac\xd0\x57\x48\xee\x45\x1c\x03\x21\x3b\x26\xf7\xb6\xd5\x4e\xee\xe1\x1b\x86\x31\x24\xc0\x0f\xa1\x9b\x08\xdd\x94\x3e\xc8\x41\x8e\xe4\xde\x52\x76\xfc\xa6\x84\x37\xd4\xd1\x48\x2e\x51\x34\x03\xd2\x7a\xb7\x01\x38\x48\xeb\xad\x3c\xf8\xc9\xa6\xf5\xee\x0e\x7e\x8a\x8f\x98\x47\x0c\x27\x62\x38\x9b\x9d\xd6\x0b\x78\x23\x69\x78\x4b\x4d\x42\x0e\x90\xd0\x9b\x59\x78\x72\x63\x26\x18\x85\x2a\x81\x8b\x72\x09\xbd\x7b\x00\x23\xb5\x08\x18\xc1\x2a\x04\xab\x90\x9c\xb0\x49\x8e\x84\x5e\xc0\x26\xa4\xf2\x26\x57\x91\xca\x5b\x7b\xb4\x83\x54\x5e\x89\x52\x79\xf7\x40\x3b\x85\x05\x6e\x61\x06\x82\x19\xa8\xd9\xa9\xbc\xc0\x33\x48\xe2\x95\x16\xe1\x20\x89\xb7\x4a\x41\xeb\x5b\x27\xf1\xb6\x55\x51\xd0\x7a\x27\x89\xeb\x42\xd0\x7a\xdc\x88\x24\x5e\xfe\x52\x95\xf5\xa5\x8c\x71\xeb\x48\xdf\x45\xfa\xae\x48\xa6\x48\xdf\x95\x6a\x38\x90\xbe\x8b\xf4\x5d\xa4\xef\x22\x7d\x17\xe9\xbb\x44\x4a\x26\xb4\x7d\xfa\x6e\x3b\x6f\xd7\x8c\x98\x50\x17\x4c\x08\xe9\xbb\x12\xa5\xef\xd6\x97\x00\x21\x71\x17\x89\xbb\xf5\xc5\x3d\x12\x20\x87\xaa\x20\x9c\x5a\x27\xee\xb6\x45\x85\x13\x3b\x9c\x51\xaf\x46\x89\xbb\x5f\x6c\xf7\x7a\x6d\xb1\x89\xe8\xa6\x15\x1a\x0b\xa9\xbb\x88\x5b\x20\x64\xd7\xd4\xdd\x4e\x2b\xf7\xf0\x0d\xc3\x16\x12\x2f\x31\x82\x34\x11\xa4\x29\x7d\x50\x83\x1c\xa9\xbb\x25\xed\xf9\x4d\x09\x6b\xa8\xa3\x89\x5c\xa2\x28\x06\x24\xef\x6e\x03\x71\x90\xbc\x5b\x79\xf8\x93\x4b\xde\xdd\x1d\xfe\x0c\x8a\xe0\x0f\xa2\x36\x11\xb5\xd9\xec\xe4\x5d\x00\x9c\x52\x01\x0e\x42\x0e\x90\xbe\x5b\x6d\xd4\x04\xc3\x50\x25\x90\x51\x3e\x7d\x77\x77\x68\x34\x2c\x82\x46\xb0\x0c\xc1\x32\x24\x27\x70\x92\x23\x7d\x17\xc0\x69\x2b\xe0\x54\xc3\xd0\x40\x89\x0c\x43\x48\xe0\x45\x02\x6f\xcd\xf1\x4e\x2e\x81\x77\x77\xbc\x53\x58\xc4\x16\xa6\x20\x98\x82\x9a\x9d\xc0\x0b\x44\xb3\x15\xa2\x41\x0a\xaf\x60\x19\x20\x85\x37\x69\x46\x0a\xef\xc6\xe1\x64\x1d\x51\xe0\x7a\x37\xe9\x39\x02\xd7\xe3\x46\xa4\xf0\xf2\x97\xaa\xad\x31\x65\x8c\x61\x47\x12\x2f\x92\x78\x45\x32\x45\x12\xaf\x54\xc3\x81\x24\x5e\x24\xf1\x22\x89\x17\x49\xbc\x48\xe2\x25\x52\x72\xa1\xed\x93\x78\xbb\x79\xdb\x66\xc4\x85\x54\x70\x21\x24\xf1\x4a\x94\xc4\x5b\x67\x0a\x54\xfb\x34\x5e\xc7\xd6\x91\xc4\xdb\x4c\xdc\x23\x01\x72\xa8\x0a\xc2\xa9\x75\x12\x6f\x57\x50\x38\x31\x29\x16\xb2\x32\x85\x97\x57\x23\xc7\x52\x39\x13\x97\x23\x3b\xa4\x50\xe1\xb0\x5b\x56\xa8\x9b\x78\x6b\x79\x79\xfe\x0b\xf9\x25\xe0\xb4\x3b\x27\xe2\x56\x07\x86\x6e\x6f\x92\xef\x0e\x8a\x27\x47\x3f\x99\x1d\x82\x88\xc1\x12\xb0\x69\x9d\x60\xa6\x44\x26\x77\xf9\x17\xa7\x94\x58\xd0\xb2\x75\x3a\x8e\x51\x5d\x1a\x0f\x3e\x4d\x90\xe1\xc4\x99\x47\x36\x32\x8f\x4e\x6c\x4b\x8f\xb0\xe1\xd3\xbb\x3b\xf2\xf8\xe7\xf9\xec\x83\xe6\x53\x72\x7f\xcf\x41\xc5\xff\xec\x6e\x3b\x2b\x07\x26\xae\xb6\x9e\xaf\x80\x89\xa5\x19\xd0\x1f\x12\x5a\xc6\xfb\x3f\xac\x67\x92\xa3\x48\x18\xc9\xb8\x76\xb9\x30\x64\xaf\x5b\x0c\x13\xd4\x6e\x7b\x33\x14\x59\x4a\x21\x98\xee\xb1\x54\x9c\x69\x78\x3e\xb5\xd6\x1a\x3d\xe2\xdb\x80\x43\xf7\xc5\xa1\xbd\x9e\x68\x82\x75\x00\x44\x49\x45\x81\xe8\xae\xeb\x9c\x23\x1f\x0f\x69\xf8\x2c\x71\x17\x00\xe0\x95\x0c\xf0\x8a\xca\x1b\x02\xf2\x02\xf2\x02\xf2\xa6\x3b\xd9\x4c\xc8\xdb\x6f\x8b\x10\xc9\x86\x86\xd3\x6a\x41\xde\x52\x4a\xf5\x02\xec\x6e\x36\xb5\x84\x71\xd0\x3d\x80\x5d\x02\xb0\x8b\x52\xdd\x80\xb9\x80\xb9\x80\xb9\x80\xb9\x80\xb9\x87\x86\xb9\x03\x61\x1c\x62\xbf\x8e\x30\xb7\xec\x0a\xdf\x00\xba\x2b\x27\x97\x2a\x9a\x5c\x85\xb5\x1a\x01\x74\x01\x74\xab\xb2\x03\x00\xea\x02\xea\x02\xea\x02\xea\x72\xcd\x80\xba\x92\x43\xdd\x61\x31\x1a\xd9\xf0\x2c\x9b\x3a\x04\xc2\xfe\x63\x6e\xfb\xda\x91\xa0\xea\x84\x95\x5e\xc9\x48\xf3\x08\xf8\xf5\x92\x2b\xfc\xa1\xb6\xf8\xca\x1f\x65\x22\x5b\xd1\xc9\x48\x40\xb0\x7b\x22\x58\x6d\x4a\xa3\xd1\x4b\xa9\x42\x04\xce\x86\x3d\x74\x6d\x36\xdd\xd3\x88\x79\x25\x16\xbd\xb2\xbf\xfc\x48\x35\x9d\xbd\x70\xfa\x6b\x21\x28\xe0\x5e\x74\x62\x9b\x99\x9d\x55\xa7\xde\x44\x38\x4b\x4a\x84\xb9\x9e\xbf\x30\x57\x69\x70\xb6\xf5\x04\x92\xfa\x98\x46\x10\xe1\x1e\x41\x13\x2c\xf8\xdb\x6f\xbf\xfd\x76\xf6\xfe\xfd\xd9\xab\x57\xe4\xc7\x1f\x9f\xce\x66\x4f\xbd\x0c\xb6\x74\x34\xdf\xa7\xae\x55\xfc\xac\xe5\x16\x78\x65\xe8\x3a\xb5\xd6\x67\xd4\xc7\xdd\xca\x03\xb4\xa5\x40\x6d\x37\x5a\x05\x39\xa5\x9b\x54\x20\xfc\xbc\xcf\x0b\x71\xf9\xcd\x19\x98\x1c\x62\xde\xfc\xc4\x0c\x2e\x7c\x8c\xe1\xa3\xf2\xca\x35\x4c\x93\xe8\xf6\x97\xcc\xec\x0c\x6e\xfb\xc5\x4d\x13\xf3\xac\x08\x59\x89\x41\xf2\x6d\xb6\x98\x5a\x31\x32\x4e\x89\xd8\x9a\xcf\x2e\xb2\x3c\x6c\x6e\x19\x1c\x00\xda\x4e\xfa\x1f\xe8\x1f\x73\x9a\xcb\xff\x6f\xca\x00\xbc\x90\x67\x00\xc8\x9f\x9a\x39\x04\x2f\xcb\x1d\x82\x48\xe1\xb1\x8f\xdb\x0d\xc4\x3b\x63\x66\x34\x75\x1d\xbc\x7a\xf8\x75\x10\x8a\xbf\xa9\xab\xe0\xb5\x0c\xab\xe0\xdc\xd6\xe5\x93\x7e\x1a\x99\xef\x24\xfc\x27\xfa\x93\xbb\x3b\xf2\xf8\xa7\xa5\x2d\x8e\xdc\xdf\xe7\x1a\xce\x3a\xde\x44\x33\xe9\xd9\xf5\xfc\x82\xba\x16\xf5\xa9\x77\x36\xb1\x67\xce\xdc\xa7\x67\x2e\x0d\xc9\x93\x77\xe6\xd8\xfa\xff\xdc\x68\xee\x59\x62\xe2\x4b\x0c\x7c\x7f\x0e\x2e\x38\xb6\xfe\xec\xbb\xf1\x78\x42\xb3\x05\x54\xb9\x11\x77\xb2\x52\x3e\xf6\x6a\x93\x68\x8c\x39\xb1\x3c\x79\xfc\xc3\x93\xed\xe5\xe2\xf9\xae\x61\x4d\x37\x93\x4b\xf4\x17\x67\x40\xac\xb3\x91\x58\xbb\x30\x69\xd6\x3c\xec\xf9\x1a\x23\x63\xb9\x35\xb5\xa5\xe5\x58\x58\x16\x21\xd5\xbe\x47\x49\xb8\x74\x55\x89\xe5\x02\x1c\xbb\x11\x56\x62\x52\x9e\xd8\x6e\xaa\xd6\x56\x8d\x04\xf9\xa2\x0c\x41\x4a\x3a\x61\xc9\x13\x82\x61\x5e\xd6\x9e\x3b\xf0\x7a\x31\x19\xa6\xaa\xbb\x18\x5f\x35\x7b\xb5\x34\x63\x90\x5f\x1f\xc7\xd1\xb8\xde\x9f\xc8\x7c\x01\xfb\xf9\x13\x77\x2c\xad\xe3\x6a\x96\x17\x0c\x42\x7e\x08\x62\x38\x94\x69\x86\xaf\xb1\x29\xbe\xc6\x6f\xc8\x43\xb9\x07\x87\x82\x3a\x39\x6a\x37\x39\x3d\xed\xf0\x91\x70\x89\xfb\xa2\x3a\xd9\x8d\x70\x31\xae\x9f\x8f\xdb\xbb\x18\x87\x82\xd2\x3c\x6a\x37\xa9\x0b\x08\x27\x63\xe5\x9c\x8c\x3b\x17\xc4\xcf\x5c\xa9\x7e\x4e\x34\x7c\x99\xf0\x65\x4a\x68\xc1\x86\x2f\x13\xbe\x4c\xf8\x32\x4b\x1d\x02\xf8\x32\x77\x19\x04\xf8\x32\xe1\xcb\x84\x2f\x13\xbe\x4c\xf8\x32\xab\x66\xec\x96\xc9\xde\x0c\x5f\x26\x7c\x99\xf0\x65\xc2\x97\x09\x5f\x26\x7c\x99\x65\x0f\x32\x7c\x99\xf0\x65\xc2\x97\x29\xa1\x2f\x73\x24\xaa\xd7\xdc\x4b\x0c\xdb\x35\xf2\x65\x96\x5f\xbc\x0e\x5e\xcc\x92\xbc\x98\x23\x51\x61\xe7\x1e\x52\x25\xe1\xc5\xac\x74\xb1\x3b\xf8\x2f\xe1\xbf\x94\xd0\x6a\x0d\xff\x25\xfc\x97\xf0\x5f\x96\x3a\x04\xf0\x5f\xee\x32\x08\xf0\x5f\xc2\x7f\x09\xff\x25\xfc\x97\xf0\x5f\x56\xcd\xc0\x2d\x93\x8d\x19\xfe\x4b\xf8\x2f\xe1\xbf\x84\xff\x12\xfe\x4b\xf8\x2f\xcb\x1e\x64\xf8\x2f\xe1\xbf\x84\xff\x52\x3e\xff\x65\xa7\x25\x3a\x7c\xab\x77\xcc\xf3\x66\x8f\xe6\xbf\x3c\xc4\xa9\x04\xf0\x60\x96\xe3\xc1\xec\xb4\x44\xa7\x75\xf5\x92\xa3\x69\xe1\xc1\x84\x07\xb3\x92\xa7\x18\xc0\x87\x09\x1f\xa6\x84\x96\x6b\xf8\x30\xe1\xc3\x84\x0f\xb3\xd4\x21\x80\x0f\x73\x97\x41\x80\x0f\x13\x3e\x4c\xf8\x30\xe1\xc3\x84\x0f\xb3\x6a\x46\x6e\x99\xec\xcc\xf0\x61\xc2\x87\x09\x1f\x26\x7c\x98\xf0\x61\xc2\x87\x59\xf6\x20\xc3\x87\x09\x1f\x26\x7c\x98\x12\xfa\x30\x55\xc1\xc9\xea\x1b\x9e\xab\x5e\xf5\xe3\x26\xdf\xd3\x99\xed\x2e\x70\x38\xba\x78\x7e\x08\x0e\x47\x4f\x3c\xdc\xf0\x2a\x4a\x7b\x08\xba\xfc\x0b\x51\xca\x03\xca\x13\x94\x36\x63\xdb\x43\x04\x19\x2f\x16\xfe\x0a\x78\x46\x36\xc3\x67\xa7\x24\x7e\xfa\x7f\x3d\xfb\x1e\xe7\x8f\x97\x79\xfe\x38\xbf\x99\xef\x07\xf1\x70\x04\x39\x9b\xec\x80\x71\x4d\x3c\x82\xbc\xa3\x0a\x8e\x20\x57\x7b\xdd\xcd\x50\x61\x29\x71\x6d\xfd\x63\xa9\xb1\x43\x9c\x31\x00\x5c\xb9\x6e\x8e\x89\x8e\x26\xef\xf5\x80\x2c\xe5\x47\x96\xbb\x2e\xea\x76\xe6\x4a\xf5\xcf\x07\x00\x80\x05\x80\x05\x80\x8d\x2f\x00\xc0\x02\xc0\xf2\x0f\x7d\x10\x00\xdb\x16\x1d\x92\xd5\xdb\xd0\xac\x59\x2d\x00\x5b\x6e\x61\x39\x40\xd7\x75\xb3\x4b\x74\xe4\x55\x6f\x00\xe8\x0a\xe8\x5a\x81\xb5\x0e\xd0\x7a\x5c\xd0\xea\xd8\x3a\x20\x2b\x20\x2b\x20\xeb\x26\x9d\x6c\x26\x64\xed\x08\x6b\x21\x1f\xf3\x5c\xd7\xa3\x41\xd6\xb2\x73\x89\x01\x5a\xd7\xcd\x2f\x61\x85\x63\x9c\xd3\x0a\xd0\x5a\x8d\xd5\x0e\xd8\x0a\x5b\x2b\x80\x2b\x80\x2b\x80\xab\x44\xc0\xb5\x2b\x28\x82\x33\xd8\x0c\xb6\xd6\x24\x80\x14\x75\x6c\x4a\xc2\xa9\x5d\x41\x1d\x1b\x54\xb1\xa9\x70\x15\x1b\xf9\xd7\xf3\x43\x95\x98\xe1\x74\x18\x2a\xcc\x14\x74\xab\x10\xa5\x2d\x45\xfa\xe0\xc9\xed\x15\x2c\x32\xa3\xd3\x49\x88\xd6\xb6\x1e\x03\x94\x9a\x91\x6a\x18\x9a\x5a\x6a\x43\x8a\x82\x33\xd1\x58\xa0\xe6\x4c\x69\xe3\xb0\xfb\x82\x40\xe5\x99\x12\x87\x01\x95\x67\x50\x79\x06\x95\x67\x4a\xad\x3c\xb3\x93\x81\xf8\x60\xf6\xe1\x5a\xd7\x95\x89\x04\x5c\x86\x68\x25\x95\x65\x29\xa5\x65\x1e\x64\x46\x6e\x53\x38\xa6\xfe\xe3\x78\xac\xda\x31\xf5\x97\x64\x29\xe5\x63\x64\x5d\x11\x8d\x19\x45\x29\xea\xc3\xf0\xb6\xfc\xfd\x5c\x82\x28\x11\xc3\xfe\x43\x89\x98\x3a\x94\x88\xe9\x09\x4a\xc4\xa8\xfd\x63\x1e\xd3\x9f\xf8\x13\xab\x93\x1c\x08\x2f\xe1\x81\xbc\x84\x3d\x41\x55\x1a\xb5\x8f\xf3\xfa\x2b\xec\x27\xdc\x75\x6f\xe8\x64\xae\x54\x3f\xab\x18\xee\x48\xb8\x23\xa5\xb4\x37\xc3\x1d\x29\xc5\x30\xc0\x1d\x29\xc9\x40\xc0\x1d\x29\xc7\x38\xc0\x1d\x29\xc5\x30\xc0\x1d\x09\x77\x24\xdc\x91\x70\x47\xc2\x1d\xd9\x08\xb3\x3d\xdc\x91\xf5\x18\x47\xb8\x23\xe1\x8e\x84\x3b\x12\xee\x48\xb8\x23\xa3\x69\x02\x77\xa4\xbc\xee\x48\x51\x75\xe2\xfe\x31\x4f\xdd\x3f\x9a\x3b\xf2\x20\xc5\xdd\xe0\x88\x2c\xcb\x11\x29\x2a\x63\xdc\x47\xc2\x22\x1c\x91\x95\xae\x11\x07\x17\x24\x5c\x90\x52\xda\x98\xe1\x82\x94\x62\x18\xe0\x82\x94\x64\x20\xe0\x82\x94\x63\x1c\xe0\x82\x94\x62\x18\xe0\x82\x84\x0b\x12\x2e\x48\xb8\x20\xe1\x82\x6c\x84\xa9\x1e\x2e\xc8\x7a\x8c\x23\x5c\x90\x70\x41\xc2\x05\x09\x17\x24\x5c\x90\xd1\x34\x81\x0b\x52\x5a\x17\x64\x5f\x74\xbe\x54\xff\x98\x07\xa4\x1e\xcd\x05\x79\xa0\x62\xfd\x70\x42\x96\xe4\x84\xec\x8b\x0e\xa4\xea\xe3\x2c\x55\x38\x21\x2b\x5e\xf3\x1f\x6e\x48\xb8\x21\xa5\xb4\x33\xc3\x0d\x29\xc5\x30\xc0\x0d\x29\xc9\x40\xc0\x0d\x29\xc7\x38\xc0\x0d\x29\xc5\x30\xc0\x0d\x09\x37\x24\xdc\x90\x70\x43\xc2\x0d\xd9\x08\x73\x3d\xdc\x90\xf5\x18\x47\xb8\x21\xe1\x86\x84\x1b\x12\x6e\x48\xb8\x21\xa3\x69\x02\x37\xa4\xb4\x6e\xc8\x81\xf0\xcc\xf0\xf8\x73\xad\xcf\x5e\xfc\x89\xfa\x5f\x6c\xf7\xba\x31\xa7\x7e\xf7\xb7\x9f\x21\xa2\x53\xbf\x13\xdf\x31\x1c\x83\x69\xc7\xe0\x3a\x87\xe0\xa6\x8e\xbf\x76\xba\x7d\x1b\xc7\x9f\xfc\x4b\x50\xca\x93\xb8\x0d\x57\xf3\x29\x87\x18\xad\x70\x7b\x18\xbb\x74\x42\x8d\x9b\x08\x34\x8e\xd9\x4c\xdb\x17\xab\x7d\xea\xcd\x3e\x3f\x3a\xf2\x69\xdc\x8e\xad\x17\x9c\xc5\xcd\x91\xf9\x92\x21\xd9\x8e\xe7\x6a\x7f\x08\xa5\x4d\x5e\x68\x96\x1e\xae\xc2\xbd\x00\x5b\x53\x0f\xd7\xe6\x0c\x6c\x39\x3c\x92\x4c\xb0\x17\xce\x41\x8f\xdd\x56\x5a\x4a\x0d\x01\x1b\x3f\x04\x87\x39\x78\xbb\xce\x90\x62\x28\x2a\xbd\x3e\x04\xa4\x00\xa4\x48\x4b\xb9\xb6\x90\x82\x19\x04\x66\x86\x5f\x75\x4c\x41\x1c\x5b\x27\x55\x40\x15\x1f\x23\x81\x03\x56\x94\x65\xeb\x01\x74\x90\x12\x3a\x1c\xc5\x70\x34\x14\x06\x0c\xf7\x13\x15\x73\xf0\xf8\x75\xce\x4c\x55\x9d\x93\x39\x60\x7c\xda\x7c\x96\x09\x0b\xb5\xc1\xfa\x24\x31\x54\xdc\x79\x39\x0f\x32\x97\xaa\x7f\x0a\x4f\x95\x20\x69\x4d\xac\x5c\x95\x41\xa4\xb0\x73\x95\x02\x48\x61\xe7\xaa\x30\x58\xad\x33\x7a\x19\x89\x9c\xab\x7d\x18\xba\xea\x88\x5e\x86\x40\x2f\x0f\x88\x5e\x60\x50\x3b\x32\x7c\x81\x41\xad\x1c\xfc\x02\x8c\x92\xee\xa6\x74\x18\xe5\x18\x06\xb5\x6e\x4b\x78\x44\xf6\x28\x51\x65\x35\x32\xa8\x95\x5b\x94\x1e\xa6\xb4\xb5\xf3\x4b\xe4\x74\x1d\xb4\x00\x46\x6b\x08\x46\xe5\x30\xa5\x95\x53\x47\xbe\x4a\x30\x14\x46\x34\x18\xd1\x2a\x08\x42\x61\x44\xab\x30\x40\xad\x35\x6e\x11\x39\x9a\x07\x2a\x70\x4b\x0d\x71\x8b\x1c\x46\xb4\xe6\xe1\x16\x98\xcf\x8e\x0c\x5c\x60\x3e\x2b\x07\xb9\x00\x9d\xa4\xbb\x29\x1d\x3a\x39\x8a\xf9\x4c\xed\x88\x60\xc2\x31\x8f\x74\x3c\x9e\xf9\xac\xec\x82\xaa\x30\xa0\xad\x9d\x61\x22\x6f\xee\x00\xe7\x34\xd6\x11\x88\xca\x61\x40\x2b\xab\x06\x6a\x95\xa0\x28\x4c\x68\x30\xa1\x55\x10\x88\xc2\x84\x56\x61\x90\x5a\x67\xe4\xd2\x16\xb9\x96\x07\x5d\x20\x97\x1a\x22\x17\x39\x4c\x68\x4d\x44\x2e\x75\x31\xa2\x55\xa4\x50\x04\x4c\x68\xe5\x20\x17\xa0\x93\x74\x37\x1f\x1e\x9d\x7c\x13\x3d\x36\x58\x73\xc1\x72\x52\xd8\xe1\x2f\xe1\xd8\x28\xde\xe4\x8a\xce\xb4\x5f\xa9\xeb\x45\x6a\x61\x18\x36\xfb\x8b\x70\x59\xe8\x9a\x7b\x1d\xde\xe9\x6b\xd3\x64\xd8\x95\xb0\x9a\x71\x24\xb8\xe5\x16\xac\xc4\x3f\xe5\xd3\x99\x63\x6a\xbe\x61\x4d\xe3\x57\x50\x4c\xc3\xf3\xb9\x79\x73\x57\x04\x03\xee\x8a\xb4\xc0\xb9\x6b\xcf\xa8\x7f\x45\xe7\x22\x2f\x0a\x77\x43\xd1\xd0\x2b\x57\x06\x5b\x12\x3c\x98\x31\xac\x89\x39\xd7\xe9\x73\xb3\x08\x22\x14\xcf\x00\x65\x36\x37\x7d\xa3\xe0\xf6\x68\x65\x29\x05\x58\x2c\xa5\xf3\xf9\x12\xc3\xca\x1f\x73\xea\x06\x88\x4a\x71\x8a\xde\x8e\x1b\x2b\x35\xd5\x3a\xa5\xc1\x5c\xe3\xcb\x23\x2a\xde\xb5\xe1\xfc\xe2\x9a\x3f\x2f\xac\x49\x41\xe7\x96\x3b\x0a\xd7\xb9\xec\x92\x4e\x4d\x31\xf3\xd7\x48\xaa\x99\x97\x17\x8d\x51\x34\x2b\x3e\x17\xea\xef\x6c\x11\x6c\xc1\xf8\xe5\x6e\x2b\x75\x14\x95\x58\x4f\x2a\x5b\x0c\x66\xe1\x97\xb8\xb1\xe4\x5e\x84\x17\x48\x1a\xd9\x64\xab\x60\x6e\x2a\x9a\xcd\x84\x93\x6c\x1b\xdc\xc6\xc1\x4f\xad\x15\xbf\xb1\xe1\xac\x99\xcc\x3d\xdf\x9e\x95\x3b\x63\x22\x01\x3c\xcf\x96\x30\x8f\xdf\xf8\xbb\xf1\x58\x33\xcd\xe2\x49\xb0\x9e\xfe\x30\xd4\x79\x69\x58\x46\x04\x74\xc3\x79\x30\x0e\x55\x5a\x52\x45\xd5\xb0\x2e\x6d\x0e\x4e\x15\xa0\xa9\x25\xae\x7c\xfc\xc3\xf7\xf7\xa7\x24\x83\x97\xd6\xce\xc6\x8c\x46\x8d\x27\x63\x41\x91\xc3\x4d\xb6\x96\xc2\x6f\xad\xdb\x5a\x0e\xf3\xe6\x6b\xb6\xa6\x27\x27\x8f\x7f\x78\x54\x04\x3f\x9f\x6c\x3e\xfd\x22\x0c\xc5\x3f\xdf\xd7\xa6\x6c\x9e\x79\xff\x58\xbe\x9d\x92\xbe\x9a\x13\x43\xd0\x56\x7c\x73\x34\xb9\x43\x39\x71\x17\xe6\x1e\xfd\x18\x3e\x28\xc5\x0d\xd9\xbf\x01\x10\xba\x0f\x75\x9b\xc1\xc6\x24\xd2\x6a\x97\x21\x0c\x55\x2c\xfb\xcb\x99\xba\x84\x6c\x8a\x6f\x47\x6d\x4a\xea\x6b\x8e\x31\xb9\x66\xf4\x2c\xfa\x72\x24\xca\xf1\x12\x57\xf3\x3b\x8b\xd2\x4b\x14\x42\xac\xaf\xd9\x87\x0e\xff\x41\x9d\x25\x7f\xf7\xb8\xbf\x55\xfe\x43\xa7\xc5\x5f\xe1\x80\x65\x9b\xfb\x5b\xd5\xc3\x25\xf7\x79\xf9\x0e\x01\x0b\xc8\xef\x78\xe2\x5f\xe1\x1f\xdc\xe7\x1f\xcc\xff\x4a\xbb\xcb\x7f\x48\x4e\x2a\x50\x06\x3a\xdf\xdf\x65\x5f\x52\xe2\xfb\x6a\x33\xb2\xa5\x44\x80\x64\x09\xdd\x43\x2c\x42\x9e\x90\x17\x11\x0c\x61\x97\x6f\x12\x4c\xf3\xcd\xfd\x37\xff\x1f\x00\x00\xff\xff\x45\xa5\x5c\x18\xc6\xd3\x03\x00") func monitoringBackendGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( @@ -184,7 +184,7 @@ func monitoringKubernetesResourcesByPodGrafanaDashboard1JsonTpl() (*asset, error return a, nil } -var _monitoringSystemGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\xeb\x72\xdb\x38\xb2\xfe\x9f\xa7\xc0\xe1\xcc\xac\xed\x59\xc9\xd1\xd5\xb6\x5c\x95\x3a\x15\x27\x93\x33\xbb\x95\xcc\x7a\x62\xcf\x6e\xe5\xa4\x5c\x5a\x88\x84\x25\xac\x49\x80\x01\x40\x5b\x8a\xcb\xf3\xec\x5b\x04\x49\x11\xbc\xe9\x4a\xdd\xf1\x27\xb1\x00\x12\x04\xbb\x1b\xfd\x7d\x68\x34\xc0\xe7\x57\x00\x18\x90\x10\x2a\xa0\xc0\x94\x70\xe3\x12\xf8\x45\x00\x18\x36\xe6\xc2\xb8\x04\x5f\xe5\x2f\x10\x96\xca\x9a\x9e\x87\x6d\xf1\x37\x62\x5c\x82\x7a\x25\x2e\xb5\xa0\x80\x9c\x7a\xcc\x44\xc6\x25\x30\xaa\x55\xf0\x7f\x0c\xde\x43\x02\x41\xb5\x6a\x28\x97\x21\x02\x7b\xb6\x7f\x89\x60\x1e\x52\xca\x07\xd8\xca\x29\xc5\x26\x25\xef\xa8\x4d\x99\xdf\x26\xeb\xf7\xe0\x71\xad\x02\x1a\xf5\x7a\x05\x34\xda\xed\x0a\xa8\x9f\xa8\x4d\x13\xe8\xc8\x67\xbf\x8d\x5f\x07\xfc\x05\xbc\xb5\x11\x13\x5c\xbd\x4e\x8c\x5c\x79\x9d\x05\xf9\xa0\x47\x21\xb3\x8c\xb0\xee\x45\xfe\x7f\xf7\x0a\x80\x17\xff\x72\xc3\x42\xdc\x64\xd8\xf5\x5b\xf2\xaf\x97\x6d\x18\xc8\xc2\x22\xf5\x0a\x46\x9f\x20\xf1\x37\xcb\xb8\x04\xc4\xb3\xed\xa0\x84\x41\x77\x70\x4b\xa9\x2d\xb0\x6b\x5c\x82\x9a\x2c\xc4\xfe\x25\xe7\xc1\x9f\x02\x31\x18\x36\x5c\x3f\xab\xd5\x3b\xe7\xed\x7a\xbb\x5e\x6f\x76\x64\xad\x8d\xc9\x83\xaf\x8a\xaf\x77\xf2\xa7\x0b\x09\xb2\xf9\x58\x19\x91\x2a\x0c\x93\xda\x36\x74\x39\xf2\x9b\xbd\x87\x36\x1f\x4b\xce\xe8\x33\x6c\x5d\xd3\x58\x9b\x81\x88\x53\x1a\x7b\x32\x2e\x41\xa3\xa5\x14\x0c\xa3\x9e\x86\xbf\x47\xfe\xef\x48\x36\xe3\xb6\xe5\x5b\x5c\x8c\x7f\xc6\x9d\xbb\x1b\x97\x09\x2c\xa4\x80\x8c\x7f\xa1\x1e\x78\xeb\xba\x46\x5c\x13\xca\x9e\xd1\xa7\x40\xea\x61\xbb\xe3\x77\x82\x36\x86\x5c\xaa\x5c\xf6\x3e\x7e\x6c\x0f\xca\x92\xe4\x7b\xfa\x2a\xfc\x88\x48\x5f\xc8\x77\xab\x25\xca\x51\xde\xe5\xaa\x8d\xfe\xa8\xfc\x1c\x5f\x72\x8f\x6d\x5b\x95\x53\xb1\x28\x2f\x52\xa2\xac\x37\xa6\x88\xb2\x9e\x2f\xca\xce\xf8\xa7\x8d\xfa\x88\x58\xc9\x27\xc1\xc7\x7e\xfa\x35\x7c\xcd\x7b\x8c\x21\x22\x72\x6a\x1c\x38\xcc\x2b\xc5\x24\xa7\x94\x0f\xe8\x53\x76\xcc\x09\x2a\xa0\x9d\x73\xf5\x23\xb4\xbd\x58\xa6\x99\x77\xb1\x31\x91\xb5\x6a\x6b\xb2\xf0\x09\x5b\x22\x61\x7a\x29\xf3\x96\x45\xfe\xc0\xb9\xa6\x98\x88\x4f\x54\xfa\x01\x59\x10\x6b\x85\xba\x63\xef\x14\x3f\xd1\x45\xcc\x44\x44\xc0\x3e\xca\x28\xda\xf5\x9b\x62\xd0\xc2\x9e\x7f\x4f\x23\x59\x9e\xb5\x0b\x86\x88\x85\x18\x92\x5e\xe6\xde\xa6\x22\x7e\x30\x47\x0c\x23\xfe\x8f\x47\xc4\x18\xb6\x50\xaa\xd3\xdc\x85\x26\xca\x33\x3f\x2e\xa0\xf9\x90\x96\x05\x17\xc8\x75\x91\xf5\x11\x93\x6c\x7f\x05\x64\x7d\x24\xb8\xe2\x6f\x55\x8f\xeb\x7b\x9d\xa1\x2b\x7b\xc7\x3d\xe7\x98\x41\x81\x8e\x19\xc4\x36\xef\x32\xf4\xcd\x43\x5c\xf0\xae\x54\xda\xb3\xef\x01\x65\xa7\xde\x1c\xfd\x38\xfe\xfb\xa8\xe2\x52\xeb\xcd\x9f\x47\x7c\xc4\x05\x72\xaa\xd0\x75\xab\x5f\x61\xf5\x7b\xad\xda\xb9\xfb\x6b\xfc\xd7\xd1\xcb\xd7\xba\x73\x77\x72\x02\x7a\x23\x70\xcc\x05\x14\x1e\x57\x9d\xab\x3f\x32\x28\x73\xa0\x6f\x72\x86\xc0\x0e\xea\x06\x92\x49\x5e\x82\x89\x40\xec\x51\x5a\x8f\x51\x77\xf2\xeb\x3e\x40\x53\x48\x7f\x5e\x4f\x54\x07\xb6\xff\x61\xfc\x8c\xe7\xe7\x7f\x3f\x3f\x07\xfd\x78\x79\xf9\xf7\xcb\x4b\xb2\x31\x86\xee\xa5\xbf\x35\xde\x1a\xe3\xe2\x97\xf0\x2f\xc5\x0d\x0d\x18\xe2\x03\x6a\x5b\x19\xf7\xe4\xa0\x0f\x8c\x3a\x8a\xc3\x1e\x97\x7f\x46\xfd\xd0\xd2\x52\x37\xdc\x0c\xf0\xbd\xc8\xde\x11\x3a\xba\x1b\x29\x5c\x10\xe9\x03\xb8\x88\x01\x8e\x4c\x4a\x2c\x70\xdc\x1b\x81\xb4\x40\x0d\x31\x86\x86\x67\x75\x3c\x42\x26\x9d\x79\x6a\x44\x72\xca\x44\xca\x9f\xc8\xc1\xd8\x8d\xbc\x29\x26\x16\x7e\xc4\x96\x07\x6d\x23\x33\x2e\xa3\x6b\x24\x22\xc5\x1d\x18\xc2\x21\x4e\x39\xb5\x9e\x67\x3e\x04\x46\xa8\xbe\xa3\xef\x3d\xc2\x31\xe9\x8b\x21\x07\x70\x53\x57\xe7\x7b\x95\xb1\xf7\xf8\x7a\x97\xe9\xe2\x08\x0e\xd1\x04\xdb\xb7\x90\x89\x1d\x28\x31\xa6\x5e\x60\x91\x0c\x7d\x73\x53\xb6\x68\xc3\x1e\xb2\xc7\xb0\x1d\x17\xd3\xfe\x15\xe4\x28\xd3\x56\xe0\x37\x93\xaf\x32\x76\x9c\x99\x62\xe5\x1d\x63\xfb\xab\xe4\x77\x3f\xee\x25\x1f\xf8\x8a\xcc\xed\x65\xe6\x09\x2b\xec\x67\x66\x9c\x8c\xb2\xb6\x00\x6d\xdc\xcf\x83\x0c\x59\xfe\x11\x3d\x8e\x3b\x9d\xa0\x4e\x7b\x0c\xe6\x89\x82\x09\x68\xae\xf4\x5b\xc3\xb9\x86\xf3\xad\x80\x73\x93\x12\xc1\xa8\x6d\x23\xb6\x79\x48\x8f\xfb\xb2\x07\xb0\x9e\x27\x58\x0d\xed\x1a\xda\x81\x86\xf6\x1d\x82\xf6\xf4\x3c\xbd\x53\x80\xec\x75\x8d\xec\x1a\xd9\x37\x8f\xec\x95\x60\x32\xf9\xe6\xa8\x31\x1c\x6a\x98\x2f\x13\xe6\x1b\xc3\xa1\x86\xfa\xf1\x5b\x68\xa8\xd7\x50\x0f\x76\x1b\xea\xcf\x52\x48\x9f\x99\xc4\x17\x41\x7d\x43\x43\x7d\x09\x50\x9f\x78\xec\x96\x20\x7d\xf2\x29\xbb\x02\xf5\x2d\x0d\xf5\xe5\x42\x7d\x4b\x43\xbd\xf2\x16\x7b\x07\xf5\x46\xcd\xd0\x48\x7f\xd0\x48\x7f\x31\x23\xd2\x37\x35\xd2\xef\xeb\xa4\x7e\x37\xa1\xbe\xad\xa1\xbe\x5c\xa8\x6f\x6b\xa8\x57\xde\x42\x43\x7d\x6e\x27\x35\xd4\xef\x0c\xd4\x4f\xcf\xb3\x3b\x2f\xc0\xfa\x96\xc6\x7a\x8d\xf5\x4b\x62\x7d\xd7\xf2\x82\xc4\xdd\x6e\x00\x23\xbc\xcb\x3d\xa7\x94\x95\xfa\xd7\x60\xd6\x27\x9a\xd4\x23\xa2\x8c\x67\x2e\x4c\x28\x66\x24\x0d\xb7\xd2\xc0\xb7\x9b\x25\x7c\xf6\x65\x0d\xe0\x23\x62\xb0\x8f\x00\x43\xdc\xa5\x84\x23\x90\xc0\x53\x4d\x09\x16\xa3\x04\x19\xd0\x53\x00\x57\xc7\xf9\xb7\x8d\x12\x98\x90\x59\xa9\x86\xfd\xa2\x6b\x68\x59\x98\xf4\xb3\x96\xe3\x57\x7e\xa6\x1e\xb1\x52\x8d\x8f\x7b\x6a\x86\x3b\x37\x52\x0d\x8e\x37\x74\xfc\xd0\x3e\xef\xb4\x3e\x34\x54\x1b\x95\xb7\xdc\x98\x30\x18\x9b\xfc\x5b\x42\x03\x51\xed\x00\x39\xe1\x38\x12\x88\xb9\xd4\x86\x02\x5d\x49\x73\x55\x2e\x45\x43\x97\x92\x00\xb8\x6b\xa7\xed\x9c\xd1\x41\x5d\x68\x62\x31\xca\x8e\x40\x9f\xad\xc4\x2e\x4c\xf0\x68\x9c\xcd\xc3\x6e\xf2\x76\x8e\x80\xe5\x58\x4e\x36\x01\x31\x4b\x73\x06\x08\x0a\x07\xba\x49\xf0\x1e\x60\x0b\xfd\x3f\x62\xf4\x6a\xec\x2f\x92\x80\x38\xc0\xfd\x81\x8d\xfb\x03\xf1\x2e\xd4\x7f\x82\x57\x04\xcc\xa9\x3d\x91\x39\x85\x76\x5b\xc8\x56\xd2\x14\x24\x97\x63\x30\xf4\x88\x18\x47\x5f\x8a\xba\x39\x27\x6e\x4f\x04\xd0\x40\xa5\x33\x23\xe8\xe9\xcf\x47\x2f\xc1\x64\xdc\x46\x85\x98\x19\xc9\x7e\x89\x09\x78\x6d\xfa\x0c\xdc\x46\x8b\xcf\xbc\x27\x60\xe7\x14\x88\xcc\x99\x3d\x87\xaf\x2b\x67\xd1\x09\xd0\x04\x81\x70\x01\x26\xe1\xa5\x33\xa4\xbd\xe7\x21\x90\x5f\xfa\x2b\xe6\x82\xf6\x19\x74\x0a\xad\x2b\x82\xcb\xb4\xf4\x8d\xe1\xdb\x8c\x97\xcc\xba\xd7\xb8\x9d\x61\x60\x76\xbf\x79\x4e\x4f\x92\xd3\x84\x20\xc2\xca\x1b\xfc\x3d\x8d\x9f\xc6\x28\xfb\x18\x05\xff\x54\x0a\x90\x0f\x7d\xf9\xc8\x91\x8b\x1b\xb9\xa8\x51\x24\x3c\xd7\xc6\x62\x6c\x58\xb9\xce\x79\x14\xbc\xd4\x55\xe8\xc0\x0d\xe8\x09\x6a\xa4\x6b\xf3\xe5\x31\xca\xc8\x23\x1f\x4f\xd6\xb0\x3d\xad\x59\x10\xd7\x6d\xc4\x53\x84\x89\x1b\xd4\x6e\xb0\x85\x1e\xf0\xb7\x83\xd8\xa0\x36\x55\x98\xad\x82\xcd\x7e\x8d\x29\x99\x6f\x3e\xcb\x78\xcb\x6f\xf3\x37\x76\x06\xd3\xea\x54\x61\x3c\xab\x4e\x55\xcc\x33\xa9\x66\x3e\x68\xdd\xe4\xee\x1b\x5d\x70\xbe\x9d\xeb\x1a\xf4\x74\x7b\xbd\xd3\x6d\x1e\x0c\xc9\xee\x7f\x68\xcf\x9f\x5c\x9b\x26\xe2\x93\x03\xec\x8b\x4e\x66\xc3\x2d\xc7\x29\x6b\xc8\x43\xe6\xc6\x44\x60\xbe\x09\xfa\x78\xef\xd9\xc0\xef\xf3\x0c\xd8\x5c\x30\xf7\x98\x2c\x8a\x7b\x88\x6d\x64\xad\x44\x12\x73\xbe\xf0\x07\xd9\x93\x49\x2f\x7b\xb5\x25\x93\xfb\xbf\xd3\x1e\x07\x2e\xa3\xbe\x7e\x90\x05\x7c\x8a\x39\x0a\x69\xc9\x72\x93\xfb\x46\xda\x7d\x1c\xc8\xe4\x3e\x36\x26\x5a\x14\xd4\x2f\x67\x72\xac\xc3\xfa\x6a\xb9\x0e\xeb\x5f\x82\xd6\x45\x01\x3b\x99\x92\xac\x57\x22\x3b\xf1\x11\xe3\x17\xc7\x15\xa3\xfc\x2a\x7f\x96\x7d\xa8\x94\x06\x40\x0e\xbe\xfb\xef\xbf\xeb\xd4\x66\x85\x5b\x01\x16\x65\x36\x32\xf4\xf0\xcd\x43\x5e\x71\xf4\x61\xe5\x24\xe7\x7a\x0c\xa2\x98\x00\x19\x8a\x90\x1d\x9a\x35\x1a\xb1\x62\xc6\xb3\x8c\x84\x16\x23\x3f\x33\x4b\x61\x4b\xa9\x50\x6f\x04\x64\xd7\x35\x0d\xd2\x34\x48\xd3\xa0\x2d\xa6\x41\x99\xb8\x7f\x21\x0f\x9a\x92\xca\xa8\x79\xd0\x3a\x78\x90\xe6\x3f\x63\xd9\x97\xcb\x7f\x9e\x28\x7b\x58\x36\x07\x72\x29\xe0\x97\x70\x1f\xf4\x62\x17\xb2\x1f\xb3\x78\xff\x1f\xda\x03\x12\x7a\x35\xe4\x6b\xc8\xd7\x90\xbf\xbd\x90\x9f\x8e\x7c\xb4\xcf\x0a\x10\x7f\x4a\x42\xe3\x0c\x88\x5f\x9c\xee\x38\x3b\xe4\x6b\x60\x3f\x50\x60\x77\xe0\x30\x89\xe9\x4f\x10\x0b\x4c\xfa\x13\x13\x14\x5f\xd6\x39\x59\x9f\x3f\x50\xb1\x49\xc8\xfe\xdd\xef\x28\xe0\xf8\xbb\xf2\xd6\x1a\xa3\x17\xc2\xe8\x95\xc3\x9f\x46\x69\xb5\x7c\x67\x50\x3a\x95\x98\x17\x8c\x38\x1b\x0a\x44\xcc\x11\xc0\x1c\x88\x01\x02\x16\xbe\xbf\x47\x0c\x11\x13\x29\x49\x4c\x80\x63\xbf\xc0\xaf\xa7\xb6\x85\xb8\x90\x74\x1a\x13\x59\x22\x3d\x0c\x78\x82\x1c\x20\x22\xff\xb6\x4e\xc1\x3f\x18\x18\xd0\x27\x60\x53\xd2\x0f\x2e\xe5\x20\x74\x8f\x00\x0b\x0e\x84\xc7\x08\x10\x14\xf4\x10\x40\x43\x64\x7a\x02\x59\xa7\xeb\x89\x1f\x14\xb2\x89\xc9\x49\x7e\x9a\x4d\x68\x36\xb1\xda\x0c\x10\x95\x4e\xc8\x51\xd4\x0d\x07\xa6\x66\x12\x8b\x31\x89\x40\x7c\x58\xb3\x89\xa5\xd9\x84\x66\x12\x9a\x49\xac\x6f\xbe\x7f\xd6\x2a\x40\xe8\xb3\xc9\x08\xad\x37\x30\x6a\x3c\x55\xf0\x74\x10\x65\xcf\x77\xbf\x79\x90\x08\x6c\xa3\xe3\xda\x69\xa7\x5d\x01\x79\x01\xf9\x2e\xf3\x48\x88\x95\x33\xec\x91\x50\x03\xf3\x36\x3a\x39\x01\xab\x86\xdf\x4e\xfb\x27\x10\xbd\x05\xa0\xf7\x20\xec\x39\xd8\x72\x1c\xfe\x3b\xed\x45\x31\x78\x9f\xf8\x47\xbb\x0c\xf5\xe6\x42\x8d\xc9\x1a\x93\xb7\x1a\x93\x33\xd3\xe6\x42\x50\x3e\xd7\xa0\xac\x41\x79\xdb\x40\x39\x58\xa7\xae\x80\xb5\x80\xf3\xee\xad\x8c\x17\x81\xb2\x5e\x26\x07\x1a\xa0\x35\x40\x87\x85\x5b\x0d\xd0\xe9\x49\xf3\x79\xa3\x00\x9f\x2f\x26\xe3\xf3\x7a\xd2\xe2\x74\x54\x7b\x27\x00\x7f\x5d\xc9\x6f\xd1\x32\xd1\x96\xe5\xb6\xef\x56\x90\x3b\xdc\xbb\x2d\xf7\x21\x02\xd7\xe3\x03\x9d\xd6\x5e\x1e\x78\xeb\xf5\x73\x0d\xe0\x6b\x9e\x61\x17\x22\xf8\x94\x2f\xe4\xee\xe4\xc2\x74\xfa\x0e\x8d\xe1\xdb\x8b\xe1\x99\x3c\x37\x86\x04\x1b\x4d\xc9\x72\xd3\x58\x9d\x3c\x42\x48\xc8\x9d\xf7\x42\x87\xbd\x35\x30\xef\x23\x30\xaf\xe3\xb0\xa3\x8b\x82\xf3\x79\x9a\x93\x0f\x3b\x62\xc8\x45\x81\x56\x2c\xe4\xda\x74\xe4\x20\x22\xde\x51\x72\x8f\xfb\x46\x76\xa0\x5e\x53\x8b\x83\xe3\x1f\xd3\x57\x9e\xcc\x71\x42\x92\x09\xcd\x01\xba\xc5\x0e\xa2\x5e\xc6\x21\xc8\x13\xfc\xae\xa0\xf9\xd0\x67\xe1\xc1\x53\x09\x77\x2d\xab\xff\xe9\x8f\x93\x8c\x18\xcd\x88\xf4\xc4\x43\xc4\xf8\xe1\x43\xa3\xd5\x69\xbf\x53\x07\x24\xeb\xf7\xe0\x71\xa3\x79\x5e\x01\xf5\x46\xa7\x02\x5a\xb5\x0a\xa8\x9d\x5e\x74\x54\x77\x6c\xfc\xd0\xe8\x74\xcc\xd6\x99\x91\xb1\x8e\x99\x12\x08\xb3\xa7\x7b\x29\x76\x4f\x28\x51\x2e\xee\x43\x4f\xe2\xe4\x73\x82\x45\x44\xef\x57\xaf\xd5\x92\x44\x22\xaa\xa8\xcd\x40\x17\x22\xd7\xfd\xd1\x1f\x55\x19\x70\x55\xaf\xf8\x04\xd9\x03\x62\x85\x14\xa2\xd0\x3c\x9b\x29\xf3\x4c\x7f\x62\x21\x63\x9d\xd9\x2f\x22\xfb\x6c\xca\x37\x84\x08\xca\x73\x0f\x17\x6c\xc6\x1c\x25\x77\xaf\x56\x1e\x73\x71\xa0\xeb\x62\xd2\xbf\x0d\x4c\xb1\x9e\x57\x3e\xc1\x9b\x86\x4e\x3b\x70\xc8\x40\x50\x20\xd0\x30\xe5\xac\x1e\x23\x1d\x4d\xf5\x79\x51\x63\x0c\x92\xfe\x94\xc6\x1a\x13\x1c\x93\x03\x87\xef\xa1\x80\xd7\x11\x57\x52\x8c\x23\xcb\xd3\x4c\x4a\x08\x32\x05\x52\x0e\xb1\xf1\xaf\xb9\xf5\x9f\x9c\x1a\x70\xf9\x24\xce\xf6\xfa\x98\xfc\x13\x31\x1e\xe6\xc1\x9e\x9d\x36\x4e\x5b\x86\xc2\xd8\xb8\xb8\xc7\xc3\xa4\x1a\xc2\xc2\x0f\x94\x44\x27\xbf\x19\xed\xda\x4f\x4a\x3d\x43\xd9\x7b\x64\x59\xe1\x2d\x52\x66\x9f\xa0\x3b\x09\xf9\x02\x1e\x92\xa4\xa6\xb2\x46\x04\x6f\x6b\xfc\xf6\xfa\x6d\xaa\x82\x8e\x6f\x98\x20\x70\xee\x42\xf6\x60\x07\xbc\x50\xb1\x7c\x7f\x82\x33\x3e\xa0\x54\x3a\x93\x66\xbd\x02\xea\xf5\x8b\x0a\xa8\x5f\x74\x7c\x67\x52\xbf\x48\x38\x93\x7b\xcf\xce\xe3\xed\x7e\xcb\x6a\x3b\x41\x33\x8d\x5a\x05\xd4\x3b\xcd\x44\x03\x13\x4f\xcf\x14\xfe\xac\xe6\x1d\xb5\x3d\x27\x75\x90\xe8\x5c\x01\xa8\x07\xaf\x87\xba\x0c\xb9\x36\x36\xe5\x41\x98\xf1\x67\x0f\xba\xc1\x67\x27\xba\x0c\x41\x6b\x14\x5d\xc2\x8b\x4e\xc6\xcc\x6d\xe2\xcd\x9f\x47\x19\xb8\xa8\x7e\x0d\xcf\x99\x2e\xfd\x88\xe9\x05\x98\xad\x51\xaf\x34\x8c\x3c\x72\x6b\x34\x6b\xdc\xc8\x25\xb1\xe9\x9a\x88\xc5\x7a\x84\x60\xd2\x07\x2e\xb5\x78\x16\x10\x39\x26\x7d\x1b\xf9\x02\x8d\xeb\xe4\xd0\x57\xed\xff\x42\xb5\x7f\x59\x3b\xd9\xfe\xa9\x4f\x90\x8d\x37\xf9\xa6\x5f\xcb\xf7\x33\x53\x6d\x5f\x5e\xf8\x5b\xe8\xb8\xfc\x19\xf2\x4a\x70\xfc\x3a\x72\x08\x39\x40\x3e\x07\xc6\x87\x60\x3d\x2f\xc6\x87\xd4\x40\x63\xfc\x32\x18\x7f\x56\x16\xc6\x37\xf2\x30\x3e\x61\x51\x1a\xe5\x4b\x47\x79\x8d\xe2\x07\x83\xe2\x2e\x32\x57\x81\xde\xa0\x0a\x34\x7f\x28\x8f\x3f\xfc\x41\xe0\x23\xc4\xb6\x6f\x0d\x9a\x43\xe8\x58\xc0\xfe\xf0\x84\xcc\x5a\xce\xc2\x44\xa1\xa9\x83\x01\x3a\x18\xa0\x69\xc4\x8a\x68\x84\x5c\xb2\x3a\x8e\xfe\x25\x02\x62\x82\x58\xd7\x41\x0e\x65\xa3\xae\xc7\x61\x1f\x75\x7b\x23\x81\x0a\x01\x3c\xf8\x34\x46\x0e\x5c\xe7\x7d\x63\x2a\xc8\x5c\x21\xd4\x42\xe5\x7f\x69\x6a\xa3\x30\x2e\x63\xe4\x16\xe6\x82\xe1\x9e\x27\x90\x05\x28\x01\x03\xca\x85\xc6\xf3\x32\xf1\x7c\xc1\x79\xbf\xd5\x6a\xc1\x26\xd4\x78\xbe\x1c\x9e\xa7\x3f\x9f\xbc\x30\x9e\xb7\xf4\xc4\x7f\xfd\x88\xae\x11\x7b\x6f\x10\x5b\xe6\x9e\x78\xce\xb1\x85\x6c\x01\x83\x89\xb8\x4b\xad\x6e\x0c\xde\xe3\x09\x38\x17\x90\xcd\xf6\x29\xea\x19\xf1\xfb\x6b\xdb\xb9\x3b\x09\xf3\x4f\x5d\x6a\xad\xfc\x6b\x91\x05\x29\x2c\xb3\x1f\xa0\x5c\x1e\xc2\x7f\x82\x43\x39\x41\x07\x91\x58\xc1\xb1\x0d\xb9\x00\x6d\xe0\x60\xe2\x09\xc4\x73\xd6\xc3\xf7\x0d\xea\x77\x27\xd5\xf0\x3c\x85\x66\xd3\x13\x29\x0a\xf6\xf2\x35\xa7\x1c\x81\x33\xe7\x5e\xbe\x95\x1f\x93\xbb\x45\x1b\x00\x1d\x38\xbc\x46\xec\xb3\xec\x4f\xb3\x18\xda\x56\xf1\xa9\x80\x76\xb2\x7c\x9e\x4c\xc3\x71\x6a\x4c\x02\x54\x83\xd2\xf7\x98\x21\x33\x3c\x28\x2a\x51\xbd\x75\x7b\x0a\x37\x14\x1c\x5e\x75\x86\xa3\xb4\xe3\x6a\x22\x4e\xaa\x5c\xf7\x11\x93\x87\xbc\x3c\x30\x65\x66\x98\x28\xf7\x25\x2d\x35\x32\x95\xf2\xed\xe9\xca\x79\x5a\xbc\x32\x14\x9d\x23\xde\x5c\xe0\x5d\x52\x54\x7a\x79\x62\x41\x25\x79\x64\xaa\x9a\xde\xcd\xae\xa6\xbd\x09\xff\x64\xc4\xc4\x91\x55\x4d\x86\x60\x92\x42\x7a\xbf\x25\x79\xd0\xd7\xd4\x02\x52\xfe\xe0\x58\xba\xb7\x0a\x90\xfa\xad\x00\x8f\xf8\xff\x9f\x00\x48\xac\x80\x79\xca\xb7\x89\xa3\x4c\x3e\x0e\xc5\xcd\xe9\x3d\x4d\x8b\x7c\x8e\xbc\x56\x60\x85\x2b\x4f\x56\x4e\x3e\x78\xc7\x32\xaa\x55\xb6\xa8\xf7\x3a\xe5\x4e\x40\xce\xe6\x9d\x80\x74\xb2\xe1\xb4\x60\x02\x52\xee\x09\x5f\x7a\x02\xb2\xad\x5b\x9c\xf6\x67\xe2\xb1\xb1\xc0\xd4\xda\xe2\x52\x72\xa7\x95\x4b\xad\x5d\xd8\x67\x75\x3d\x6b\xd0\x4a\x6f\xbe\xd2\x9b\xaf\x4a\xa7\x0a\xfb\xbc\xf9\xaa\x73\x5e\x00\xd9\xf1\xf9\x5f\x4b\x6f\xbe\x7a\x77\xfd\x07\xf8\xc3\x9f\x86\x2d\xb9\x03\x6b\x77\x98\xd3\xdc\xa1\xdb\x4e\xc1\xd7\xcf\x9a\x53\x8e\x79\x39\x8c\x63\xd8\xf6\x3b\x0a\x6b\x28\xf8\xb1\x95\x5c\x88\x50\x0b\x75\xc7\xac\x26\xc9\x87\x2e\x63\x66\x64\xba\x5e\x18\x6d\x89\xce\x75\x93\xc6\x72\xf9\xfc\x0c\x4e\x6f\x3c\xe7\x33\x14\x08\xbc\xbc\x28\x54\xe9\xcf\x65\xa3\x30\xe5\xd0\xa4\x19\x36\xa4\xe7\xd3\xa4\xd2\x22\xb6\x9b\xa4\x56\x63\xe7\xac\xe3\x30\x5b\xce\xa2\x74\xb8\x45\x29\xdf\x2a\x12\x55\xaf\xb5\x0b\xe0\xbb\x53\x2e\x8b\xfa\xdd\xa3\x02\xae\x95\x45\x99\x32\xcb\x23\xd5\xef\x35\x50\xab\x7b\x25\xbb\xa0\x5e\x53\xd3\x0b\xca\x23\x5d\xf5\x5a\xc1\x37\x63\x5a\x35\xcd\xba\x56\xc2\xba\x60\x1f\x85\x4a\x4d\x80\xd1\xd6\xb2\x31\x93\x51\x69\x97\xc9\xc3\x77\x26\x72\xb4\x01\x7d\xfa\x15\x41\x4b\x76\x21\x79\x5b\x00\x94\x8a\x1d\x99\xd4\x4e\x79\x1b\x0b\x71\xb3\x50\x9b\x25\xd2\x3f\x2e\x46\xf6\x24\x54\x93\x3e\xc2\x17\xc6\x6d\x12\x55\x83\xc1\x8c\x62\x76\xf4\xe5\xcb\x97\x2f\xd5\x4f\x9f\xaa\xef\xdf\x83\x5f\x7f\xbd\x74\x9c\x4b\x9e\x62\x5b\x2e\x14\x02\x31\x92\xdf\x56\xe4\xab\x06\xd8\xb2\x10\x99\xbe\xe0\x37\xee\x56\x96\xb4\x44\x02\xa5\x2c\xb4\xcb\x0c\x10\xc5\xd9\xb8\x77\xcb\xbc\x90\xb2\xf0\x92\x22\x8e\x01\x0f\x4c\x0d\xd1\xb0\xe2\x76\x4c\xa9\x8c\xf7\x0c\xdb\x36\xb0\xe8\x13\x31\x32\x97\xfd\xc1\x72\x3e\xe8\xab\x88\x50\xa6\xdb\x82\x1f\xd2\xa9\x85\xf9\x6c\x31\x21\x62\xe2\x39\x3d\xc4\x92\xf7\x79\x04\x2b\xa4\x60\x3e\xe9\x7f\x46\xdf\x3c\x94\x59\x98\x3c\x14\x05\x5c\x6d\x8f\x02\xc0\x4f\x87\xa9\x82\x77\xe5\xaa\x20\x84\x20\xf9\x73\x3e\x45\x7c\xc4\x0e\x3e\xd4\x71\xf0\x7e\xf3\xe3\x20\x10\xff\xa1\x8e\x82\x5f\xb6\x61\x14\x5c\x53\x6b\xfb\xa4\x9f\x64\xd0\x0b\x09\xff\xb5\xf5\xfa\xf9\x19\x9c\xfe\x16\xc5\xa8\xc0\xcb\x4b\xa6\xa0\xda\xe4\x26\xb4\x51\xf5\xc1\xeb\x21\x46\x90\x40\xbc\x6a\x52\xc7\xf5\x04\xaa\x32\x14\xcc\x72\x78\xd5\xa5\xd6\xff\x3e\x42\x56\x8d\x43\x5f\x71\xe0\xeb\x2f\x7e\x85\x4b\xad\x37\x3f\x76\xbb\x26\x4a\x6f\x27\x50\x34\xee\xa6\xa5\xbc\xee\xd1\xb6\x45\x3a\x56\xc4\xf2\xfa\xf4\xe7\xd7\xf3\xcb\x85\x0b\x86\x49\x7f\x36\xb9\x84\x7f\x29\x41\xb5\xfd\x0f\x9e\xca\x53\x67\x13\x95\x98\x70\x01\x73\xce\x96\x9d\x3f\xa2\x5a\xb8\xd6\x9c\x28\x5f\x22\xb1\x35\xb9\x60\x1f\x0d\xc3\x2e\x0b\x19\x93\x94\xb5\x49\x59\x22\x21\x70\xef\xc4\x79\x55\x86\x38\xb7\xda\x78\xc1\x6b\xa0\x55\x9e\x4c\x99\x5d\xf1\x08\xb2\x25\xd7\x3a\x0c\x61\xbe\xd7\xe3\xe7\xb0\x14\xfe\xcb\x7a\x96\xe9\xa6\xaf\xc6\xc9\x20\xff\x46\x72\x9a\x18\x24\xdc\x57\x42\x56\x05\x63\xe2\x94\x2a\xd6\x2b\x75\x87\xb2\x52\xf7\x0a\x6c\x6a\x71\xad\xde\xcc\xa8\x3a\x58\xa5\x89\x57\x2f\x96\x5e\x5c\xfb\x24\x37\x8b\xe8\x2c\xa5\x69\xaa\x28\xd8\x61\xda\x8a\x73\x86\x0f\x78\xc1\x6c\xdb\x13\x92\xb6\x32\xb9\x68\xc6\x0d\x5b\x8b\xd3\x8d\x0a\x18\x3f\xe2\x7f\xde\x1c\xe9\xd4\xa1\xf2\x52\x87\x54\xa7\xa9\xb3\x87\x96\xe3\x24\xd2\xde\x35\x27\x39\xc8\xec\xa1\x46\xc1\xb6\xa9\x56\xb3\x74\x82\xa3\x13\x88\x4a\xe5\x43\x8d\x82\x6f\x7b\xb5\x5a\x9a\x0f\xed\x6e\xaa\xd0\xa6\xd2\x7e\x72\x0f\x3b\xd3\x59\x3f\xe3\x6e\xe5\xf2\x8d\x48\xa4\x1b\x5f\x70\xdc\xc1\xc4\x1f\x0b\x99\x01\xef\x98\x5b\x07\x3a\xfd\x67\xab\xd4\x70\xa8\xe9\x0f\x5b\x91\x04\x14\xea\x42\xe7\x01\x95\xa6\x87\xc5\x07\x84\xce\x06\x2a\x51\x0d\x3a\x1b\x48\x67\x03\xe9\x6c\xa0\x52\xb3\x81\xb6\x31\xda\xb9\xd7\xb9\x3e\xa1\x98\xcb\x13\xf0\x96\x4a\xb4\x94\x74\x9f\x0d\x5a\xe7\x3c\xc9\x3c\x87\xa2\xd3\x75\xe5\xf3\x1c\x8a\x3c\x4b\x49\xe9\xd9\xee\x31\x72\x60\x1a\xdd\x8a\x9c\x1d\x35\xb4\xae\xd3\x76\x74\xda\xce\x56\x2d\x91\xbd\x02\x1b\x5b\xd5\xea\x64\x54\x1d\xac\x8d\x28\x91\xf9\x65\x57\xb5\x7e\x43\xe2\x89\xb2\x07\x9d\xb7\x33\x45\x17\xcd\x82\x4f\xec\xb7\xca\x3d\x98\x71\x9f\xd6\xa9\xca\xca\xd7\x59\xe2\x34\xc5\xad\xcc\xd7\xc1\x0c\x0a\xa4\xb0\x20\x12\x0c\xc1\x2e\x43\x26\xc2\x8f\x21\x11\xca\x9c\x8a\xb8\x04\xf3\x28\xfd\x58\xc4\x85\x93\x76\xb6\xee\x58\xc4\xcf\x81\xcc\xc1\x15\x24\x56\x60\xe9\x4b\xd1\x8f\x43\xcd\xc2\x51\x02\x4b\x19\x74\x8d\x0d\xec\xca\x5d\x69\x7e\xce\x8e\x9f\x91\xa8\x8f\x53\x5e\x05\x6c\x17\x1c\xce\xd8\x3a\xd7\xb0\xad\x61\xbb\x14\xd8\x96\x93\x47\x07\x0b\x8d\xdb\x6b\xc4\xed\xdb\x50\xe8\x1a\xb8\xcb\x8a\x0d\x68\x70\xde\x4a\x70\x5e\x4b\xa0\xa1\x55\x70\xf8\x9e\xfa\x31\xca\xa5\x23\x0d\x6f\x4d\xe1\x33\xed\xcf\xc8\xa4\xcc\x5a\x6f\x02\xed\x84\x6f\xd3\x6e\x92\xce\x5c\xcc\xaf\xa8\xa2\xe3\xf6\xf4\x21\xc7\x2b\xa0\x33\x53\xbf\x7f\xbe\xcf\x74\x47\xfa\x55\xf9\x65\x2c\x68\xa2\x63\x06\xb1\xcd\xbb\xe1\xf7\x67\x31\x25\x5d\x97\x52\x5b\xf9\x5d\xd2\x1a\x49\xe5\x28\x6e\xf2\xa8\x02\x8e\x7e\xac\xfb\xff\x72\x01\x05\xf2\xff\x38\x3e\xfd\xf9\xe4\x28\x64\x44\xf1\x85\x8b\x13\x23\x99\xd9\xee\x18\x15\x30\x91\x35\xcd\xf0\x11\x88\xb8\x33\xbb\x40\x9e\x3e\xfb\xca\x04\xd7\xd4\x02\xef\x62\x05\x6e\x64\xdd\x65\xe7\xf9\xd3\x0c\x1f\x91\xb2\xa9\x09\xd3\xcb\x7a\x11\x69\x49\xad\xca\x1d\x22\xb3\xda\xd4\xa7\x21\x76\x27\xea\x91\xa6\x09\xf5\xc6\x34\x9a\xd0\x2e\xd8\x64\xdc\xde\xf8\x26\xe3\x74\xf1\x5e\xf0\x84\x7d\xe6\x01\xa0\x00\xfa\x39\xfe\x8e\x4a\xca\x8b\x58\x00\xbe\xa3\xa4\x88\x6c\x4a\x70\x1a\xda\x97\x40\xf6\x9d\x40\xf1\x18\xc1\xc1\x35\xa5\x36\x90\x3b\xbe\x34\x94\x2f\x07\xe5\xf5\x0d\x41\xf9\x6c\x18\xa9\xb1\x7c\x8f\xb1\x3c\x51\x30\x19\xcc\xeb\x1a\xcc\x35\x98\x97\x02\xe6\x4f\x10\x0b\x4c\xfa\x9b\xc3\xf3\xc3\xc6\xec\x7f\x05\xe2\xd7\xb0\xad\x67\xe0\x7b\x84\xda\xed\x24\x64\xbf\x0a\xdb\xf4\x07\xa5\x3f\xde\xe4\x60\xaf\x05\x9e\xc0\xe0\xe6\x00\x39\x30\x0e\xb5\xd6\x2f\x82\x62\x31\x0a\xc6\x8d\x05\xd9\x43\x70\xa5\x80\xfd\xd8\x18\x8c\x60\xb3\x58\x28\x34\x83\x8f\xb8\x40\x8e\x31\x7e\x92\x40\x8e\x6b\x43\x39\xb4\xa2\xee\x1b\x36\xe6\x42\x31\x26\x15\x9e\xb1\xb4\x63\x75\x1c\x61\x62\xda\x9e\x85\xde\xda\x79\xa8\x96\xaf\x1c\xc3\xf1\x6c\x81\x73\x2e\x0f\x87\x83\x91\xc3\x1f\x12\x30\xa5\x6e\xb0\x32\xbe\x79\x88\x8d\xe4\x66\x35\x46\x1d\x24\x06\xc8\x53\x5d\xa7\x22\xca\x7a\xa2\xb4\x8f\x86\x29\xd3\x36\xf8\x03\x76\xff\x60\xf6\xcd\x88\x98\x79\xc8\x1d\xba\x01\xa5\x73\xe9\x71\x98\x50\xbf\x2d\xb7\x4b\x66\x5f\x3e\xa6\x17\x09\x23\x8f\x94\x96\xda\x3b\x86\x86\x61\x04\x35\xb9\x05\x30\x39\x06\x1e\xc3\x27\x65\x2f\xcb\x1b\x5b\x0b\x6b\xd1\x18\xc3\x9d\x31\x87\x32\x73\x6f\x52\x74\xa9\xbc\x88\x2a\x10\x49\x0f\x6c\x64\x8a\x1c\x7f\x3e\xbb\x68\x66\x13\x4e\x3c\xa4\x95\x41\xad\x9a\xd6\x84\x67\xcc\x68\x35\xa6\xc7\x05\x75\x66\xb4\x98\x84\x55\x4e\xe7\xd6\xd2\xc3\xdf\x63\x82\xa3\x8f\x0b\x05\xeb\x12\x01\x60\xc4\x1b\x39\x30\xb9\xa7\x93\x3f\x21\x1d\x38\x87\xea\xe9\xcf\x47\x2f\x15\x90\x4a\xa7\x98\x6a\x35\x29\xb8\x1a\x1b\x4d\xe1\xb2\xe7\x2c\x8e\x60\xc2\xbd\xd3\xdc\xc1\x4a\x84\x30\xc5\x9b\xbc\x3e\xf6\x49\x5e\xc8\xf5\xee\xfe\x7a\x52\xfd\xea\x13\xbe\xc4\x66\xd1\x29\xf6\x12\x32\x15\xb5\x75\x01\xfb\xd2\x30\xf8\xef\xd1\xab\x19\xc9\xda\x8c\x0c\xfc\xb2\xfc\x8b\x43\x6b\x0c\x84\xa4\x54\x78\x1c\xdd\x06\x0d\x25\xe6\x1f\xf2\x7f\x9f\x6e\xbc\x04\x60\x81\xa5\x5a\x42\x98\xb8\x0f\x78\x9f\x41\xe8\x53\xb5\x1e\x11\x23\x43\xd0\xb0\xcc\x48\xdc\xe6\x62\xf3\x41\x4e\x01\xc2\x9b\x43\x41\x76\x23\xee\xaa\xba\x02\xa3\xad\xac\xb4\x44\xf8\x27\x7f\x34\xd5\x1f\x0a\x0d\x36\xda\xca\xdf\x75\xf5\x47\xb3\xa6\xd6\x28\xf4\xad\xa1\xfc\x5d\xb7\x82\x81\x79\x17\xbd\x83\xcf\xc2\xb3\x2e\xaa\xf8\x29\x6a\xc3\x67\x6a\xc3\xea\x53\x1a\x2d\xf5\x47\xbc\xb1\xda\x38\xb7\xd4\xfe\x46\x7d\x49\x88\xef\x3b\x25\xb1\x63\x88\xb9\x72\x80\xed\xe0\x35\xb8\x09\x60\x5d\xd6\x3e\xc6\x14\xe1\xd5\xcb\xab\xff\x06\x00\x00\xff\xff\xe4\x64\x6c\xc0\xa5\x30\x01\x00") +var _monitoringSystemGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6d\x73\xdb\xb8\xb5\xfe\x9e\x5f\x81\xcb\xdd\xad\x9d\xad\xe4\xe8\x35\xb6\x3c\x93\xb9\x13\x27\x9b\xbb\xed\x24\x5b\x6f\xec\x6d\x27\x37\xe3\x51\x21\x12\x96\x50\x93\x00\x03\x80\xb6\x14\x8f\xf7\xb7\x77\x08\x92\x22\xf8\xa6\x57\x4a\xa6\x24\x7c\x49\x2c\x00\x04\x81\x73\x0e\xce\xf3\x00\x38\x00\x1f\x5f\x00\x60\x40\x42\xa8\x80\x02\x53\xc2\x8d\x73\xe0\x27\x01\x60\xd8\x98\x0b\xe3\x1c\x7c\x95\xbf\x40\x98\x2a\x73\x06\x1e\xb6\xc5\xdf\x88\x71\x0e\x9a\xb5\x38\xd5\x82\x02\x72\xea\x31\x13\x19\xe7\xc0\xa8\xd7\xc1\xff\x31\x78\x0b\x09\x04\xf5\xba\xa1\x14\x43\x04\x0e\x6c\xbf\x88\x60\x1e\x52\xd2\x47\xd8\xca\x49\xc5\x26\x25\xef\xa8\x4d\x99\x5f\x27\x1b\x0e\xe0\x71\xa3\x06\x5a\xcd\x66\x0d\xb4\xba\xdd\x1a\x68\xbe\x54\xab\x26\xd0\x91\xef\x7e\x1b\x77\x07\xfc\x05\xbc\xb5\x11\x13\x5c\x2d\x27\x26\xae\x2c\x67\x41\x3e\x1a\x50\xc8\x2c\x23\xcc\x7b\x92\xff\xdf\xbc\x00\xe0\xc9\x2f\x6e\x58\x88\x9b\x0c\xbb\x7e\x4d\x7e\x79\x59\x87\x81\x2c\x2c\x52\x5d\x30\x86\x04\x89\xbf\x59\xc6\x39\x20\x9e\x6d\x07\x29\x0c\xba\xa3\x6b\x4a\x6d\x81\x5d\xe3\x1c\x34\x64\x22\xf6\x8b\x9c\x06\x7f\x0a\xc4\x60\x58\x71\xf3\x75\xa3\xd9\x3b\xed\x36\xbb\xcd\x66\xbb\x27\x73\x6d\x4c\xee\x7c\x55\x7c\xbd\x91\x3f\x5d\x48\x90\xcd\xa7\xca\x88\x54\x61\x98\xd4\xb6\xa1\xcb\x91\x5f\xed\x2d\xb4\xf9\x54\x72\xc6\x90\x61\xeb\x92\xc6\xda\x0c\x44\x9c\xd2\xd8\x83\x71\x0e\x5a\x1d\x25\x61\x1c\xb5\x34\xfc\x3d\xf1\x7f\x47\xb2\x99\xd6\x2d\x7b\x71\x36\xfd\x19\x37\xee\x66\x9a\x26\xb0\x90\x02\x32\xfe\x85\x06\xe0\xad\xeb\x1a\x71\x4e\x28\x7b\x46\x1f\x02\xa9\x87\xf5\x4e\xfb\x04\x6d\x0c\xb9\x54\xb9\x6c\x7d\xfc\xda\x01\x94\x29\xc9\x7e\xfa\x2a\xfc\x88\xc8\x50\xc8\xbe\x35\x12\xe9\x28\xaf\xb8\x6a\xa3\x3f\x2a\x3f\xa7\x45\x6e\xb1\x6d\xab\x72\x2a\x16\xe5\x59\x4a\x94\xcd\xd6\x1c\x51\x36\xf3\x45\xd9\x9b\xfe\xb4\xd1\x10\x11\x2b\xf9\x26\x78\x3f\x4c\x77\xc3\xd7\xbc\xc7\x18\x22\x22\x27\xc7\x81\xe3\xbc\x54\x4c\x72\x52\xf9\x88\x3e\x64\xc7\x9c\xa0\x02\xda\x39\xa5\xef\xa1\xed\xc5\x32\xcd\xf4\xc5\xc6\x44\xe6\xaa\xb5\xc9\xc4\x07\x6c\x89\x84\xe9\xa5\xcc\x5b\x26\xf9\x03\xe7\x92\x62\x22\x3e\x51\xe9\x07\x64\x42\xac\x15\xea\x4e\xbd\x53\xfc\x46\x17\x31\x13\x11\x01\x87\x28\xa3\x68\xd7\xaf\x8a\x41\x0b\x7b\xfe\x33\xad\x64\x7a\xd6\x2e\x18\x22\x16\x62\x48\x7a\x99\x5b\x9b\x8a\xf8\xc5\x1c\x31\x8c\xf8\x3f\xee\x11\x63\xd8\x42\xa9\x46\x73\x17\x9a\x28\xcf\xfc\xb8\x80\xe6\x5d\x5a\x16\x5c\x20\xd7\x45\xd6\x47\x4c\xb2\xed\x15\x90\x0d\x91\xe0\x8a\xbf\x55\x3d\xae\xef\x75\xc6\xae\x6c\x1d\xf7\x9c\x63\x06\x05\x3a\x66\x10\xdb\xbc\xcf\xd0\x37\x0f\x71\xc1\xfb\x52\x69\x8f\xbe\x07\x94\x8d\x7a\x73\xf4\xe3\xf4\xef\xa3\x9a\x4b\xad\x37\x7f\x1e\xf1\x09\x17\xc8\xa9\x43\xd7\xad\x7f\x85\xf5\xef\x8d\x7a\xef\xe6\xaf\xf1\x5f\x47\x4f\x5f\x9b\xce\xcd\xcb\x97\x60\x30\x01\xc7\x5c\x40\xe1\x71\xd5\xb9\xfa\x23\x83\x32\x07\xfa\x26\x67\x08\xec\xa0\x7e\x20\x99\x64\x11\x4c\x04\x62\xf7\xd2\x7a\x8c\xa6\x93\x9f\xf7\x01\x9a\x42\xfa\xf3\x66\x22\x3b\xb0\xfd\x0f\xd3\x77\x3c\x3e\xfe\xfb\xf1\x31\x68\xc7\xd3\xd3\xbf\x9f\x9e\x92\x95\x31\x74\x2b\xfd\xad\xf1\xd6\x98\x26\x3f\x85\x7f\x29\x6e\x68\xc4\x10\x1f\x51\xdb\xca\xb8\x27\x07\x7d\x60\xd4\x51\x1c\xf6\x34\xfd\x33\x1a\x86\x96\x96\x7a\xe0\x6a\x84\x6f\x45\xf6\x89\xd0\xd1\x5d\x49\xe1\x82\x48\x1f\xc0\x45\x0c\x70\x64\x52\x62\x81\xe3\xc1\x04\xa4\x05\x6a\x88\x29\x34\x3c\xaa\xe3\x11\x32\xe9\xcc\x53\x23\x92\x53\x26\x52\xfe\x44\x0e\xc6\x7e\xe4\x4d\x31\xb1\xf0\x3d\xb6\x3c\x68\x1b\x99\x71\x19\x95\x91\x88\x14\x37\x60\x0c\xc7\x38\xe5\xd4\x06\x9e\x79\x17\x18\xa1\xda\x47\xdf\x7b\x84\x63\xd2\x17\x43\x0e\xe0\xa6\x4a\xe7\x7b\x95\xa9\xf7\xf8\x7a\x93\x69\xe2\x04\x8e\xd1\x0c\xdb\xb7\x90\x89\x1d\x28\x31\xa6\x59\x60\x91\x0c\x7d\x73\x53\xb6\x68\xc3\x01\xb2\xa7\xb0\x1d\x27\xd3\xe1\x05\xe4\x28\x53\x57\xe0\x37\x93\x5d\x99\x3a\xce\x4c\xb2\xd2\xc7\xd8\xfe\x6a\xf9\xcd\x8f\x5b\xc9\x47\xbe\x22\x73\x5b\x99\x79\xc3\x06\xdb\x99\x19\x27\x93\xac\x2d\x40\x1b\x0f\xf3\x20\x43\xa6\x7f\x44\xf7\xd3\x46\x27\xa8\xd3\x1e\x83\x79\x22\x61\x06\x9a\x2b\xed\xd6\x70\xae\xe1\xbc\x12\x70\x6e\x52\x22\x18\xb5\x6d\xc4\x9e\x1f\xd2\xe3\xb6\xec\x01\xac\xe7\x09\x56\x43\xbb\x86\x76\xa0\xa1\x7d\x87\xa0\x3d\x3d\x4f\xef\x15\x20\x7b\x53\x23\xbb\x46\xf6\xe7\x47\xf6\x5a\x30\x99\x7c\x73\xd4\x1a\x8f\x35\xcc\x97\x09\xf3\xad\xf1\x58\x43\xfd\xb4\x17\x1a\xea\x35\xd4\x83\xdd\x86\xfa\xd7\x29\xa4\xcf\x4c\xe2\x8b\xa0\xbe\xa5\xa1\xbe\x04\xa8\x4f\xbc\xb6\x22\x48\x9f\x7c\xcb\xae\x40\x7d\x47\x43\x7d\xb9\x50\xdf\xd1\x50\xaf\xf4\x62\xef\xa0\xde\x68\x18\x1a\xe9\x0f\x1a\xe9\xcf\x16\x44\xfa\xb6\x46\xfa\x7d\x9d\xd4\xef\x26\xd4\x77\x35\xd4\x97\x0b\xf5\x5d\x0d\xf5\x4a\x2f\x34\xd4\xe7\x36\x52\x43\xfd\xce\x40\xfd\xfc\x38\xbb\xd3\x02\xac\xef\x68\xac\xd7\x58\xbf\x26\xd6\xf7\x2d\x2f\x08\xdc\xed\x07\x30\xc2\xfb\xdc\x73\x4a\xd9\xa9\x7f\x05\x16\x7d\xa3\x49\x3d\x22\xca\x78\xe7\xca\x84\x62\x41\xd2\x70\x2d\x0d\xbc\xda\x2c\xe1\xb3\x2f\x6b\x00\xef\x11\x83\x43\x04\x18\xe2\x2e\x25\x1c\x81\x04\x9e\x6a\x4a\xb0\x1a\x25\xc8\x80\x9e\x02\xb8\x7a\x9d\xbf\x6a\x94\xc0\x84\xcc\x4a\x55\xec\x27\x5d\x42\xcb\xc2\x64\x98\xb5\x1c\x3f\xf3\x33\xf5\x88\x95\xaa\x7c\xda\x52\x33\x3c\xb9\x91\xaa\x70\x7a\xa0\xe3\x87\xee\x69\xaf\xf3\xa1\xa5\xda\xa8\x7c\xe4\xca\x84\xc1\xd8\xe4\xdf\x12\x1a\x88\x72\x47\xc8\x09\xc7\x91\x40\xcc\xa5\x36\x14\xe8\x42\x9a\xab\x52\x14\x8d\x5d\x4a\x02\xe0\x6e\x9c\x74\x73\x46\x07\x75\xa1\x89\xc5\x24\x3b\x02\x7d\xb6\x12\xbb\x30\xc1\xa3\x71\xb6\x0c\xbb\xc9\x3b\x39\x02\xd6\x63\x39\xd9\x00\xc4\x2c\xcd\x19\x21\x28\x1c\xe8\x26\xc1\x7b\x84\x2d\xf4\xff\x88\xd1\x8b\xa9\xbf\x48\x02\xe2\x08\x0f\x47\x36\x1e\x8e\xc4\xbb\x50\xff\x09\x5e\x11\x30\xa7\xee\x4c\xe6\x14\xda\x6d\x21\x5b\x49\x53\x90\x5c\x8e\xc1\xd0\x3d\x62\x1c\x7d\x29\x6a\xe6\x92\xb8\x3d\x13\x40\x03\x95\x2e\x8c\xa0\x27\x3f\x1f\x3d\x05\x93\x71\x1b\x15\x62\x66\x24\xfb\x35\x26\xe0\x8d\xf9\x33\x70\x1b\xad\x3e\xf3\x9e\x81\x9d\x73\x20\x32\x67\xf6\x1c\x76\x57\xce\xa2\x13\xa0\x09\x02\xe1\x02\x4c\xc2\xa2\x0b\x84\xbd\xe7\x21\x90\x9f\xfa\x2b\xe6\x82\x0e\x19\x74\x0a\xad\x2b\x82\xcb\xb4\xf4\x8d\xf1\xdb\x8c\x97\xcc\xba\xd7\xb8\x9e\x71\x60\x76\xbf\x79\xce\x40\x92\xd3\x84\x20\xc2\xcc\x2b\xfc\x3d\x8d\x9f\xc6\x24\xfb\x1a\x05\xff\x54\x0a\x90\x0f\x7d\xf9\xc8\x91\x8b\x1b\xb9\xa8\x51\x24\x3c\xd7\xc6\x62\x6a\x58\xb9\xce\x79\x12\x74\xea\x22\x74\xe0\x06\xf4\x04\x35\xd2\xb9\xf9\xf2\x98\x64\xe4\x91\x8f\x27\x5b\x38\x9e\xd6\x2e\x58\xd7\x6d\xc5\x53\x84\x99\x07\xd4\xae\xb0\x85\xee\xf0\xb7\x83\x38\xa0\x36\x57\x98\x9d\x82\xc3\x7e\xad\x39\x91\x6f\x3e\xcb\x78\xcb\xaf\xf3\x0f\x76\x06\xd3\xea\x54\x62\x3c\xab\x4e\x65\x2c\x33\xa9\x66\x3e\x68\x5d\xe5\x9e\x1b\x5d\x71\xbe\x9d\xeb\x1a\xf4\x74\x7b\xbb\xd3\x6d\x1e\x0c\xc9\xfe\x7f\xe8\xc0\x9f\x5c\x9b\x26\xe2\xb3\x17\xd8\x57\x9d\xcc\x86\x47\x8e\x53\xd6\x90\x87\xcc\xad\x99\xc0\x7c\x15\xb4\xf1\xd6\xb3\x81\xdf\xe6\x05\xb0\xb9\x60\xee\x31\x5b\x14\xb7\x10\xdb\xc8\xda\x88\x24\x96\xec\xf0\x07\xd9\x92\x59\x9d\xbd\xa8\xc8\xe4\xfe\xef\x74\xc0\x81\xcb\xa8\xaf\x1f\x64\x01\x9f\x62\x4e\x42\x5a\xb2\xde\xe4\xbe\x95\x76\x1f\x07\x32\xb9\x8f\x8d\x89\x16\x2d\xea\x97\x33\x39\xd6\xcb\xfa\x6a\xba\x5e\xd6\x3f\x07\x9d\xb3\x02\x76\x32\x27\x58\xaf\x44\x76\xe2\x23\xc6\x2f\x8e\x2b\x26\xf9\x59\xfe\x2c\xfb\x50\x29\x0d\x80\x1c\x7c\xf7\xfb\xbf\xeb\xd4\x66\x83\x47\x01\x56\x65\x36\x72\xe9\xe1\x9b\x87\xbc\xe2\xd5\x87\x8d\x93\x9c\xcb\x29\x88\x62\x02\xe4\x52\x84\x6c\xd0\xa2\xab\x11\x1b\x66\x3c\xeb\x48\x68\x35\xf2\xb3\xb0\x14\x2a\x4a\x85\x06\x13\x20\x9b\xae\x69\x90\xa6\x41\x9a\x06\x55\x98\x06\x65\xd6\xfd\x0b\x79\xd0\x9c\x50\x46\xcd\x83\xb6\xc1\x83\x34\xff\x99\xca\xbe\x5c\xfe\xf3\x40\xd9\xdd\xba\x31\x90\x6b\x01\xbf\x84\xfb\xa0\x15\xbb\x10\xfd\x98\xc5\xfb\xff\xd0\x01\x90\xd0\xab\x21\x5f\x43\xbe\x86\xfc\xea\x42\x7e\x7a\xe5\xa3\xfb\xba\x00\xf1\xe7\x04\x34\x2e\x80\xf8\xc5\xe1\x8e\x8b\x43\xbe\x06\xf6\x03\x05\x76\x07\x8e\x93\x98\xfe\x00\xb1\xc0\x64\x38\x33\x40\xf1\x69\x9b\x93\xf5\xe5\x17\x2a\x9e\x13\xb2\x7f\xf7\x1b\x0a\x38\xfe\xae\xf4\x5a\x63\xf4\x4a\x18\xbd\x71\xf8\xd3\x28\xad\xa6\xef\x0c\x4a\xa7\x02\xf3\x82\x11\x67\x43\x81\x88\x39\x01\x98\x03\x31\x42\xc0\xc2\xb7\xb7\x88\x21\x62\x22\x25\x88\x09\x70\xec\x27\xf8\xf9\xd4\xb6\x10\x17\x92\x4e\x63\x22\x53\xa4\x87\x01\x0f\x90\x03\x44\xe4\xdf\xd6\x09\xf8\x07\x03\x23\xfa\x00\x6c\x4a\x86\x41\x51\x0e\x42\xf7\x08\xb0\xe0\x40\x78\x8c\x00\x41\xc1\x00\x01\x34\x46\xa6\x27\x90\x75\xb2\x9d\xf5\x83\x42\x36\x31\x3b\xc8\x4f\xb3\x09\xcd\x26\x36\x1b\x01\xa2\xd2\x09\x39\x8a\xfa\xe1\xc0\xd4\x4c\x62\x35\x26\x11\x88\x0f\x6b\x36\xb1\x36\x9b\xd0\x4c\x42\x33\x89\xed\xcd\xf7\x5f\x77\x0a\x10\xfa\xf5\x6c\x84\xd6\x07\x18\x35\x9e\x2a\x78\x3a\x8a\xa2\xe7\xfb\xdf\x3c\x48\x04\xb6\xd1\x71\xe3\xa4\xd7\xad\x81\xbc\x05\xf9\x3e\xf3\x48\x88\x95\x0b\x9c\x91\x50\x17\xe6\x6d\xf4\xf2\x25\xd8\x34\xfc\xf6\xba\x3f\x81\xa8\x17\x80\xde\x82\xb0\xe5\xa0\xe2\x38\xfc\x77\x3a\x88\xd6\xe0\x7d\xe2\x1f\x9d\x32\xd4\x87\x0b\x35\x26\x6b\x4c\xae\x34\x26\x67\xa6\xcd\x85\xa0\x7c\xaa\x41\x59\x83\x72\xd5\x40\x39\xd8\xa7\xae\x81\xad\x80\xf3\xee\xed\x8c\x17\x81\xb2\xde\x26\x07\x1a\xa0\x35\x40\x87\x89\x95\x06\xe8\xf4\xa4\xf9\xb4\x55\x80\xcf\x67\xb3\xf1\x79\x3b\x61\x71\x7a\x55\x7b\x27\x00\x7f\x5b\xc1\x6f\xd1\x36\x51\xc5\x62\xdb\x77\x6b\x91\x3b\x3c\xbb\x2d\xcf\x21\x02\xd7\xe3\x23\x1d\xd6\x5e\x1e\x78\xeb\xfd\x73\x0d\xe0\x5b\x9e\x61\x17\x22\xf8\x9c\x2f\xe4\xee\xe4\xc6\x74\xfa\x09\x8d\xe1\xd5\xc5\xf0\x4c\x9c\x1b\x43\x82\x4d\xe6\x44\xb9\x69\xac\x4e\x5e\x21\x24\xe4\xc9\x7b\xa1\x97\xbd\x35\x30\xef\x23\x30\x6f\xe3\xb2\xa3\xb3\x82\xfb\x79\xda\xb3\x2f\x3b\x62\xc8\x45\x81\x56\x2c\xe4\xda\x74\xe2\xf8\xc8\x97\x1d\xa2\x97\xd4\xe2\xe0\xf8\xc7\xb8\xcc\xcb\x25\x6e\x45\x32\xa1\x39\x42\xd7\xd8\x41\xd4\xcb\x38\x01\x79\x6b\xdf\x05\x34\xef\x86\x2c\xbc\x6c\x2a\xe1\xa2\x65\xf6\x3f\xfd\xb1\x91\x11\x9d\x19\x11\x9d\x78\x58\x18\x3f\x7c\x68\x75\x7a\xdd\x77\xea\x20\x64\xc3\x01\x3c\x6e\xb5\x4f\x6b\xa0\xd9\xea\xd5\x40\xa7\x51\x03\x8d\x93\xb3\x9e\xea\x82\x8d\x1f\x5a\xbd\x9e\xd9\x79\x6d\x64\x2c\x62\xa1\xa0\xc1\xec\x8d\x5e\x8a\xad\x13\x4a\x94\xc2\x43\xe8\x49\x6c\x7c\x4c\x30\x87\xa8\x7f\xcd\x46\x23\x49\x1e\xa2\x8c\xc6\x02\x14\x21\x72\xd7\x1f\xfd\x91\x94\x01\x54\xb5\xc4\x27\xc8\xee\x10\x2b\xa4\x0d\x85\x26\xd9\x4e\x99\x64\xfa\xb3\x0a\x19\x8b\xcc\x7e\x05\xd9\x67\x50\xbe\x21\x44\xf0\x9d\x7b\xa1\x60\x3b\xe6\x25\xb9\xe7\xb3\xf2\xd8\x8a\x03\x5d\x17\x93\xe1\x75\x60\x8a\xcd\xbc\xf4\x19\x1e\x34\x74\xd4\x81\x13\x06\x82\x02\x81\xc6\x29\x07\x75\x1f\xe9\x68\xae\x9f\x8b\x2a\x63\x90\x0c\xe7\x54\xd6\x9a\xe1\x8c\x1c\x38\x7e\x0f\x05\xbc\x8c\xf8\x91\x62\x1c\x59\x6e\x66\x52\x42\x90\x29\x90\x72\x71\x8d\x5f\xe6\xda\x7f\x73\x6a\xc0\xe5\x13\x37\xdb\x1b\x62\xf2\x4f\xc4\x78\x18\xfb\xfa\xfa\xa4\x75\xd2\x31\x14\x96\xc6\xc5\x2d\x1e\x27\xd5\x10\x26\x7e\xa0\x24\xba\xed\xcd\xe8\x36\x7e\x52\xf2\x19\xca\x3e\x23\xd3\x0a\x1f\x91\x32\xfb\x04\xdd\x59\x68\x17\x70\x8f\x24\x1d\x95\x39\x22\xe8\xad\xf1\xdb\xab\xb7\xa9\x0c\x3a\x7d\x60\x86\xc0\xb9\x0b\xd9\x9d\x1d\x70\x41\xc5\xf2\xfd\x49\xcd\xf4\x52\x52\xe9\x4c\xda\xcd\x1a\x68\x36\xcf\x6a\xa0\x79\xd6\xf3\x9d\x49\xf3\x2c\xe1\x4c\x6e\x3d\x3b\x8f\xab\xfb\x35\xab\xf5\x04\xd5\xb4\x1a\x35\xd0\xec\xb5\x13\x15\xcc\xbc\x31\x53\xf8\x33\x99\x77\xd4\xf6\x9c\xd4\xe5\xa1\x4b\x2d\x3a\xdd\x79\x03\xd4\x67\xc8\xb5\xb1\x29\x2f\xbf\x8c\x3f\x75\xd0\x0f\x3e\x35\xd1\x67\x08\x5a\x93\xa8\x08\x2f\xba\x0d\x33\xb7\x8a\x37\x7f\x1e\x29\x40\x51\xff\x1a\xde\x2a\x5d\xfa\x85\xd2\x2b\xf0\x58\xa3\x59\x6b\x19\x79\x54\xd6\x68\x37\xb8\x91\x4b\x59\xd3\x39\x11\x67\xf5\x08\xc1\x64\x08\x5c\x6a\xf1\x2c\x14\x72\x4c\x86\x36\xf2\x45\x19\xe7\xc9\x41\xaf\x5a\xfe\x99\x6a\xf9\x32\x77\xb6\xe5\x53\x9f\x0e\x1b\x6f\xf2\x8d\xbe\x91\xef\x61\xe6\x5a\xbd\x2c\xf8\x5b\xe8\xb2\xfc\xf9\xf0\x46\x10\xfc\x32\x72\x05\x39\x10\xbe\x04\xba\x87\x30\xbd\x2c\xba\x87\xa4\x40\xa3\xfb\x3a\xe8\xfe\xba\x2c\x74\x6f\xe5\xa1\x7b\xc2\xa2\x34\xbe\x97\x8e\xef\x1a\xbf\x0f\x06\xbf\x5d\x64\x96\x8b\xdb\xa0\x0e\x34\x67\x28\x83\x33\xfc\x41\xe0\x3d\xc4\xb6\x6f\x01\x9a\x37\xe8\x99\xff\xfe\x70\x83\xcc\x6e\xcd\xca\xe4\xa0\xad\xa7\xfe\x7a\xea\xaf\xa9\xc3\x86\xa8\x83\xdc\x94\x3a\x8e\xfe\x25\x02\x62\x82\x58\xdf\x41\x0e\x65\x93\xbe\xc7\xe1\x10\xf5\x07\x13\x81\x0a\xa1\x3b\xf8\xf8\x45\x02\xa8\xf3\xbe\x1f\x15\x44\xa5\x10\x6a\xa1\xf2\xbf\x22\xf5\xac\x00\x2e\x57\xc1\x2d\xcc\x05\xc3\x03\x4f\x20\x0b\x50\x02\x46\x94\x0b\x8d\xe4\x65\x22\xf9\x8a\xb3\x7c\xab\xd3\x81\x6d\xa8\x91\x7c\x3d\x24\x4f\x7f\x1a\x79\x65\x24\xef\xe8\x69\xfe\xf6\xb1\x5c\x63\xf5\xde\x60\xb5\x8c\x2b\xf1\x9c\x63\x0b\xd9\x02\x06\x93\x6f\x97\x5a\xfd\x18\xb6\xa7\x93\x6e\x2e\x20\x5b\xec\x33\xd3\x73\x91\xfb\x6b\xd7\xb9\x79\x19\x46\x95\xba\xd4\xda\xf8\x37\x20\x0b\x02\x53\x16\xbf\x16\xb9\x3c\x6c\xff\x04\xc7\x72\x52\x0e\x22\x81\x82\x63\x1b\x72\x01\xba\xc0\xc1\xc4\x13\x88\xe7\xec\x78\xef\x1b\xc8\xef\x4e\x00\xe1\x69\x0a\xc7\xe6\x87\x47\x14\x9c\xd0\x6b\xcf\xb9\xd8\x66\xc9\x13\x7a\x1b\xbf\xfc\xb6\x42\xc7\xfa\x1c\x38\xbe\x44\xec\xb3\x6c\x4f\xbb\x18\xd4\x36\xf1\x01\x80\x6e\x32\x7d\x99\xf8\xc1\x69\xc0\x4b\x02\x4e\x83\xd4\xf7\x98\x21\x33\xbc\xfe\x29\x91\x5d\xb9\x93\x82\x5b\x5f\x04\xde\x74\xc4\xa2\xb4\xe0\x7a\x62\x55\x54\x29\xf7\x11\x93\xbb\xbc\xb8\x2e\x65\x36\x98\x48\xf7\x65\x2c\x75\x31\x97\xe6\xed\xdd\xae\x78\x5a\xb0\x72\xc9\x39\x47\xb0\xb9\x60\xbb\xa6\x90\xf4\xd6\xc3\x12\x8a\xf1\xc8\x5c\xd5\xbc\x5b\x5c\x35\x7b\xb0\xb4\x93\x11\x10\x47\x56\x3d\xb9\xbc\x92\x14\xcf\xfb\x8a\xc4\x2f\x5f\x52\x0b\x48\xc9\x83\x63\xe9\xc6\x6a\x40\x6a\xb6\x06\x3c\xe2\xff\xff\x12\x40\x62\x05\xdc\x52\xf6\x26\x5e\x41\xf2\x91\x26\xae\x4e\x9f\x45\x5a\xe5\x33\xe2\x8d\x02\x2b\xdc\x78\x90\x71\xf2\xc5\x3b\x16\x09\xad\xf2\x41\x7d\x46\x29\x77\x8a\xf1\x7a\xd9\x29\x46\x2f\xbb\x54\x16\x4c\x31\xca\xbd\x99\x4b\x4f\x31\xaa\x7a\x34\x69\x7f\xa6\x16\xcf\xb0\xe8\xb4\xb5\x35\x27\x79\x36\xca\xa5\xd6\x2e\x9c\x8c\xba\x5c\x74\x41\x4a\x1f\x97\xd2\xc7\xa5\x4a\x27\x09\xfb\x7c\x5c\xaa\x77\x5a\x00\xd6\xf1\x8d\x5d\x6b\x1c\x97\x7a\x77\xf9\x07\xf8\xc3\x9f\x74\xad\x7c\x66\x6a\x77\x78\xd2\xd2\x4b\xb1\xbd\x82\x6f\x94\xb5\xe7\x5c\xc6\x72\x18\x97\xa5\xed\xf7\xaa\xaa\xa1\x60\x46\x25\x99\x0f\xa1\x16\xea\x4f\x39\x4c\x92\xfd\x9c\xc7\x3c\xc8\x74\xbd\x70\x55\x25\xba\x7d\x4d\x1a\xcb\xf9\xe3\x23\x38\xb9\xf2\x9c\xcf\x50\x20\xf0\xf4\xa4\x10\xa3\x3f\x57\x5f\x6d\x29\x87\x14\x2d\x70\x60\x3c\x9f\x14\x95\xb6\x02\xfb\x9c\x44\x6a\xea\x90\xf5\x7a\x4b\xc5\x39\x93\x5e\x56\x51\xd2\x2b\x45\x99\x9a\x8d\x6e\x01\x70\xf7\xca\xe2\x4c\xbf\x7b\x54\xc0\x2d\x71\x26\x53\x46\x67\xa4\xda\xba\x05\x22\x75\xab\xc4\x06\x34\x1b\x6a\x70\x40\x79\x14\xab\xd9\x28\xf8\x8e\x4b\xa7\xa1\x39\xd6\x46\x38\x16\x1c\xa2\x50\xa9\x09\x00\xaa\x2c\xf7\x32\x19\x95\x76\x99\xbc\x10\x67\x26\x23\x1b\xd1\x87\x5f\x11\xb4\x64\x13\x92\x8f\x05\xe0\xa8\xd8\x91\x49\xed\x94\x87\xb1\x10\x37\x0b\xb5\x59\x22\xd9\xe3\x62\x62\xcf\x42\x32\xe9\x23\x7c\x61\x5c\x27\x91\x34\x18\xcc\x28\x66\x44\x5f\xbe\x7c\xf9\x52\xff\xf4\xa9\xfe\xfe\x3d\xf8\xf5\xd7\x73\xc7\x39\xe7\x29\x86\xe5\x42\x21\x10\x23\xf9\x75\x45\xbe\x6a\x84\x2d\x0b\x91\xf9\xdb\x78\xd3\x66\x65\x89\x4a\x24\x50\xca\x42\xbb\xcc\x80\x4f\x1c\x45\x7b\xb3\x4e\x87\x94\x4d\x95\x14\x59\x0c\xb8\x5f\x6a\x88\x86\x19\xd7\x53\x1a\x65\xbc\x67\xd8\xb6\x81\x45\x1f\x88\x91\x29\xf6\x07\xcb\xf9\xc8\xae\x22\x42\x19\x26\x0b\x7e\x48\x87\x04\xe6\x33\xc4\x84\x88\x89\xe7\x0c\x10\x4b\x3e\xe7\x11\xac\x10\x81\xe5\xa4\xff\x19\x7d\xf3\x50\x66\xd3\xf1\x50\x14\x70\x51\x1d\x05\x80\x9f\x0e\x53\x05\xef\xca\x55\x41\x08\x41\xf2\xe7\x72\x8a\xf8\x88\x1d\x7c\xa8\xe3\xe0\xfd\xf3\x8f\x83\x40\xfc\x87\x3a\x0a\x7e\xa9\xc2\x28\xb8\xa4\x56\xf5\xa4\x9f\x64\xd0\x2b\x09\xff\x95\xf5\xea\xf1\x11\x9c\xfc\x16\xad\x48\x81\xa7\xa7\x4c\x42\xbd\xcd\x4d\x68\xa3\xfa\x9d\x37\x40\x8c\x20\x81\x78\xdd\xa4\x8e\xeb\x09\x54\x67\x28\x98\xe5\xf0\xba\x4b\xad\xff\xbd\x87\xac\x1e\x2f\x74\xc5\xcb\x5c\x7f\xf1\x33\x5c\x6a\xbd\xf9\xb1\xdf\x37\x51\xfa\x18\x80\xa2\x71\x37\x2d\xe5\x6d\x8f\xb6\x0a\xe9\x58\x11\xcb\xab\x93\x9f\x5f\x2d\x2f\x17\x2e\x18\x26\xc3\xc5\xe4\x12\xfe\xa5\x2c\xa4\xed\xf3\x52\xa9\xbc\x03\x36\x91\x89\x09\x17\x30\xe7\xa6\xd7\xe5\xd7\x4f\x0b\xf7\x91\x13\xe9\x6b\x84\xa5\x26\xb7\xe1\xa3\x01\xd8\x67\x21\x57\x92\x52\x36\x29\x4b\x04\xf8\xed\x91\x20\x2f\xca\x10\x64\x45\x0d\x16\xbc\x02\x5a\xcd\x51\xc0\xeb\x86\xc7\x8b\x2d\x39\xd5\xbe\x8b\xf1\xfd\x61\x8f\x96\xc3\x50\xf2\x2f\xdb\xd9\x70\x9b\xbf\xaf\x26\x17\xed\x9f\x25\x16\x89\x41\xc2\x7d\x25\x64\x55\x30\xa5\x43\xa9\x64\xbd\xe7\x76\x28\x7b\x6e\x2f\xc0\x73\x6d\x93\x35\xdb\x19\x55\x07\x7b\x2f\xf1\x9e\xc4\x1a\xdb\x64\x9f\xe4\x91\x0e\x1d\x5d\x54\x2c\xfe\x82\x93\x9e\x9d\x38\xb2\xf7\x80\xb7\xbe\xaa\x1e\x48\x54\xc9\xa0\xa0\x05\x0f\x54\xad\x42\x2e\x6a\x60\x5a\xf9\xff\xbc\x39\xd2\x21\x3f\xe5\x85\xfc\xa8\x8e\x52\x47\xfd\xac\xc7\x40\xa4\xa5\x6b\x06\x72\x90\x51\x3f\xad\x82\x63\x4d\x9d\x76\x89\x74\x46\x07\xfe\x94\xc4\x7e\x5a\x05\xdf\xc9\xea\x74\x34\xfb\xd9\xdd\x10\x9f\xe7\x0a\xd7\xc9\xbd\x5c\x4c\x47\xeb\x4c\x9b\x95\xcb\x31\x22\x91\x3e\xfb\x46\xe1\x0e\x06\xec\x58\xc8\x0c\xb8\xc6\xd2\x3a\xd0\x61\x3b\x95\x52\xc3\xa1\x86\x2d\x54\x22\x78\x27\xd4\x85\x8e\xdf\x29\x4d\x0f\xab\x0f\x08\x1d\xc5\x53\xa2\x1a\x74\x14\x8f\x8e\xe2\xd1\x51\x3c\xa5\x46\xf1\x54\x6b\x6d\x73\xaf\x63\x74\x42\x01\x97\x21\xda\x8a\xca\xb2\x94\x30\x9d\x67\xb1\xc8\x65\x82\x70\xf6\x5f\x8f\xdb\x8a\xc3\xd9\x7f\x49\x96\x12\x8a\x53\xd5\x11\x71\x30\x5a\xac\x44\xac\x8d\xba\x54\xae\xc3\x6d\x74\xb8\x4d\xa5\x36\xbb\x5e\x80\x67\xdb\x9f\xea\x65\x54\x1d\xec\x78\x28\xeb\xed\xab\xef\x4f\xfd\x86\xc4\x03\x65\x77\x3a\xde\xa6\x50\xfe\xed\x82\x0f\xcf\x77\xca\xbd\xf6\x70\x9f\x76\x9c\xca\x8a\xb3\x59\xe3\xae\xc2\x4a\xc6\xd9\x60\x06\x05\x52\xd8\x0e\x09\x06\x5f\x9f\x21\x13\xe1\xfb\x90\xf0\x64\xee\x1c\x5c\x89\x67\x94\x7e\xe9\xe0\xca\xc1\x36\x95\xbb\x74\xf0\x73\x20\x6d\x70\x01\x89\x15\xd8\xf8\x5a\x64\xe3\x50\xa3\x67\x94\xc5\xa1\x0c\x96\xc6\x06\x76\xe1\x6e\x34\xae\x66\xc7\x6f\x20\xd4\xd7\x14\x6f\x02\xb0\x0b\xae\x3e\xec\x9c\x6a\xc0\xd6\x80\x5d\x0a\x60\xcb\xa9\xa2\x83\x85\x46\xec\xad\x20\xf6\x75\x28\x6e\x0d\xd9\x65\xad\x01\x68\x58\xae\x24\x2c\x6f\x65\x41\xa1\x53\x70\xcd\x9d\xfa\xe9\xc6\x35\x56\x14\xde\x9a\xc2\x67\xd7\x9f\x91\x49\x99\xb5\xad\x90\xd7\x19\x5f\x6f\x7d\x4e\xf2\x72\xb6\xbc\x72\x8a\x2e\xb6\xd3\x97\x07\x6f\x80\xbc\xcc\xfd\x36\xf8\x3e\x93\x1b\xe9\x4b\xe5\x37\xa5\xa0\x89\x8e\x19\xc4\x36\xef\x87\x5f\x68\xc5\x94\xf4\x5d\x4a\x6d\xe5\xf7\xda\x3b\x1f\xb5\xa3\xb8\xb2\xa3\x1a\x38\xfa\xb1\xe9\xff\xcb\x05\x14\xc8\xff\xe3\xf8\xe4\xe7\x97\x47\x21\xff\x89\x0b\xae\x4e\x83\x64\x14\xba\x63\xd4\xc0\x4c\x8e\xb4\xc0\xa7\x14\xe2\xc6\xec\x02\x55\xfa\xec\xab\x11\x5c\x52\x0b\xbc\x8b\x55\xf7\x2c\xbb\x29\x3b\xcf\x96\x16\xf8\x08\x93\x4d\x4d\x98\xde\xac\x8b\x28\x4a\x6a\xaf\xed\x10\x79\xd4\x73\x7d\x60\x61\x77\x56\x37\xd2\x04\xa1\xd9\x9a\x47\x10\xba\x05\xc7\x7f\xbb\xcf\x7e\xfc\x37\x9d\xbc\x17\x0c\x61\x9f\x19\x00\x28\x00\x7d\x8e\xbf\xa3\xb5\xe3\x1c\x56\x00\xee\x28\xc8\x21\x1b\xb8\x9b\x06\xf5\x35\x30\x7d\x27\xf0\x3b\xc6\x6e\x70\x49\xa9\x0d\xe4\xb9\x2c\x0d\xe2\xeb\x81\x78\xf3\x99\x40\x7c\x31\x74\xd4\x28\xbe\xc7\x28\x9e\x48\x98\x0d\xe3\x4d\x0d\xe3\x1a\xc6\x4b\x81\xf1\x07\x88\x05\x26\xc3\xe7\x40\xf2\xc3\x46\xeb\x7f\x05\x82\xd7\x80\xad\x67\xdd\x7b\x84\xd7\xdd\x24\x58\xbf\x08\xeb\xf4\x07\xa5\x3f\xde\xe4\x60\x6f\x04\x9e\xc0\xe0\xe6\x08\x39\x30\x5e\x58\x6d\x9e\x05\xc9\x62\x12\x8c\x1b\x0b\xb2\xbb\xa0\xa4\x80\xc3\xd8\x18\x8c\xe0\x30\x57\x28\x34\x83\x4f\xb8\x40\x8e\x31\x7d\x93\x40\x8e\x6b\x43\x39\xb4\xa2\xe6\x1b\x36\xe6\x42\x31\x26\x15\x98\xb1\xb4\x63\x75\x1c\x61\x62\xda\x9e\x85\xde\xda\x79\x78\x96\xaf\x1c\xc3\xf1\x6c\x81\x73\x8a\x87\xc3\xc1\xc8\x61\x0e\x09\x80\x52\x0f\x40\x19\xdf\x3c\xc4\x26\xf2\x30\x19\xa3\x0e\x12\x23\xe4\xa9\xae\x53\x11\x65\x33\x91\x3a\x44\xe3\x94\x69\x1b\xfc\x0e\xbb\x7f\x30\xfb\x6a\x42\xcc\x3c\xcc\x0e\xdd\x80\xd2\xb8\xf4\x38\x4c\xa8\xdf\x96\xc7\x19\xb3\x9d\x8f\x89\x45\xc2\xc8\x23\xa5\xa5\xce\x76\xa1\x71\xb8\x6a\x9a\x3c\xa2\x97\x1c\x03\xf7\xe1\x9b\xb2\xc5\xf2\xc6\xd6\xca\x5a\x34\xa6\x40\x67\x2c\xa1\xcc\xdc\x87\x14\x5d\x2a\x1d\x51\x05\x22\x89\x81\x8d\x4c\x91\xe3\xcf\x17\x17\xcd\x62\xc2\x89\x87\xb4\x32\xa8\x55\xd3\x9a\xf1\x8e\x05\xad\xc6\xf4\xb8\xa0\xce\x82\x16\x93\xb0\xca\xf9\xac\x5a\x7a\xf8\x5b\x4c\x70\xf4\xd1\x9e\x60\x17\x22\x00\x8c\xf8\x30\x06\x26\xb7\x74\xf6\x67\x97\x03\xe7\x50\x3f\xf9\xf9\xe8\xa9\x06\x52\x01\x13\x73\xad\x26\x05\x57\x53\xa3\xc9\xd9\xd8\x5c\xc4\x05\xe4\x3e\x35\xcf\x05\x6c\xa4\xe3\x73\x3c\xc8\xab\x63\x9f\xd8\x85\xfc\xee\xe6\xaf\x2f\xeb\x5f\x7d\x92\x97\x38\xc0\x39\xc7\x46\x42\x76\xa2\xd6\x2e\xe0\x50\x1a\x03\xff\x3d\xea\x9a\x91\xcc\xcd\xc8\xc0\x4f\xcb\x2f\x1c\x5a\x60\x20\x24\x25\xc3\xe3\xe8\x3a\xa8\x28\x31\xdb\x90\xff\xfb\x14\xe3\x29\x00\x08\x2c\x15\x12\x42\xc3\x6d\xc0\xf5\x0c\x42\x1f\xea\xcd\x88\x0c\x19\x82\x86\x69\x46\xe2\x31\x17\x9b\x77\x92\xf0\x87\x0f\x87\x82\xec\x47\x7c\x55\x1d\xfe\x46\x57\xd9\x51\x89\x30\x4f\xfe\x68\xab\x3f\x14\xea\x6b\x74\x95\xbf\x9b\xea\x8f\x76\x43\xcd\x51\x28\x5b\x4b\xf9\xbb\x69\x05\x83\xf1\x26\xea\x83\xcf\xbc\xb3\x6e\xa9\xf8\x2d\x6a\xc5\xaf\xd5\x8a\xd5\xb7\xb4\x3a\xea\x8f\xf8\xb0\xb3\x71\x6a\xa9\xed\x8d\xda\x92\x10\xdf\x77\x4a\x62\x67\x10\xf3\xe3\x00\xcf\xc1\x2b\x70\x15\x40\xb9\xcc\xbd\x8f\x69\xc1\x8b\xa7\x17\xff\x0d\x00\x00\xff\xff\x64\xce\x85\x85\x85\x2f\x01\x00") func monitoringSystemGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( @@ -204,7 +204,7 @@ func monitoringSystemGrafanaDashboard1JsonTpl() (*asset, error) { return a, nil } -var _monitoringZyncGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\xeb\x72\xdb\xb8\x92\xfe\x9f\xa7\xc0\x72\xe6\x1c\x3b\x59\xc9\x96\x64\xcb\xb7\xaa\xd4\x56\x9c\x4c\x76\xce\x56\x92\xf5\x38\xce\x54\x65\x52\x29\x1d\x88\x6c\x49\x58\x93\x00\x03\x80\xb6\x14\x97\xe7\xd9\xb7\x08\xde\xc0\x9b\x24\xdb\xb4\x2d\x5b\xf8\x31\x13\x0b\x20\x41\xb0\xbb\xd1\xdf\x87\x46\x83\xb8\x7a\x81\x90\x85\x29\x65\x12\x4b\xc2\xa8\xb0\x8e\x50\x58\x84\x90\xe5\x12\x21\xad\x23\xf4\x4d\xfd\x42\x71\xa9\xaa\x19\x06\xc4\x95\xff\xa2\xd6\x11\xea\xb6\xb2\x52\x07\x4b\x2c\x58\xc0\x6d\xb0\x8e\x90\xd5\x6e\xa3\xff\xe6\x78\x84\x29\x46\xed\xb6\xa5\x5d\x06\x14\x0f\xdd\xf0\x12\xc9\x03\xd0\xca\x27\xc4\xa9\x28\x25\x36\xa3\x6f\x99\xcb\x78\xd8\x26\x1f\x0f\xf1\x66\xa7\x85\x7a\xdd\x6e\x0b\xf5\xfa\xfd\x16\xea\xbe\xd4\x9b\xa6\xd8\x53\xcf\x7e\x93\xbd\x0e\xfa\x27\x7a\xe3\x02\x97\x42\xbf\x4e\xce\x7c\x75\x9d\x83\xc5\x64\xc8\x30\x77\xac\xb8\xee\x5a\xfd\xfb\xfd\x05\x42\xd7\xe1\xe5\x16\x38\x44\x16\x7a\x6b\x8d\x29\xc8\x7f\x39\xd6\x11\xa2\x81\xeb\x46\x25\x1c\xfb\x93\x33\xc6\x5c\x49\xfc\x44\x26\x16\x91\xc0\x55\x17\xd2\x12\x97\xd0\xf3\x50\xbc\xdf\xbe\xab\x9f\x3e\xa6\xe0\x8a\x54\xc0\x89\x78\x2d\x9b\xb9\x2e\xf6\x05\x84\x8f\x18\x61\x57\xa4\xd2\xb0\xc6\x9c\x38\x27\x2c\xd3\x50\x24\xb6\x82\x16\x2e\xad\x23\xd4\xdb\xd5\x0a\xa6\xd6\x11\xea\x68\xbf\x67\xe1\xef\xe4\x7d\xd3\xb6\x49\xf8\xb8\x83\x7e\xfa\x3b\xeb\xdd\xf7\xb4\x4c\x12\xa9\x44\x61\xfd\x35\xa3\xb6\x95\x15\xc7\xd2\xe4\xec\x32\x92\x63\xdc\x6a\xfa\x46\xd8\x25\x58\x28\x25\xaa\xbe\x67\x0f\x1d\x62\x55\x92\x7f\xcb\x50\x29\x1f\x80\x8e\xa5\x7a\xb3\x4e\xae\x1c\xaa\x2e\xd7\xad\xee\x57\xed\x67\x7a\xc9\x88\xb8\xae\x2e\xa5\x7a\x41\x1e\x14\x04\xd9\xed\x2d\x10\x64\xb7\x5a\x90\xfb\xfb\xe9\x6f\x17\xc6\x40\x9d\xfc\xa3\xf0\xc5\xb8\xf8\x1e\xa1\xe2\x03\xce\x81\xca\x8a\x1a\x0f\x4f\xab\x4a\x09\xad\x28\x15\x13\x76\x59\x1e\x46\x92\x49\xec\x56\x5c\x7d\x81\xdd\x20\x13\x6a\xe9\x65\x5c\x42\x55\xad\xde\x9a\x2a\xbc\x24\x8e\xcc\x59\x5e\xc1\xba\x55\x51\x38\x40\x4e\x18\xa1\xf2\x23\x53\x43\x5b\x15\x64\x6a\x61\x7e\xea\x70\xb2\x27\xfa\xc0\x6d\xa0\x12\x8f\xa1\xa4\x69\x3f\x6c\x8a\x63\x87\x04\xe1\x3d\xbd\x7c\x79\xd9\x30\x38\x50\x07\x38\x28\xc7\x31\x72\x99\xcc\x1e\x2c\x80\x13\x10\xff\x7b\x01\x9c\x13\x07\x0a\x9d\x16\x3e\xb6\xa1\xca\xfe\x84\xc4\xf6\x79\x51\x16\x42\x82\xef\x83\xf3\x81\xd0\x72\x7f\x25\xe6\x63\x90\x42\x73\xa1\xba\x13\x0d\xbd\xcb\xd4\x57\xbd\x13\x81\xb7\xc9\xb1\x84\x4d\x8e\x89\x2b\x06\x1c\x7e\x04\x20\xa4\x18\x28\xa5\x5d\x85\x4e\x4d\x75\xea\xf5\xc6\xaf\xe9\xdf\x1b\x2d\x9f\x39\xaf\xff\xde\xf8\x39\xa3\x76\xfb\x1b\x6e\xff\xec\xb4\x0f\xbf\xff\x67\xf6\xd7\xc6\xf5\xb7\xae\xf7\xfd\xe5\x4b\x34\x9c\xa1\x4d\x21\xb1\x0c\x84\xee\x29\xc3\x41\xc1\xb8\x87\x43\x63\xb3\x24\xf1\x60\x10\xc9\x24\x7f\x09\xa1\x12\xf8\x85\xb2\x1b\xab\xeb\x55\xd7\xbd\xc7\xb6\x54\xce\xb9\x9b\xab\x8e\xac\xfe\x7d\xfa\x8c\xab\xab\x7f\x5f\x5d\x45\xfd\xb8\xbe\xfe\xf7\xf5\x75\xbe\x31\x0e\x23\xe5\x51\xad\x37\x56\x5a\x7c\x1d\xff\xa5\xb9\x9f\x09\x07\x31\x61\xae\x53\x72\x4b\x1e\xbc\xe7\xcc\xd3\x5c\x72\x5a\x7e\x0a\xe3\xd8\xc6\x0a\x37\x7c\x9e\x90\x91\x2c\xdf\xa1\x39\x38\x94\xe8\x01\xf9\xc0\x91\x00\x9b\x51\x07\x6d\x0e\x67\xa8\x28\x4e\x4b\xa6\xae\xff\x4a\x1f\x87\x98\x2b\x1f\x5e\x18\x89\x82\x71\x59\x70\x24\x6a\x10\x0e\x12\x37\x4a\xa8\x43\x2e\x88\x13\x60\xd7\x2a\x8d\xc7\xe4\x1a\x85\x38\x59\x07\xa6\x78\x4a\x0a\xde\x6c\x18\xd8\xe7\x91\xf1\xe9\x6f\x18\x7a\x8d\x78\x2c\x86\x42\xa8\xc0\xce\xc2\xd5\xd5\xde\x24\xf5\x1a\xdf\xbe\x97\xba\x38\xc3\x53\x98\x63\xf3\x0e\xd8\xc4\xc3\x0a\x59\xba\x35\xf6\xc8\xe1\x87\x5f\xb0\x44\x17\x0f\x41\x99\x61\xa1\x98\x8d\x8f\xb1\x80\x52\x5b\x91\xbf\xcc\xbf\x4a\xea\x30\x4b\xc5\xda\x3b\x66\xd6\xd7\xaa\xee\x7e\xd6\x4b\x31\x09\x15\x59\xd9\xcb\xd2\x13\xee\xb1\x9f\xa5\x51\x32\x2b\xdb\x02\x76\xc9\xb8\x0a\x2a\x54\xf9\x07\xb8\x48\x3b\x9d\x63\x41\xcf\x18\xc5\x73\x05\xf3\x60\xfc\xc0\xc0\xb8\x81\xf1\x55\x81\x71\x9b\x51\xc9\x99\xeb\x02\x7f\x7c\x28\xcf\xfa\xf2\xe4\xe1\xbc\x4a\xac\x06\xd2\x0d\xa4\x23\x03\xe9\x4f\x08\xd2\x8b\x13\xf3\xc3\x1a\x44\x3f\x34\x88\x6e\x10\xfd\x31\x11\xbd\x15\x4d\x1f\x5f\xff\xbd\xd1\xfb\x16\x96\xbc\x32\x18\xdf\x1c\xc6\xf7\xa6\x53\x83\xf3\xe9\x5b\x18\x9c\x37\x38\x8f\x9e\x36\xce\xef\x15\x60\xbe\x34\x73\xaf\xc1\xf9\x83\x8e\xc1\xf9\x06\x70\x3e\xf7\xd8\x15\x81\xf9\xfc\x53\x9e\x00\xce\xef\x1a\x9c\x6f\x1a\xe7\x77\x0d\xce\x6b\x6f\xf1\xec\x70\xde\xea\x58\x06\xe6\xd7\x1a\xe6\x0f\x96\x84\xf9\xae\x81\xf9\xe7\x3a\x9d\x7f\x72\x38\xdf\x37\x38\xdf\x34\xce\xf7\x0d\xce\x6b\x6f\x61\x70\xbe\xb2\x93\x06\xe7\x9f\x0c\xce\x2f\xce\xa7\xdb\xaf\x01\xfa\x1d\x03\xf4\x06\xe8\xef\x08\xf4\x03\x27\x88\x52\x72\x07\x11\x8c\x88\x81\x08\xbc\x3b\x2e\xcc\x6f\xa3\x65\x9f\x65\xb3\x80\xca\xbb\x3d\xed\xd6\x14\x62\x49\x9a\x70\xa6\x8c\x7a\x31\x2f\xa8\x71\xc9\x35\xd2\x77\x86\x03\x1e\xd0\xb8\x8f\xf7\x26\xf7\x8a\xa7\x3c\x01\x89\xbf\xb1\x25\xb9\x80\x53\xb0\x19\x77\x6a\x04\x7f\x7c\x6b\xc1\x5f\x10\xb8\x7c\x08\xd1\x57\x3e\xe7\x09\x08\xff\x4f\x02\x97\x35\x42\x7f\xbb\x22\x2c\xf8\x34\x14\x30\xc2\x17\xc0\xf1\x18\x10\x07\xe1\x33\x2a\x00\xe5\x18\xa3\x21\xbd\xb7\x23\xbd\x25\x5a\xa7\x51\x4a\xb3\x86\xb5\x6a\xa4\xd7\xc6\xdc\x29\x34\x1c\x16\x9d\x60\xc7\x21\x74\x5c\xb6\x9c\xb0\xf2\x94\x05\xd4\x29\x34\x9e\xf6\xd4\x8e\x77\x15\x15\x1a\x4c\x37\x1b\xfd\xd2\xdf\x3f\xdc\x7d\xdf\xd3\x6d\x54\xdd\xf2\xd9\xc6\xd1\xd8\x14\x3f\x72\x1a\x48\x6a\x27\xe0\xc5\xe3\x48\x02\xf7\x99\x8b\x25\x1c\x2b\x73\xd5\x2e\x85\xa9\xcf\x68\x44\x4d\x3b\x5b\xfd\x8a\xd1\xc1\x7c\x6c\x13\x39\x2b\x8f\xc0\x90\x8f\x67\x1e\x4c\x8a\x64\x9c\xdd\x84\xbf\x3b\x20\x6c\x4e\xfc\x78\x73\x52\x66\xd4\x8d\x66\xd4\x96\x89\xfc\x04\xb0\xf4\xb0\x9f\xa7\xa7\x13\xe2\xc0\x5f\xc0\xd9\x71\xea\x2f\xf2\x94\x6f\x42\xc6\x13\x97\x8c\x27\xf2\x6d\xac\xff\x1c\x73\x8e\xe6\x06\xf3\x37\xdb\xc4\x76\x5b\xcb\xc7\x8b\x24\xbb\x92\x45\x73\xb8\x00\x2e\xe0\x6b\x5d\x37\x1b\x67\xa6\x91\x5e\x97\xc0\xcf\xad\x7c\xc0\xc9\x85\x5a\xd8\x4c\xe4\x5f\x09\x99\xcb\x05\x99\x3a\x73\xf1\x54\x45\x99\x5c\xb8\x7d\x74\x69\x0e\x7e\x2e\x80\xc9\x8a\x18\x51\xfc\xba\x2a\x56\x94\x03\x4e\x14\xc9\x16\x11\x1a\x5f\xba\xc4\x5e\x8e\x2a\x14\x0a\x4b\x7f\x27\x42\xb2\x31\xc7\x5e\xad\x85\x25\x90\x59\x94\xbe\x35\x7d\x53\xf2\x94\x65\x17\x9b\xb5\x33\x8d\x4c\xef\x53\xe0\x0d\xd5\x14\x2c\x27\x88\xb8\xf2\x33\xf9\x59\xc4\x50\x6b\x56\x7e\x8c\x86\x81\x3a\x0d\xa8\x86\xbf\x6a\xf4\xa8\xc4\x8e\x4a\xe4\xa8\x13\x9e\xef\x12\x99\x1a\x56\xa5\x83\x9e\x45\x2f\x75\x1c\x3b\x71\x0b\x07\x92\x59\xc5\xda\x6a\x79\xcc\x4a\xf2\x30\x98\x72\x07\x4c\xa9\xc5\x85\xfd\x9b\xc6\x77\x7a\xfd\x87\x83\x85\xf9\x9b\x37\x9e\x39\x2c\x54\x2e\x55\x64\x21\xf3\xd7\x1b\x94\x49\x32\x22\x76\xb4\x4b\xda\x40\x88\xf5\x49\x97\x47\xe5\xa2\x83\x01\x14\x64\x00\xc5\x00\xca\x03\x4d\x52\x6e\x8f\x39\xa5\xb9\xc8\x43\x82\xce\xfc\xfd\x05\xeb\x0e\x3a\x12\x28\xa6\xd2\xc0\x0d\xb2\xce\x22\x49\x18\xa0\x31\x40\x63\x80\xe6\x89\x02\x4d\x71\x72\xb3\xd3\x7b\x30\x9c\x39\x9c\x9f\xf8\xb6\xee\x38\x13\xe5\x64\x6d\xbb\xe4\x02\x0c\xd6\x58\x1f\xc8\x05\x50\x10\x06\x6c\x0c\xd8\x18\xb0\x79\xaa\x60\x53\x9a\xd5\x3c\x24\xda\xcc\xdf\x4d\x65\xd0\x46\xa1\x0d\x07\xec\xcc\x0c\xdc\x58\xa7\x80\x1d\x62\xf0\xc6\xe0\xcd\xad\xf1\xe6\x01\xbe\x80\xb9\x53\xb3\xa3\xa4\xaf\xa5\x41\x2e\xfa\x04\x26\xfa\x23\xd0\xb3\x70\x56\xfd\x33\x98\x85\x65\xff\x48\x45\x88\x8d\xd0\xff\xb1\xa1\x40\x40\x7f\x04\x10\x80\x83\x30\x75\x90\x72\x65\x48\x32\x34\x04\x04\x53\xb0\x03\x09\x0e\x7a\xf3\xf9\xcd\x09\xda\xa4\xa1\x9f\x46\x23\x4c\x5c\x70\x5a\x88\x32\x8e\xc6\x4c\x22\x98\xfa\x84\x83\xf3\x72\xab\x89\x2c\xe1\x85\xca\xdb\xad\xf9\x7e\xe9\xae\xf9\xba\x87\xc9\x12\x5e\x9e\x29\x78\x78\xba\xf9\x23\x80\x41\x68\xff\x03\x61\x4f\xc0\x09\x5c\x70\x96\xde\x0d\xf4\x23\x80\xad\x57\x1b\xad\x70\xdc\xbf\xde\x88\xc1\x3f\xc2\xfd\x90\x63\x72\x09\x4e\xd8\xf2\x7d\xef\xfb\x59\x82\x05\xe8\xdd\x79\x0a\x5b\x7f\x4e\x95\xf7\xf9\x9f\xd0\x2b\xa9\x2c\xd2\xbb\x65\x3a\xf6\x8a\xe3\x6e\xfd\x32\x1d\x3b\x35\x26\x48\x19\x85\xfb\xcc\x21\x34\xfb\x7b\xf4\xf2\x27\xb3\xbf\x67\x49\xa2\x50\xa0\x07\x82\x79\x31\x91\x27\x14\xc9\x09\xa0\x51\x20\x03\x0e\x2d\x34\x0c\x24\xa2\x2c\xfc\xef\xf2\x41\xe8\xc3\xc2\x38\xed\xee\x41\x35\x7d\x38\x34\x9b\x8c\x0c\x7d\x78\x1c\xfa\x90\xde\x6e\x28\x44\x23\x14\xe2\x73\x22\x4f\x43\x23\x0c\x8d\x30\x34\x62\x05\x69\x84\x9c\x60\x19\xf3\x00\x84\x25\x72\x01\x0b\x89\x18\xb5\xa1\x85\x1c\xe2\x28\xc2\xc0\x03\x8a\x58\x20\xc3\x5b\xb0\x94\xe0\xf9\x52\x84\xa4\x83\x83\xe4\x33\x15\xa6\x90\x13\xe0\x30\x62\x1c\x10\xe6\x80\x52\x1f\x8a\x46\x8c\xa7\x57\xcd\x22\x4e\x22\x18\xa3\xf7\xc3\x2e\x4a\x81\xf9\x5a\x7a\xb1\x6b\xe8\x85\xa1\x17\x8f\x42\x2f\xa2\x71\x66\xb8\x45\x23\xdc\xe2\x7d\xe4\xb4\x0c\xb1\x30\xc4\xc2\x10\x8b\x15\x25\x16\x59\x54\x22\xb0\x6d\x10\x62\x14\xb8\xee\xec\x56\xf8\x7f\xe3\x2c\xb0\xfe\x5e\x0d\xfc\xef\x19\xf8\x37\xf0\xff\x38\xf0\x4f\x28\x11\x13\x43\x00\x9a\x22\x00\xb1\x38\x0d\x05\x30\x14\xc0\x50\x80\x55\xa5\x00\x49\x6c\x81\x3a\x88\xe3\xfa\x40\x42\x4b\x8b\x22\x5c\x32\xba\x21\xd1\x10\x54\x0d\x09\x6f\x1e\x63\x72\xbb\xb0\xc1\xcd\xf3\xf9\x6a\x79\x43\xdf\xf0\x06\xc3\x1b\x1e\x85\x37\xc4\xcb\x72\x86\x36\x34\x42\x1b\x7e\x8b\xa4\x69\x58\x83\x61\x0d\x6b\xca\x1a\x1e\x22\xb1\x73\x6f\xa7\x1a\x46\xbb\xd9\xe2\x7e\x55\x62\x27\x07\x1f\x22\xad\x38\xe0\xbb\x6c\xe6\x01\x95\x6f\x19\x1d\x91\xb1\x55\x1e\xc9\x27\xcc\x11\x68\xf3\xd7\xe2\x95\x2f\x6f\x90\x0c\x6a\x63\x7b\x02\x67\xc4\x03\x16\x94\x3c\x86\xda\xcd\x70\x8c\xed\xf3\x31\x8f\x93\x6c\x73\x98\xa9\xaa\xff\x0c\xc7\x4f\x49\x8c\x76\xc2\xc8\xb2\xa1\x63\xfd\xf2\xbe\xb7\x7b\xd8\x7f\xab\x0f\x54\x75\x8e\x7e\x6f\x67\xbf\x85\xba\xbd\xc3\x16\xda\xed\xb4\x50\x67\xeb\xe0\x30\x77\x96\xfe\x2f\xbd\xc3\x43\x7b\x77\xcf\x2a\x59\xc7\x52\xec\xac\x3c\x42\xeb\x46\xa7\x35\xc6\x81\x02\xe8\xab\x1c\x47\x49\xde\xaf\xdb\xe9\xe4\x69\x4a\x52\xd1\x29\xbb\x98\xa2\x71\xa6\xbe\xfd\x43\x38\xaa\x4a\xa8\xae\x5f\xf1\x11\xf3\x73\xe0\xa2\x2e\xcb\xbb\xd6\x3c\x77\x0a\xe6\x59\xfc\x90\x7d\xc9\x3a\x77\x4b\x6d\x4f\x88\xa3\x0c\x21\xe1\x10\x95\x3b\x30\xf6\xb2\xb5\x23\x1d\x47\xad\x79\x94\xc9\xc3\xbe\x4f\xe8\xf8\x2c\x32\xc5\x6e\x55\xf9\x1c\x2f\x1b\x3b\xf3\xc8\x51\x87\x6c\x59\xc2\xb4\xe0\xac\x2e\x12\x1d\x2d\xf4\x79\x49\x63\x1c\xd3\xf1\x82\xc6\x7a\x73\x1c\x93\x87\xa7\xef\xb0\xc4\x27\x09\x49\xd3\x8c\xa3\x4c\x10\x6d\x46\x29\xd8\x12\xb2\xef\x6a\xaa\x6b\xce\xc2\x27\x17\x06\x5c\x35\x7b\x74\x83\x31\xa1\x7f\x02\x17\xf1\x24\x63\x6f\xab\xb7\xb5\x6b\x69\x54\x51\xc8\x11\x99\xe6\xd5\x10\x17\xbe\x67\x34\xc9\x72\xb7\xfa\x9d\x7f\x68\xf5\x1c\xca\xf7\xa8\xb2\xda\x5b\x94\xcc\x3e\x62\x7f\x8e\xae\x46\x11\x51\xc9\x73\x62\x55\x23\xa3\xb7\xb5\x3e\x6d\xbf\x29\x54\xb0\xf4\x86\x39\x02\x17\x3e\xe6\xe7\x6e\xc4\x47\x35\xcb\x0f\xa7\x41\xe9\x66\x2d\xe5\x4c\x76\xba\x2d\xd4\xed\x1e\xb4\x50\xf7\xe0\x30\x74\x26\xdd\x83\x9c\x33\x19\x05\x6e\xd5\x8c\x21\x6c\x59\x6f\x27\x6a\xa6\xd7\x69\xa1\xee\xe1\x4e\xae\x81\xb9\x5b\x8c\x24\x1e\xba\x61\x3b\x81\x57\xf8\x9e\xdd\x8d\xf6\x0c\x9d\x07\x43\x18\x70\xf0\xdd\xf8\x23\x30\xd9\x0e\x9f\x41\xb4\xc1\x67\xa0\x72\x7c\x93\x4b\x44\x1d\x99\xae\x6c\xe2\xf5\xdf\x1b\x25\xb8\x68\x7f\x8b\xbf\x76\xda\xf8\x87\x4e\x6f\x41\x7d\xad\x6e\xab\x67\x55\xb1\x5f\x6b\xa7\x23\xac\x4a\x96\x5b\xac\x49\xb2\x77\x03\x4a\x09\x1d\x23\x9f\x39\xa2\x0c\x88\x82\xd0\xb1\x0b\xa1\x40\xb3\x3a\x35\xf4\x75\xfb\x3f\xd0\xed\x5f\xd5\xce\xb7\x7f\x16\x12\x67\xeb\x75\xb5\xe9\x77\xaa\xfd\xcc\x42\xdb\x57\x17\x7e\x8a\x1d\x57\x38\xcb\xbe\x17\x1c\x3f\x49\x1c\x42\x05\x90\xdf\x00\xe3\x63\xb0\xbe\x29\xc6\xc7\xd4\xc0\x60\xfc\x5d\x30\x7e\xaf\x29\x8c\xef\x57\x61\x7c\xce\xa2\x0c\xca\x37\x8e\xf2\x06\xc5\xd7\x06\xc5\x7d\xb0\xef\x03\xbd\x51\x1b\x19\xfe\xd0\x1c\x7f\xf8\x42\xf1\x05\x26\x6e\x68\x0d\x86\x43\x98\x58\xc0\xf3\xe1\x09\xa5\x15\x9f\x5b\x13\x85\x3d\x13\x0c\x30\xc1\x00\x43\x23\xee\x89\x46\xa8\xa5\x99\xcd\xe4\xff\x54\x62\x42\x81\x0f\x3c\xf0\x18\x9f\x0d\x02\x81\xc7\x30\x18\xce\x24\xd4\x02\x78\xb4\x9a\x56\x01\xd7\x55\x27\x9d\x44\x2b\x6b\x94\x39\xd0\xfc\x79\x27\x8f\x0a\xe3\x2a\x46\xee\x10\x21\x39\x19\xaa\x4c\x3c\x46\xd1\x84\x09\x69\xf0\xbc\x49\x3c\xbf\xe5\xbc\xdf\xd9\xdd\xc5\x3b\xd8\xe0\xf9\xdd\xf0\xbc\x78\x48\xed\x6d\xf1\x7c\xa7\x12\xcf\xcd\xc4\xdf\x4c\xfc\x0d\x62\x2f\x9d\xf3\x12\x4e\xc1\x1d\x70\x25\x8e\x26\xe2\x3e\x73\x06\x19\x78\xa7\x13\x70\x21\x31\x5f\xee\xc0\xdf\x25\xf1\xfb\x5b\xdf\xfb\xfe\x32\xfe\xdc\x97\xcf\x9c\x7b\x3f\xb3\xac\x26\xc7\xe5\xf8\x11\x10\xfe\x23\x9e\xaa\x09\x3a\x4a\xc4\x8a\x36\x5d\x2c\x24\xea\x23\x8f\xd0\x40\x82\xa8\x58\x0f\x7f\x6e\x50\xff\xd8\x79\x90\xb7\x4f\x48\x5c\x9c\x48\x51\x73\x14\x6b\xb7\xd9\x33\xd7\x43\x84\xfc\xcd\xf3\xe5\xac\x9c\x23\x94\x7c\xb9\xb0\x5c\xf3\x44\x93\x18\x3d\x3c\x3d\x01\x7e\xaa\xfa\xb3\x53\x0f\x6d\x61\x01\xc2\x02\xfd\x0c\xdf\x7d\x01\x8a\x2d\x9b\xe2\xd8\xcf\x97\xdf\x24\xc5\x31\x4d\x8d\xc9\x81\x6a\x54\xfa\x8e\x70\xb0\xe3\x2c\xdc\x5c\xf5\xca\xe5\x45\x3e\x52\x70\xf8\xee\x78\xd0\x9b\x8b\x07\xca\x8e\xdb\xb9\x38\xa9\x76\xdd\x07\x42\xcf\xab\xf2\xc0\xb4\x99\x61\xae\x3c\x94\xb4\xd2\xc8\x42\xca\xf7\x4c\x57\xce\x8b\xe2\x55\xa1\xe8\x0a\xf1\xde\xf6\x74\x56\xb3\x3c\x71\x0f\x4a\x0a\xe8\x42\x35\xbd\x5d\x5e\x4d\xcf\x26\xfc\x53\x12\x93\x00\xa7\x9d\x0f\xc1\xe4\x85\xf4\x6e\x45\x12\xa5\x4f\x98\x13\xe5\x47\xa3\x4d\xe5\xde\x5a\x48\xe9\xb7\x85\x02\x1a\xfe\xfb\x52\xed\xe9\x50\xcc\x53\xbd\x4d\x16\x65\x0a\x71\x28\x6b\xce\xa4\x54\x37\x99\x52\x7d\xef\xc9\xca\xf9\x07\x3f\xb1\x8c\x6a\x9d\x2d\xae\xd5\x46\xac\xe5\x27\x20\x7b\x37\x9d\x80\xec\x97\xc3\x69\xd1\x86\x28\x33\xff\xb8\xff\xf9\x47\x53\xf3\x8e\x3b\x6c\xad\x7a\x3e\xf3\x8e\x47\x8b\x4b\x3d\x58\x58\x4a\x6d\xc8\xf2\x99\xf3\x14\xf6\x61\x9d\x2c\x1b\xb3\x32\xc7\xea\x2f\x4d\x20\x1e\x10\x82\xcd\xde\x2b\xbd\x7c\xa5\xf6\x5e\x1d\xd4\x7d\x97\x3d\x03\x81\xbb\x6e\xbd\x7a\x7b\xf2\x05\x7d\x09\x27\x61\x77\xdc\x7f\xf5\x74\x78\xd3\x8d\x03\xb7\x07\xdd\x6a\x2d\xec\xcf\x3f\xc5\x65\x4d\x36\x92\x3f\xef\x18\xac\xa5\xc1\xc7\x4a\x52\x21\xca\x1c\x18\xa4\xa4\x26\x4f\x87\x8e\x32\x62\x64\xfb\x41\x1c\x6b\x49\x8e\xeb\x51\xc6\x72\x74\x75\x85\xb6\x3e\x07\xde\x29\x96\x80\xae\xaf\x35\xa6\xf4\xf7\x5d\x63\x30\xcd\xb0\xa4\xf9\xc1\xda\x39\x2c\xa9\xb1\x78\xed\x63\x32\xab\xd4\x39\x9b\x28\xcc\x8a\x93\x28\x13\x6c\xd1\xca\x57\x8b\x43\xd5\x7c\x3d\xb6\xdf\x2c\x87\xfa\x23\x60\x12\x3f\x28\x87\xb2\x55\x86\x47\xa1\xdf\x0f\x40\xac\x46\x5a\x66\x41\xb7\xa3\xa7\x16\x34\x48\xb9\x6a\x4e\x93\x6a\x76\xa9\xdc\x30\xae\x74\x04\x8c\xa1\xea\x50\xb5\x95\x65\x62\x36\x67\xca\x2a\x73\x02\x9c\xcf\xcf\x26\xec\xf2\x77\xc0\x8e\xea\x42\xfe\xb6\x08\x24\x35\x3b\xb2\x99\x5b\xf0\x34\x0e\x08\xbb\x56\x9b\x0d\x52\x3f\x21\x67\xee\x3c\x44\x53\x1e\x22\x14\xc6\x59\x1e\x51\xa3\xa1\x0c\x19\x33\xfa\xfa\xf5\xeb\xd7\xf6\xc7\x8f\xed\x77\xef\xd0\xef\xbf\x1f\x79\xde\x91\x28\x30\x2d\x1f\x4b\x09\x9c\x56\xb7\x95\x1e\xe6\x47\x1c\x07\xe8\xe2\xa5\xbe\xb4\x5b\x65\xc2\x92\x08\x94\xf1\xd8\x2e\x4b\x20\x94\xe5\xe1\x7e\xbf\xcb\x0b\x69\x4b\x2e\x05\xd2\x18\x71\xc0\xc2\x10\x8d\x2b\xce\x52\x3a\x65\xbd\xe3\xc4\x75\x91\xc3\x2e\xa9\x55\xba\xec\x0b\x77\xcb\xc9\x62\x9a\x08\x55\xa2\x2d\xfa\xa5\x98\x54\x58\xcd\x14\x73\x22\xa6\xd1\xc1\x7e\xb9\xba\x80\x12\x8d\x10\xdc\x4c\xfa\xa7\xf1\x11\x91\xeb\xa9\x80\xe3\xd5\x51\x00\xfa\xc7\x7a\xaa\xe0\x6d\xb3\x2a\x88\x21\x48\xfd\xbc\x99\x22\x3e\x10\x8f\xac\xeb\x38\x78\xf7\xf8\xe3\x20\x12\xff\xba\x8e\x82\xdf\x56\x61\x14\x9c\x30\x67\xf5\xa4\x9f\x67\xd0\xb7\x12\xfe\xb6\xb3\x7d\x75\x85\xb6\x3e\x25\xf1\x29\x74\x7d\x5d\x2a\x68\xef\x08\x1b\xbb\xd0\x3e\x0f\x86\xc0\x29\x48\x10\x6d\x9b\x79\x7e\x20\xa1\xcd\x21\x9a\xe3\x88\xb6\xcf\x9c\xff\xba\xc0\xbc\x9d\x85\xbd\xb2\xa0\xd7\x3f\xc3\x0a\x9f\x39\xaf\x7f\x1d\x0c\x6c\x28\x6e\x24\xd0\x34\xee\x17\xa5\xfc\xd0\xa3\x6d\x85\x74\xac\x89\x65\x7b\xeb\xd5\xf6\xcd\xe5\x22\x24\x27\x74\xbc\x9c\x5c\xe2\xbf\xb4\x80\xda\xf3\x0f\x9c\xe2\xa1\x0b\xc5\x90\xa9\x90\x58\x4d\x6f\x4b\x23\xeb\x86\xd1\xd4\xda\x65\xe6\x5c\xf9\x1d\x52\x5a\xf3\x6b\xf5\xc9\x30\x4c\x4e\x94\x17\x4a\xd6\x36\xe3\xb9\x54\xc0\x67\x27\xce\xe3\x26\xc4\xb9\xd2\xc6\x8b\xb6\x91\x51\x79\x3e\x59\xf6\x9e\x47\x90\xab\xb8\xd6\x7a\x08\xf3\x9d\x19\x3f\xeb\xa5\xf0\xdf\x1e\x66\x89\x6e\xf1\x4a\x9c\x0a\xf1\x3f\x4a\x3a\x13\xc7\x54\x84\x4a\x28\xab\x20\x25\x4e\x85\x62\xb3\x4a\xb7\x2e\xab\x74\x2f\xd0\x23\x2d\xac\x1d\xd6\x9c\xaf\x90\x6d\xbe\xbf\xf3\xc2\xda\x47\xb5\x49\xc4\xe4\x27\x2d\x50\x44\xcd\xc6\xd2\x2c\x55\x78\x8d\x17\xcb\x56\x3d\x11\x69\x25\x93\x8a\x96\xdc\xa6\x75\x7b\xaa\xd1\x42\xe9\x23\xfe\xe3\xf5\x86\x49\x19\x6a\x2e\x65\x48\x77\x99\x26\x6b\xe8\x6e\x7c\x44\xd9\xbb\xe1\x23\xeb\x98\x35\xd4\xed\xd4\x6c\x96\xda\x6f\x9c\xdd\x98\xcc\xa1\x26\xc9\x50\xb7\xd3\xaf\x56\xdc\x8e\x61\x43\x4f\x37\x49\xe8\xb1\x12\x7e\x2a\x3f\x70\x66\xf2\x7d\xd2\x6e\x55\xb2\x8d\x44\xa4\x8f\xbe\xd4\xf8\x04\x53\x7e\x1c\xb0\x23\xd6\x71\x63\x1d\x98\xc4\x9f\x95\x52\xc3\xba\x26\x3e\xac\x44\xfa\x4f\xac\x0b\x93\x01\xd4\x98\x1e\x6e\x3f\x20\x4c\x1e\x50\x83\x6a\x30\x79\x40\x26\x0f\xc8\xe4\x01\x35\x9a\x07\xb4\x8a\xb1\xce\x67\x9d\xe5\x13\x8b\xb9\x39\x01\xaf\xa8\x44\x1b\x49\xf4\x79\x44\xeb\xbc\x49\x1a\xcf\xba\xe8\xf4\xa1\x32\x79\xd6\x45\x9e\x8d\x24\xf3\xac\xf6\x18\x59\x33\x8d\xae\x44\xb6\x8e\x1e\x59\x37\x09\x3b\x26\x61\x67\xa5\x16\xc8\x5e\xa0\xc7\x5a\xd3\xea\xf6\x4a\xaa\x8e\x76\x55\x37\xb8\x17\xfe\x13\xc8\x4b\xc6\xcf\x4d\xce\xce\x22\x5d\xd4\x1d\xab\x9f\x2d\x30\x9a\x75\xaa\x95\xfb\x84\xe2\x4a\x66\xeb\x10\x8e\x25\x68\x2c\x88\x46\x43\x70\xc0\xc1\x06\x72\x11\x13\xa1\xd2\xa7\x10\xef\xc0\x3c\x1a\xff\x16\xe2\xad\x53\x76\x56\xee\x5b\x88\xa7\x91\xcc\xd1\x31\xa6\x4e\x64\xe9\x77\xa2\x1f\xeb\x9a\x83\xa3\x05\x96\x4a\xe8\x9a\x19\xd8\xb1\x7f\xaf\xd9\x39\x4f\xfc\xc3\x88\xe6\x13\xca\xf7\x00\xdb\xbd\x9a\x2f\x32\x66\x67\x97\x19\xd8\x36\xb0\x7d\x27\xd8\x56\x93\x47\x8f\x48\x83\xdb\x0f\x88\xdb\x67\xb1\xd0\x0d\x70\x37\x15\x1b\x30\xe0\xbc\x1a\xe0\xfc\x22\x6e\x36\x1c\x73\xe1\x70\x8a\x92\x2f\x23\xdd\x58\xc2\x9e\x80\x87\xb3\x63\x98\x23\x18\x8b\xd2\xd2\xd4\x84\x1f\xf3\xf3\xe8\x4a\x89\xc7\x99\xda\xad\x68\x51\x35\x16\x9c\xf5\x73\x46\x6d\x2b\x7d\x8e\x04\xcf\x77\xb1\x24\x74\x9c\xf6\xdf\x72\x89\x90\x9a\xd1\xe8\x30\x1c\x1d\xe2\xa9\x03\x2d\xa1\xb6\x1b\x38\xf0\xa6\xfa\x8c\xc4\x4a\xf5\x58\x5e\xe0\x4a\x52\x71\x79\x72\x92\x66\x05\x4f\xc8\x01\x92\xbe\x0c\x69\xfd\x08\x80\xcf\xd4\x92\x2e\x67\x1e\xc8\x09\x04\xba\x29\x6b\x82\xec\xe6\x4a\xc7\x30\x2d\x04\x46\x2d\x71\x4e\xfc\x2f\xdc\xfd\x1c\x0a\xa8\xdc\xb9\x64\xb8\x6b\x9d\x2b\x8e\xb7\x9c\xfe\xdd\xe4\x3c\xd7\xc2\xcb\x67\x3c\x22\x67\xe6\x89\xca\x0a\x2b\xac\xf1\x39\x7e\xc5\x85\xf2\xfc\x28\x48\x8f\xf5\x2b\x5d\x56\x35\xba\x6e\xad\x45\x2b\x85\x32\xeb\x06\xca\xac\xbc\x49\xd3\xa5\xf6\x22\xba\x40\x14\x11\x70\xa3\x33\x4e\xcb\xf1\xef\x65\x45\xb3\x9c\x70\xb2\x31\xad\x8d\x6a\xdd\xb4\xe6\x3c\x63\x49\xab\xb1\x03\x21\x99\xb7\xa4\xc5\xe4\xac\x72\x31\x87\x56\x53\xb0\x11\xa1\x24\xf9\xf8\x9e\x52\xd8\x20\x02\x86\x6c\xb9\x83\xd0\x11\x9b\x7f\xba\x42\xe8\x1a\xda\x5b\xaf\x36\xae\x5b\xa8\x40\x39\x16\xda\x4c\x01\x94\x52\x93\xa9\x0d\x42\x2e\xe3\x06\xe6\xdc\xbb\xc8\x19\xdc\x83\x08\x16\x78\x92\xed\xcd\xad\x57\x2f\xab\xa8\xdc\xf6\xf2\xd6\x12\xf3\x11\xbd\x7d\x89\xc7\xca\x2c\xc4\x1f\xc9\xab\x59\xf9\xda\x92\x0c\xc2\xb2\xea\x8b\x63\x5b\x8c\x84\xa4\x55\x04\x02\xce\xa2\x86\x72\xf3\x0c\xf5\x6f\x48\x2a\xae\x23\xa8\x20\x4a\x2d\x31\x48\xa4\xc7\xf6\xb2\xcb\x76\x37\xa1\x3f\xc9\xc9\xbc\x51\x8c\x38\xbb\xcd\x27\xf6\xb9\xa2\xfa\xf1\xcd\xb1\x28\x07\x09\x47\xd5\x1d\x81\xd5\xd7\x8e\x5b\xed\xea\x67\xaf\xe6\x0e\x62\xed\x7a\xd9\xdf\x7d\xed\xef\xae\xfe\x63\xa7\xa3\xd7\x68\x24\xad\xa7\xfd\xdd\x75\xa2\x61\xf9\x3d\x79\x87\x90\x51\x97\x1d\x54\xfd\x53\xf4\x86\xf7\xf4\x86\xf5\xa7\xf4\x76\xf5\x1f\xda\xb9\xcd\xfb\x8e\xde\xdf\xa4\x2f\x39\xf1\xfd\x64\x34\x73\x0b\x19\x0d\x8e\x70\x1d\x6d\xa3\xbf\x42\x6b\x52\x75\x17\x19\x39\x78\x71\xfd\xe2\xff\x03\x00\x00\xff\xff\x90\x4e\xdc\x6b\xde\x14\x01\x00") +var _monitoringZyncGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x6f\xdb\xb8\x9a\xfe\xde\x5f\xc1\xd5\xcc\x39\x49\xbb\x76\x62\x3b\x71\x6e\x40\xb1\x68\xda\xe9\xce\x59\xb4\xdd\x4c\x9a\x0e\xd0\x29\x0a\x1f\x5a\x7a\x63\x73\x23\x91\x2a\x49\x25\x76\x83\xcc\x6f\x5f\x88\xba\x51\x37\xdb\x71\x94\xc4\x89\xf9\x61\xa6\x31\x29\x51\xd4\x7b\x7b\x1e\x92\x2f\xc5\xeb\x17\x08\x59\x98\x52\x26\xb1\x24\x8c\x0a\xeb\x08\x85\x45\x08\x59\x2e\x11\xd2\x3a\x42\xdf\xd4\x2f\x14\x97\xaa\x9a\x61\x40\x5c\xf9\x2f\x6a\x1d\xa1\x6e\x2b\x2b\x75\xb0\xc4\x82\x05\xdc\x06\xeb\x08\x59\xed\x36\xfa\x6f\x8e\xcf\x31\xc5\xa8\xdd\xb6\xb4\xcb\x80\xe2\xa1\x1b\x5e\x22\x79\x00\x5a\xf9\x98\x38\x15\xa5\xc4\x66\xf4\x2d\x73\x19\x0f\xdb\xe4\xa3\x21\xde\xec\xb4\x50\xaf\xdb\x6d\xa1\x5e\xbf\xdf\x42\xdd\x97\x7a\xd3\x14\x7b\xea\xd9\x6f\xb2\xd7\x41\xff\x44\x6f\x5c\xe0\x52\xe8\xd7\xc9\xa9\xaf\xae\x73\xb0\x18\x0f\x19\xe6\x8e\x15\xd7\xdd\xa8\x7f\xbf\xbf\x40\xe8\x26\xbc\xdc\x02\x87\xc8\x42\x6f\xad\x11\x05\xf9\x2f\xc7\x3a\x42\x34\x70\xdd\xa8\x84\x63\x7f\x7c\xc6\x98\x2b\x89\x9f\xc8\xc4\x22\x12\xb8\xea\x42\x5a\xe2\x12\x7a\x11\x8a\xf7\xdb\x77\xf5\xd3\xc7\x14\x5c\x91\x0a\x38\x11\xaf\x65\x33\xd7\xc5\xbe\x80\xf0\x11\xe7\xd8\x15\xa9\x34\xac\x11\x27\xce\x09\xcb\x34\x14\x89\xad\xa0\x85\x2b\xeb\x08\xf5\x76\xb5\x82\x89\x75\x84\x3a\xda\xef\x69\xf8\x3b\x79\xdf\xb4\x6d\x12\x3e\xee\xa0\x9f\xfe\xce\x7a\xf7\x3d\x2d\x93\x44\x2a\x51\x58\x7f\x4d\xa9\x6d\x65\xc5\xb1\x34\x39\xbb\x8a\xe4\x18\xb7\x9a\xbe\x11\x76\x09\x16\x4a\x89\xaa\xef\xd9\x43\x87\x58\x95\xe4\xdf\x32\x54\xca\x07\xa0\x23\xa9\xde\xac\x93\x2b\x87\xaa\xcb\x75\xab\xfb\x55\xfb\x99\x5e\x72\x4e\x5c\x57\x97\x52\xbd\x20\x0f\x0a\x82\xec\xf6\xe6\x08\xb2\x5b\x2d\xc8\xfd\xfd\xf4\xb7\x0b\x23\xa0\x4e\xfe\x51\xf8\x72\x54\x7c\x8f\x50\xf1\x01\xe7\x40\x65\x45\x8d\x87\x27\x55\xa5\x84\x56\x94\x8a\x31\xbb\x2a\xbb\x91\x64\x12\xbb\x15\x57\x5f\x62\x37\xc8\x84\x5a\x7a\x19\x97\x50\x55\xab\xb7\xa6\x0a\xaf\x88\x23\x73\x96\x57\xb0\x6e\x55\x14\x3a\xc8\x09\x23\x54\x7e\x64\xca\xb5\x55\x41\xa6\x16\xe6\xa7\x01\x27\x7b\xa2\x0f\xdc\x06\x2a\xf1\x08\x4a\x9a\xf6\xc3\xa6\x38\x76\x48\x10\xde\xd3\xcb\x97\x97\x0d\x83\x03\x75\x80\x83\x0a\x1c\xe7\x2e\x93\xd9\x83\x05\x70\x02\xe2\x7f\x2f\x81\x73\xe2\x40\xa1\xd3\xc2\xc7\x36\x54\xd9\x9f\x90\xd8\xbe\x28\xca\x42\x48\xf0\x7d\x70\x3e\x10\x5a\xee\xaf\xc4\x7c\x04\x52\x68\x21\x54\x0f\xa2\x61\x74\x99\xf8\xaa\x77\x22\xf0\x36\x39\x96\xb0\xc9\x31\x71\xc5\x80\xc3\x8f\x00\x84\x14\x03\xa5\xb4\xeb\x30\xa8\xa9\x4e\xbd\xde\xf8\x35\xfd\x7b\xa3\xe5\x33\xe7\xf5\xdf\x1b\x3f\xa7\xd4\x6e\x7f\xc3\xed\x9f\x9d\xf6\xe1\xf7\xff\xcc\xfe\xda\xb8\xf9\xd6\xf5\xbe\xbf\x7c\x89\x86\x53\xb4\x29\x24\x96\x81\xd0\x23\x65\xe8\x14\x8c\x7b\x38\x34\x36\x4b\x12\x0f\x06\x91\x4c\xf2\x97\x10\x2a\x81\x5f\x2a\xbb\xb1\xba\x5e\x75\xdd\x7b\x6c\x4b\x15\x9c\xbb\xb9\xea\xc8\xea\xdf\xa7\xcf\xb8\xbe\xfe\xf7\xf5\x75\xd4\x8f\x9b\x9b\x7f\xdf\xdc\xe4\x1b\xe3\x70\xae\x22\xaa\xf5\xc6\x4a\x8b\x6f\xe2\xbf\xb4\xf0\x33\xe6\x20\xc6\xcc\x75\x4a\x61\xc9\x83\xf7\x9c\x79\x5a\x48\x4e\xcb\x4f\x61\x14\xdb\x58\xe1\x86\xcf\x63\x72\x2e\xcb\x77\x68\x01\x0e\x25\x7a\x40\x3e\x70\x24\xc0\x66\xd4\x41\x9b\xc3\x29\x2a\x8a\xd3\x92\x69\xe8\xbf\xd6\xfd\x10\x73\x15\xc3\x0b\x9e\x28\x18\x97\x85\x40\xa2\x9c\x70\x90\x84\x51\x42\x1d\x72\x49\x9c\x00\xbb\x56\xc9\x1f\x93\x6b\x14\xe2\x64\x1d\x98\xe0\x09\x29\x44\xb3\x61\x60\x5f\x44\xc6\xa7\xbf\x61\x18\x35\x62\x5f\x0c\x85\x50\x81\x9d\x85\xab\xab\xa3\x49\x1a\x35\xbe\x7d\x2f\x75\x71\x8a\x27\x30\xc3\xe6\x1d\xb0\x89\x87\x15\xb2\x74\x6b\xec\x91\xc3\x0f\xbf\x60\x89\x2e\x1e\x82\x32\xc3\x42\x31\x1b\x1d\x63\x01\xa5\xb6\xa2\x78\x99\x7f\x95\x34\x60\x96\x8a\xb5\x77\xcc\xac\xaf\x55\xdd\xfd\xac\x97\x62\x1c\x2a\xb2\xb2\x97\xa5\x27\xdc\x63\x3f\x4b\x5e\x32\x2d\xdb\x02\x76\xc9\xa8\x0a\x2a\x54\xf9\x07\xb8\x4c\x3b\x9d\x63\x41\xcf\x18\xc5\x73\x05\xb3\x60\xfc\xc0\xc0\xb8\x81\xf1\x55\x81\x71\x9b\x51\xc9\x99\xeb\x02\x7f\x7c\x28\xcf\xfa\xf2\xe4\xe1\xbc\x4a\xac\x06\xd2\x0d\xa4\x23\x03\xe9\x4f\x08\xd2\x8b\x03\xf3\xc3\x1a\x44\x3f\x34\x88\x6e\x10\xfd\x31\x11\xbd\x15\x0d\x1f\x5f\xff\xbd\xd1\xfb\x16\x96\xbc\x32\x18\xdf\x1c\xc6\xf7\x26\x13\x83\xf3\xe9\x5b\x18\x9c\x37\x38\x8f\x9e\x36\xce\xef\x15\x60\xbe\x34\x72\xaf\xc1\xf9\x83\x8e\xc1\xf9\x06\x70\x3e\xf7\xd8\x15\x81\xf9\xfc\x53\x9e\x00\xce\xef\x1a\x9c\x6f\x1a\xe7\x77\x0d\xce\x6b\x6f\xf1\xec\x70\xde\xea\x58\x06\xe6\xd7\x1a\xe6\x0f\x16\x84\xf9\xae\x81\xf9\xe7\x3a\x9c\x7f\x72\x38\xdf\x37\x38\xdf\x34\xce\xf7\x0d\xce\x6b\x6f\x61\x70\xbe\xb2\x93\x06\xe7\x9f\x0c\xce\xcf\xcf\xa7\xdb\xaf\x01\xfa\x1d\x03\xf4\x06\xe8\xef\x08\xf4\x03\x27\x88\x52\x72\x07\x11\x8c\x88\x81\x08\xbc\x3b\x2e\xcc\x6f\xa3\x45\x9f\x65\xb3\x80\xca\xbb\x3d\x6d\x69\x0a\xb1\x20\x4d\x38\x53\x46\x3d\x9f\x17\xd4\x84\xe4\x1a\xe9\x3b\xc3\x01\x0f\x68\xdc\xc7\x7b\x93\x7b\xc5\x53\x9e\x80\xc4\xdf\xd8\x92\x5c\xc2\x29\xd8\x8c\x3b\x35\x82\x3f\x5e\x5a\xf0\x97\x04\xae\x1e\x42\xf4\x95\xcf\x79\x02\xc2\xff\x93\xc0\x55\x8d\xd0\xdf\xae\x08\x0b\x3e\x0d\x05\x8c\xf0\x25\x70\x3c\x02\xc4\x41\xf8\x8c\x0a\x40\x39\xc6\x68\x48\xef\x72\xa4\xb7\x44\xeb\x34\x4a\x69\xd6\xb0\x56\x8d\xf4\xda\x98\x3b\x85\x86\xc3\xa2\x13\xec\x38\x84\x8e\xca\x96\x13\x56\x9e\xb2\x80\x3a\x85\xc6\xd3\x9e\xda\xf1\xae\xa2\x42\x83\xe9\x66\xa3\x5f\xfa\xfb\x87\xbb\xef\x7b\xba\x8d\xaa\x5b\x3e\xdb\x38\xf2\x4d\xf1\x23\xa7\x81\xa4\x76\x0c\x5e\xec\x47\x12\xb8\xcf\x5c\x2c\xe1\x58\x99\xab\x76\x29\x4c\x7c\x46\x23\x6a\xda\xd9\xea\x57\x78\x07\xf3\xb1\x4d\xe4\xb4\xec\x81\x21\x1f\xcf\x22\x98\x14\x89\x9f\xdd\x86\xbf\x3b\x20\x6c\x4e\xfc\x78\x73\x52\x66\xd4\x8d\x66\xd4\x96\x89\xfc\x18\xb0\xf4\xb0\x9f\xa7\xa7\x63\xe2\xc0\x5f\xc0\xd9\x71\x1a\x2f\xf2\x94\x6f\x4c\x46\x63\x97\x8c\xc6\xf2\x6d\xac\xff\x1c\x73\x8e\xc6\x06\xb3\x37\xdb\xc4\x76\x5b\xcb\xc7\x8b\x24\xbb\x92\x45\x73\xb8\x04\x2e\xe0\x6b\x5d\x37\x1b\x67\xa6\x91\x5e\x17\xc0\xcf\xad\xfc\x84\x93\x0b\xb5\xb0\x99\xc8\xbf\x12\x32\x17\x9b\x64\xea\xcc\xc4\x53\x35\xcb\xe4\xc2\xf2\xb3\x4b\x33\xf0\x73\x0e\x4c\x56\xcc\x11\xc5\xaf\xab\xe6\x8a\x72\xc0\x89\x22\xd9\x22\x42\xe3\x4b\x17\xd8\xcb\x51\x85\x42\x61\xe9\xef\x44\x48\x36\xe2\xd8\xab\xb5\xb0\x04\x32\x8b\xd2\xb7\x26\x6f\x4a\x91\xb2\x1c\x62\xb3\x76\x26\x91\xe9\x7d\x0a\xbc\xa1\x1a\x82\xe5\x04\x11\x57\x7e\x26\x3f\x8b\x18\x6a\x4d\xcb\x8f\xd1\x30\x50\xa7\x01\xd5\xf0\x57\x8d\x1e\x95\xd8\x51\x89\x1c\x75\xc2\xf3\x5d\x22\x53\xc3\xaa\x0c\xd0\xd3\xe8\xa5\x8e\xe3\x20\x6e\xe1\x40\x32\xab\x58\x5b\x2d\x8f\x69\x49\x1e\x06\x53\xee\x80\x29\xb5\xb8\xb0\x7f\xdb\xf9\x9d\x5e\xff\xe1\x60\x61\xf6\xe6\x8d\x67\x0e\x0b\x95\x4b\x15\xd9\x94\xf9\xeb\x0d\xca\x24\x39\x27\x76\xb4\x4b\xda\x40\x88\xf5\x49\x97\x47\xe5\xa2\x83\x01\x14\x64\x00\xc5\x00\xca\x03\x0d\x52\x96\xc7\x9c\xd2\x58\xe4\x21\x41\x67\xf6\xfe\x82\x75\x07\x1d\x09\x14\x53\x69\xe0\x06\x59\x67\x91\x24\x0c\xd0\x18\xa0\x31\x40\xf3\x44\x81\xa6\x38\xb8\xd9\xe9\x3d\x18\xce\x1c\xce\x4e\x7c\x5b\x77\x9c\x89\x72\xb2\xb6\x5d\x72\x09\x06\x6b\xac\x0f\xe4\x12\x28\x08\x03\x36\x06\x6c\x0c\xd8\x3c\x55\xb0\x29\x8d\x6a\x1e\x12\x6d\x66\xef\xa6\x32\x68\xa3\xd0\x86\x03\x76\xa6\x06\x6e\xac\x53\xc0\x0e\x31\x78\x63\xf0\x66\x69\xbc\x79\x80\x2f\x60\xee\xd4\xec\x28\xe9\x6b\x69\x90\xf3\x3e\x81\x89\xfe\x08\xf4\x2c\x9c\x55\xff\x0c\x66\x61\xd9\x3f\x52\x11\x62\xe7\xe8\xff\xd8\x50\x20\xa0\x3f\x02\x08\xc0\x41\x98\x3a\x48\x85\x32\x24\x19\x1a\x02\x82\x09\xd8\x81\x04\x07\xbd\xf9\xfc\xe6\x04\x6d\xd2\x30\x4e\xa3\x73\x4c\x5c\x70\x5a\x88\x32\x8e\x46\x4c\x22\x98\xf8\x84\x83\xf3\x72\xab\x89\x2c\xe1\xb9\xca\xdb\xad\xf9\x7e\xe9\xae\xf9\xba\x87\xc9\x12\x5e\x9c\x29\x78\x78\xb2\xf9\x23\x80\x41\x68\xff\x03\x61\x8f\xc1\x09\x5c\x70\x16\xde\x0d\xf4\x23\x80\xad\x57\x1b\xad\xd0\xef\x5f\x6f\xc4\xe0\x1f\xe1\x7e\xc8\x31\xb9\x04\x27\x6c\xf9\xbe\xf7\xfd\x2c\xc0\x02\xf4\xee\x3c\x85\xad\x3f\xa7\x2a\xfa\xfc\x4f\x18\x95\x54\x16\xe9\xdd\x32\x1d\x7b\x45\xbf\x5b\xbf\x4c\xc7\x4e\x8d\x09\x52\x46\xe1\x3e\x73\x08\xcd\xfe\x1e\xbd\xfc\xc9\xec\xef\x59\x90\x28\x14\xe8\x81\x60\x5e\x4c\xe4\x09\x45\x72\x0c\xe8\x3c\x90\x01\x87\x16\x1a\x06\x12\x51\x16\xfe\x77\xf5\x20\xf4\x61\xee\x3c\xed\xee\x41\x35\x7d\x38\x34\x9b\x8c\x0c\x7d\x78\x1c\xfa\x90\xde\x6e\x28\x44\x23\x14\xe2\x73\x22\x4f\x43\x23\x0c\x8d\x30\x34\x62\x05\x69\x84\x1c\x63\x19\xf3\x00\x84\x25\x72\x01\x0b\x89\x18\xb5\xa1\x85\x1c\xe2\x28\xc2\xc0\x03\x8a\x58\x20\xc3\x5b\xb0\x94\xe0\xf9\x52\x84\xa4\x83\x83\xe4\x53\x35\x4d\x21\xc7\xc0\xe1\x9c\x71\x40\x98\x03\x4a\x63\x28\x3a\x67\x3c\xbd\x6a\x1a\x71\x12\xc1\x18\xbd\x1f\x76\x51\x9a\x98\xaf\xa5\x17\xbb\x86\x5e\x18\x7a\xf1\x28\xf4\x22\xf2\x33\xc3\x2d\x1a\xe1\x16\xef\xa3\xa0\x65\x88\x85\x21\x16\x86\x58\xac\x28\xb1\xc8\x66\x25\x02\xdb\x06\x21\xce\x03\xd7\x9d\x2e\x85\xff\xb7\xce\x02\xeb\xef\xd5\xc0\xff\x9e\x81\x7f\x03\xff\x8f\x03\xff\x84\x12\x31\x36\x04\xa0\x29\x02\x10\x8b\xd3\x50\x00\x43\x01\x0c\x05\x58\x55\x0a\x90\xcc\x2d\x50\x07\x71\x5c\x3f\x91\xd0\xd2\x66\x11\xae\x18\xdd\x90\x68\x08\xaa\x86\x84\x37\x8f\x30\x59\x6e\xda\xe0\xf6\xf9\x7c\xb5\xbc\xa1\x6f\x78\x83\xe1\x0d\x8f\xc2\x1b\xe2\x65\x39\x43\x1b\x1a\xa1\x0d\xbf\x45\xd2\x34\xac\xc1\xb0\x86\x35\x65\x0d\x0f\x91\xd8\xb9\xb7\x53\x0d\xa3\xdd\x6c\x71\xbf\x2a\xb1\x93\x83\x0f\x91\x56\x1c\xf0\x5d\x36\xf5\x20\xe7\x9e\x89\x0f\x9f\x30\x47\xa0\xcd\x5f\xb3\x6b\x5e\xde\x22\x01\xd4\xc6\xf6\x18\xce\x88\x07\x2c\x28\x45\x09\xb5\x83\xe1\x18\xdb\x17\x23\x1e\x27\xd6\xe6\x70\x52\x55\xff\x19\xfa\x4c\x49\x74\x76\xc2\xc2\x32\x77\xb1\x7e\x79\xdf\xdb\x3d\xec\xbf\xd5\x9d\x53\x9d\x9d\xdf\xdb\xd9\x6f\xa1\x6e\xef\xb0\x85\x76\x3b\x2d\xd4\xd9\x3a\x38\xcc\x9d\x9f\xff\x4b\xef\xf0\xd0\xde\xdd\xb3\x4a\x16\xb1\x10\x23\x2b\x7b\x65\x9d\x47\x5a\x23\x1c\x28\x50\xbe\xce\xf1\x92\xe4\xfd\xba\x9d\x4e\x9e\x9a\x24\x15\x9d\x72\x58\x29\x1a\x64\x1a\xcf\x3f\x84\x9e\x54\x42\x72\xfd\x8a\x8f\x98\x5f\x00\x17\x75\x99\xdd\xb5\x26\xb9\x53\x30\xc9\xe2\xc7\xeb\x4b\x16\xb9\x5b\x6a\x7b\x4c\x1c\x65\x08\x09\x6f\xa8\xdc\x75\xb1\x97\xad\x17\xe9\xd8\x69\xcd\xa2\x49\x1e\xf6\x7d\x42\x47\x67\x91\x29\x76\xab\xca\x67\x44\xd6\x38\x80\x47\xc1\x39\x64\xc8\x12\x26\x85\x00\x75\x99\xe8\x68\x6e\x9c\x4b\x1a\xe3\x98\x8e\xe6\x34\xd6\x9b\x11\x8c\x3c\x3c\x79\x87\x25\x3e\x49\x88\x99\x66\x1c\x65\x52\x68\x33\x4a\xc1\x96\x90\x7d\x4b\x53\x5d\x73\x16\x3e\xb9\xe0\x70\xd5\x8c\xd1\x0d\x46\x84\xfe\x09\x5c\xc4\x03\x8b\xbd\xad\xde\xd6\xae\xa5\xd1\x43\x21\xcf\xc9\x24\xaf\x86\xb8\xf0\x3d\xa3\x49\x66\xbb\xd5\xef\xfc\x43\xab\xe7\x50\xbe\x47\x95\xd5\xde\xa2\x64\xf6\x11\xfb\x33\x74\x75\x1e\x91\x93\x3c\x0f\x56\x35\x32\x7a\x5b\xeb\xd3\xf6\x9b\x42\x05\x4b\x6f\x98\x21\x70\xe1\x63\x7e\xe1\x46\x1c\x54\xb3\xfc\x70\xe8\x93\x6e\xd0\x52\xc1\x64\xa7\xdb\x42\xdd\xee\x41\x0b\x75\x0f\x0e\xc3\x60\xd2\x3d\xc8\x05\x93\xf3\xc0\xad\x1a\x25\x84\x2d\xeb\xed\x44\xcd\xf4\x3a\x2d\xd4\x3d\xdc\xc9\x35\x30\x73\x5b\x91\xc4\x43\x37\x6c\x27\xf0\x0a\xdf\xb0\xbb\xd5\x3e\xa1\x8b\x60\x08\x03\x0e\xbe\x1b\x7f\xf8\x25\xdb\xd5\x33\x88\x36\xf5\x0c\x54\x5e\x6f\x72\x89\xa8\x23\xd0\x95\x4d\xbc\xfe\x7b\x43\x03\x8a\xf6\xb7\xf8\xdb\xa6\x8d\x7f\xd6\x74\x09\xa2\x6b\x75\x5b\x3d\xab\x8a\xeb\x5a\x3b\x1d\x61\x55\x72\xda\x62\x4d\x92\xab\x1b\x50\x4a\xe8\x08\xf9\xcc\x11\x65\x28\x14\x84\x8e\x5c\x08\x45\x99\xd5\x29\xa7\xd7\x2d\xff\x40\xb7\x7c\x55\x3b\xdb\xf2\x59\x48\x93\xad\xd7\xd5\x46\xdf\xa9\x8e\x30\x73\xad\x5e\x5d\xf8\x29\x0e\x59\xe1\x98\xfa\x5e\x10\xfc\x24\x09\x05\x15\x10\x7e\x0b\x74\x8f\x61\xfa\xb6\xe8\x1e\x93\x02\x83\xee\x77\x41\xf7\xbd\xa6\xd0\xbd\x5f\x85\xee\x39\x8b\x32\xf8\xde\x38\xbe\x1b\xfc\x5e\x1b\xfc\xf6\xc1\x6e\x16\xb7\x51\x1b\x19\xce\xd0\x04\x67\xf8\x42\xf1\x25\x26\x6e\x68\x01\x86\x37\x98\x91\xff\xf3\xe1\x06\xa5\x35\x9d\xa5\xc9\xc1\x9e\x19\xfa\x9b\xa1\xbf\xa1\x0e\xf7\x44\x1d\xd4\xe2\xcb\x66\xf2\x7f\x2a\x31\xa1\xc0\x07\x1e\x78\x8c\x4f\x07\x81\xc0\x23\x18\x0c\xa7\x12\x6a\xa1\x3b\x5a\x2f\xcb\x01\x75\xd5\x29\x26\xd1\xaa\x19\x65\x0e\x34\x7f\x96\xc9\xa3\x02\xb8\x9a\x05\x77\x88\x90\x9c\x0c\x55\x96\x1d\xa3\x68\xcc\x84\x34\x48\xde\x24\x92\x2f\x39\xca\x77\x76\x77\xf1\x0e\x36\x48\x7e\x37\x24\x2f\x1e\x40\xbb\x2c\x92\xef\x54\x22\xb9\x19\xe6\x9b\x61\xbe\xc1\xea\x85\xf3\x59\xc2\x61\xb7\x03\xae\xc4\xd1\xe0\xdb\x67\xce\x20\x83\xed\x74\xd0\x2d\x24\xe6\x8b\x1d\xe6\x3b\x17\xb9\xbf\xf5\xbd\xef\x2f\xe3\x8f\x78\xf9\xcc\xb9\xf7\x93\xc8\x6a\x32\x57\x8e\x1f\x01\xdb\x3f\xe2\x89\x1a\x94\xa3\x44\xa0\x68\xd3\xc5\x42\xa2\x3e\xf2\x08\x0d\x24\x88\x8a\x15\xef\xe7\x06\xf2\x8f\x9d\xdd\xb8\x7c\x9a\xe1\xfc\xf4\x88\x9a\x03\x56\xbb\xcd\x9e\xa4\x1e\x62\xe3\x6f\x9e\x2f\xa7\xe5\xcc\x9f\xe4\x7b\x84\xe5\x9a\x27\x9a\x9a\xe8\xe1\xc9\x09\xf0\x53\xd5\x9f\x9d\x7a\x50\x0b\x0b\x10\x16\xe8\x67\xf8\xee\x73\xf0\x6b\xd1\xc4\xc5\x7e\xbe\xfc\x36\x89\x8b\x69\xc2\x4b\x0e\x4e\xa3\xd2\x77\x84\x83\x1d\xe7\xd6\xe6\xaa\x57\x2e\xdb\xf1\xc1\x27\x81\xef\x8e\x04\xbd\x99\x48\xa0\x2c\xb8\x9d\x9b\x15\xd5\xae\xfb\x40\xe8\x45\x55\x5e\x97\x36\x1a\xcc\x95\x87\x32\x56\xba\x98\x4b\xf3\x9e\xdd\xaa\x78\x51\xb0\x6a\xca\xb9\x42\xb0\xcb\x9e\xb3\x6a\x96\x1e\x1a\x52\x4c\x40\xe7\xaa\xe6\xed\xe2\xaa\x79\x06\x53\x3b\x25\x01\x09\x70\xda\xf9\xe9\x95\xbc\x78\xde\xad\x48\x82\xf3\x09\x73\xa2\xbc\x66\xb4\xa9\xc2\x58\x0b\x29\xcd\xb6\x50\x40\xc3\x7f\x5f\xaa\xbd\x18\x8a\x5b\xaa\xb7\xc9\x66\x90\x42\xa4\xc9\x9a\x33\xa9\xd0\x4d\xa6\x42\xdf\x7b\x92\x71\xfe\xc1\x4f\x2c\x13\x5a\xe7\x83\x6b\xb5\x81\x6a\xf1\x21\xc6\xde\x6d\x87\x18\xfb\xe5\xa9\xb2\x68\x23\x93\x19\x61\xdc\xff\x08\xa3\xa9\x91\xc5\x1d\xb6\x44\x3d\x9f\x91\xc5\x23\xcc\x39\x3d\xd8\x94\x93\xda\x42\xe5\x33\xe7\x29\xec\x9c\x3a\x59\x74\x3e\xca\x1c\x84\xbf\x30\x75\x78\x40\xf0\x35\xbb\xa5\xf4\xf2\x95\xda\x2d\x75\x50\xf7\x25\xf5\x2c\xfc\x2f\xbf\x59\xea\xed\xc9\x17\xf4\x25\x1c\x72\x2d\xbd\x63\xea\xe9\xb0\xa4\x5b\x4f\xc4\x1e\x74\xab\x25\xbf\x3f\xfb\xac\x95\x35\xd9\xee\xfd\xbc\xe7\x54\x2d\x0d\x32\x56\x92\xf8\x50\xe6\xc0\x20\xa5\x30\x79\xf2\x73\x94\xd1\x20\xdb\x0f\xe2\x39\x95\xe4\x50\x1d\x65\x2c\x47\xd7\xd7\x68\xeb\x73\xe0\x9d\x62\x09\xe8\xe6\x46\xe3\x45\x7f\x2f\x3f\xd7\xd2\x0c\x27\x9a\x3d\xf9\x3a\x83\x13\x35\x36\xff\xfa\x98\x3c\x2a\x0d\xc8\x66\xb6\x65\xc5\x29\x93\x99\x54\xd1\xca\x57\x8b\x31\xd5\x7c\xdd\xb5\xdf\x14\x63\xfa\x23\x60\x12\x3f\x10\x63\xb2\x55\x66\x46\xa1\xaf\x0f\x40\xa3\xce\xb5\xbc\x80\x6e\x47\x4f\x0c\x68\x90\x60\xd5\x9c\xf0\xd4\xec\x42\xb7\xe1\x57\xa9\xd5\x8f\xa0\xea\xa0\xb3\x95\xe5\x5d\x36\x67\xca\x2a\x73\x02\x9c\xcd\xc6\xc6\xec\xea\x77\xc0\x8e\xea\x42\xfe\xb6\x08\x18\x35\x3b\xb2\x99\x5b\x88\x2e\x0e\x08\xbb\x56\x9b\x0d\x12\x3d\x21\xa7\xee\x2c\x14\x53\x11\x22\x14\xc6\x59\x1e\x45\x23\x57\x86\x8c\x0d\x7d\xfd\xfa\xf5\x6b\xfb\xe3\xc7\xf6\xbb\x77\xe8\xf7\xdf\x8f\x3c\xef\x48\x14\xd8\x95\x8f\xa5\x04\x4e\xab\xdb\x4a\x0f\xd8\x23\x8e\x03\x74\xfe\x02\x5e\xda\xad\x32\x49\x49\x04\xca\x78\x6c\x97\x25\xe0\xc9\xf2\x67\xbf\xdf\xe5\x85\xb4\xe5\x94\x02\x51\x8c\x78\x5f\xc1\x45\xe3\x8a\xb3\x94\x42\x59\xef\x38\x71\x5d\xe4\xb0\x2b\x6a\x95\x2e\xfb\xc2\xdd\x72\xaa\x97\x26\x42\x95\x20\x8b\x7e\x29\x26\x03\x56\xb3\xc3\x9c\x88\x69\x74\xd8\x5e\xae\x2e\xa0\x44\x23\x01\xb7\x93\xfe\x69\x7c\x6c\xe3\x7a\x2a\xe0\x78\x75\x14\x80\xfe\xb1\x9e\x2a\x78\xdb\xac\x0a\x62\x08\x52\x3f\x6f\xa7\x88\x0f\xc4\x23\xeb\xea\x07\xef\x1e\xdf\x0f\x22\xf1\xaf\xab\x17\xfc\xb6\x0a\x5e\x70\xc2\x9c\xd5\x93\x7e\x9e\x41\x2f\x25\xfc\x6d\x67\xfb\xfa\x1a\x6d\x7d\x4a\x66\xa3\xd0\xcd\x4d\xa9\xa0\xbd\x23\x6c\xec\x42\xfb\x22\x18\x02\xa7\x20\x41\xb4\x6d\xe6\xf9\x81\x84\x36\x87\x68\x8c\x23\xda\x3e\x73\xfe\xeb\x12\xf3\x76\x36\xc9\x95\x4d\x71\xfd\x33\xac\xf0\x99\xf3\xfa\xd7\xc1\xc0\x86\xe2\x06\x00\x4d\xe3\x7e\x51\xca\x0f\xed\x6d\x2b\xa4\x63\x4d\x2c\xdb\x5b\xaf\xb6\x6f\x2f\x17\x21\x39\xa1\xa3\xc5\xe4\x12\xff\xa5\x4d\xa2\x3d\xe7\x69\x52\x3c\x74\xa1\x38\x41\x2a\x24\x56\x03\xdb\x92\x4f\xdd\x72\xee\xb4\x76\x09\x39\x57\x7e\x87\x84\xd4\xfc\x0a\x7c\xe2\x80\xc9\xf9\xee\x42\x49\xd9\x66\x3c\x97\xda\xf7\x8c\x04\x79\xdc\x84\x20\x57\xd4\x60\xd1\x36\x32\x6a\x4e\x52\x5d\xef\xd9\x5f\x5c\xc5\xa9\x9e\xbb\x18\xdf\xad\xb7\xb7\xac\x87\x92\x7f\x7b\x98\xc5\xb6\xf9\x6b\x6a\x6a\xca\xfe\x51\xd2\x90\x38\xa6\x22\x54\x42\x59\x05\x29\x1d\x2a\x14\x9b\xf5\xb6\x75\x59\x6f\x7b\x81\x1e\x69\x89\xec\xb0\xe6\x24\x83\x6c\x2b\xfc\x1d\x96\xc8\x3e\xaa\xad\x1c\x26\xaf\xa8\x56\xf8\x35\x1b\x3c\xb3\x84\xde\x35\x5e\xf6\x5a\xf5\x04\xa2\x95\x4c\x06\x5a\x70\x1b\xd5\x32\xc4\xa2\x85\xd2\xc6\xff\xe3\xf5\x86\x49\xf5\x69\x2e\xd5\x47\x0f\x93\x26\xdb\xe7\x6e\xec\x43\x59\xba\x61\x1f\xeb\x98\xed\xd3\xed\xd4\x6c\x66\xda\x6f\x90\xcb\x98\x8c\x9f\x66\xa8\x4f\xb7\xd3\xaf\x56\xd6\x8e\xe1\x3e\x4f\x37\xb9\xe7\xb1\x12\x75\x2a\x3f\x28\x66\xf2\x74\xd2\x6e\x55\x32\x8c\x44\xa4\x8f\xbe\x44\xf8\x04\x53\x75\x1c\xb0\x23\xa6\x71\x6b\x1d\x98\x84\x9d\x95\x52\xc3\xba\x26\x2c\xac\x44\xda\x4e\xac\x0b\x93\xb9\xd3\x98\x1e\x96\x77\x08\x93\xbf\xd3\xa0\x1a\x4c\xfe\x8e\xc9\xdf\x31\xf9\x3b\x8d\xe6\xef\xac\xd6\xcc\xe6\xb3\xce\xce\x89\x05\xdc\x84\x68\x57\x54\x96\x8d\x24\xe8\x3c\x8a\x45\xde\x26\xfd\xe6\xf9\xeb\xf1\xa1\x32\x70\x9e\xbf\x24\x1b\x49\xc2\x59\x55\x8f\x58\x1b\x2d\xae\x44\x96\x8d\x3e\x53\x6e\x12\x6d\x4c\xa2\xcd\x4a\x2d\x75\xbd\x40\x8f\xb5\x3a\xd5\xed\x95\x54\x1d\xed\x71\x6e\x64\x37\xfa\x27\x90\x57\x8c\x5f\x98\x5c\x9b\x7a\xf9\xd7\x1d\x36\x9f\x2d\x0f\x9a\x15\xa7\x95\xfb\x40\xe1\x4a\x66\xd9\x10\x8e\x25\x68\x6c\x87\x46\xce\x37\xe0\x60\x03\xb9\x8c\x09\x4f\xe9\x43\x83\x4b\xf1\x8c\xc6\xbf\x34\xb8\x74\xaa\xcd\xca\x7d\x69\xf0\x34\x92\x36\x3a\xc6\xd4\x89\x6c\xfc\x4e\x64\x63\x5d\x73\x67\xb4\xc9\xa1\x12\x96\x66\x06\x76\xec\xdf\x6b\x56\xcd\x13\xff\xec\xa0\xf9\x34\xf1\x3d\x00\x76\xaf\xe6\x7b\x87\xd9\x79\x5f\x06\xb0\x0d\x60\xdf\x09\xb0\xd5\x50\xd1\x23\xd2\x20\xf6\x83\x20\xf6\x59\x2c\x6e\x03\xd9\x4d\xcd\x01\x18\x58\x5e\x0d\x58\x7e\x11\x37\x1b\xfa\x5c\xe8\x4e\x51\xea\x64\xa4\x1b\x4b\xd8\x63\xf0\x70\x76\x5c\x71\x04\x60\x51\x52\x99\x1a\xd8\x63\x7e\x11\x5d\x29\xf1\x28\x53\xbb\x15\x2d\x89\xc6\x82\xb3\x7e\x4e\xa9\x6d\xa5\xcf\x91\xe0\xf9\x2e\x96\x84\x8e\xd2\xfe\x5b\x2e\x11\x52\x33\x1a\x1d\x80\xa3\x23\x2f\x75\x88\x25\xd4\x76\x03\x07\xde\x54\x9f\x28\x58\xa9\x1e\xcb\x0b\x5c\x49\x2a\x2e\x4f\xce\x9d\xac\x60\x08\x39\x28\xd2\x17\x11\xad\x1f\x01\xf0\xa9\x5a\x90\xe5\xcc\x03\x39\x86\x40\x37\x65\x4d\x90\xdd\x5c\xe9\x08\x26\x85\x09\x50\x4b\x5c\x10\xff\x0b\x77\x3f\x87\x02\x2a\x77\x2e\x71\x77\xad\x73\x45\x7f\xcb\xe9\xdf\x4d\x4e\x3f\x2d\xbc\x7c\xc6\x20\x72\x66\x9e\xa8\xac\xb0\x3e\x1a\x9f\x7d\x57\x5c\xe6\xce\x7b\x41\x7a\x14\x5e\xe9\xb2\x2a\xef\x5a\x5a\x8b\x56\x0a\x62\xd6\x2d\x94\x59\x79\x93\xa6\x4b\xed\x45\x74\x81\x28\x0a\xe0\x46\x27\x82\x96\xe7\xb9\x17\x15\xcd\x62\xc2\xc9\x7c\x5a\xf3\x6a\xdd\xb4\x66\x3c\x63\x41\xab\xb1\x03\x21\x99\xb7\xa0\xc5\xe4\xac\x72\x3e\x7b\x56\x83\xaf\x73\x42\x49\xf2\xc9\x3b\xa5\xb0\x41\x04\x0c\xd9\x82\x06\xa1\xe7\x6c\xf6\x79\x05\x61\x68\x68\x6f\xbd\xda\xb8\x69\xa1\x02\xe5\x98\x6b\x33\x05\x50\x4a\x4d\xa6\x62\xb2\x71\x91\x00\x50\x79\xd7\xbc\x00\x70\x0f\xaf\x3d\x27\x7a\x6c\x6f\x6e\xbd\x7a\x59\x45\xdf\xb6\x17\xb7\x90\x98\x83\xe8\xed\x4b\x3c\x52\xa6\x20\xfe\x48\x5e\xcd\xca\xd7\x96\x64\x10\x96\x55\x5f\x1c\xdb\x5f\x24\x24\xad\x22\x10\x70\x16\x35\x94\x1b\x55\xa8\x7f\x43\x22\x71\x13\xc1\x03\x51\x0a\x89\x81\x21\x3d\xd8\x96\x5d\xb5\xbb\x09\xe5\x49\xce\xae\x8d\xe6\x82\xb3\xdb\x7c\x62\x5f\x28\x62\x1f\xdf\x1c\x8b\x72\x90\xf0\x52\xdd\xf9\xad\xbe\x76\x2c\x69\x57\x3f\xa3\x34\x77\x60\x69\xd7\xcb\xfe\xee\x6b\x7f\x77\xf5\x1f\x3b\x1d\xbd\x46\x23\x66\x3d\xed\xef\xae\x13\xb9\xe2\xf7\xe4\x1d\x42\x16\x5d\x0e\x4a\xf5\x4f\xd1\x1b\xde\xd3\x1b\xd6\x9f\xd2\xdb\xd5\x7f\x68\x27\x1b\xef\x3b\x7a\x7f\x93\xbe\xe4\xc4\xf7\x93\xd1\x2c\x14\x64\xd4\x37\xc2\x72\xb4\x8d\xfe\x0a\xad\x49\xd5\x5d\x66\x84\xe0\xc5\xcd\x8b\xff\x0f\x00\x00\xff\xff\x23\x65\x66\x79\xdc\x13\x01\x00") func monitoringZyncGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper.go b/pkg/handlers/apimanager_managed_routes_event_mapper.go index 346a176bc..36c006aa9 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper.go @@ -6,8 +6,10 @@ import ( appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/go-logr/logr" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -45,7 +47,7 @@ func (h *APIManagerRoutesEventMapper) getAPIManagerOwnerReconcileRequest(object h.Logger.V(2).Info("Evaluating OwnerReference", "GroupVersion", refGV, "Kind", ref.Kind, "Name", ref.Name) // Compare the OwnerReference Group and Kind against the APIManager Kind - // or the name of the Zync Que DeploymentConfig. + // or the name of the Zync Que Deployment. // If the OwnerReference of the received object is an APIManager we // return a reconcile Request using the name from the OwnerReference and the @@ -62,19 +64,19 @@ func (h *APIManagerRoutesEventMapper) getAPIManagerOwnerReconcileRequest(object return request } - // If the OwnerReference of the received object is a DeploymentConfig and + // If the OwnerReference of the received object is a Deployment and // its name is Zync Que's name then we fetch that Object and recursively // try to find an OwnerReference that is an APIManager. If it is found // we return it. - zyncQueKind := "DeploymentConfig" - zyncQueGroup := appsv1.GroupVersion.Group + zyncQueKind := reconcilers.DeploymentKind + zyncQueGroup := k8sappsv1.GroupName // An alternative to hardcode Zync-Que name would be just try to recurse // OwnerReferences until there are no more of them. That would be // potentially more costly. zyncQueDeploymentName := component.ZyncQueDeploymentName if ref.Kind == zyncQueKind && refGV.Group == zyncQueGroup && ref.Name == zyncQueDeploymentName { h.Logger.V(2).Info("OwnerReference to Zync-Que detected. Recursively looking for APIManager OwnerReferences...") - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} getErr := h.K8sClient.Get(context.Background(), types.NamespacedName{Name: ref.Name, Namespace: object.GetNamespace()}, existing) if getErr != nil { // If there's an error getting the object it might be due to diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go index 69da83318..1fcb35b44 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go @@ -5,12 +5,11 @@ import ( "testing" "github.com/go-logr/logr" + routev1 "github.com/openshift/api/route/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - - routev1 "github.com/openshift/api/route/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -18,7 +17,8 @@ import ( appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + k8sappsv1 "k8s.io/api/apps/v1" ) func TestAPIManagerRoutesEventMapperMap(t *testing.T) { @@ -35,10 +35,10 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { }, } - zyncQue := &appsv1.DeploymentConfig{ + zyncQue := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", + Kind: reconcilers.DeploymentKind, + APIVersion: reconcilers.DeploymentAPIVersion, }, ObjectMeta: metav1.ObjectMeta{ Name: component.ZyncQueDeploymentName, @@ -57,12 +57,12 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -126,8 +126,8 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { Name: "asecret", }, metav1.OwnerReference{ - APIVersion: appsv1.GroupVersion.String(), - Kind: "DeploymentConfig", + APIVersion: reconcilers.DeploymentAPIVersion, + Kind: reconcilers.DeploymentKind, Name: component.ZyncQueDeploymentName, }, }, diff --git a/pkg/helper/deployment.go b/pkg/helper/deployment.go new file mode 100644 index 000000000..0c48acf53 --- /dev/null +++ b/pkg/helper/deployment.go @@ -0,0 +1,32 @@ +package helper + +import ( + k8sappsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// IsDeploymentAvailable returns true when the provided Deployment has the "Available" condition set to true +func IsDeploymentAvailable(d *k8sappsv1.Deployment) bool { + dConditions := d.Status.Conditions + for _, dCondition := range dConditions { + if dCondition.Type == k8sappsv1.DeploymentAvailable && dCondition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// IsDeploymentProgressing returns true when the provided Deployment is progressing with new ReplicaSet +func IsDeploymentProgressing(d *k8sappsv1.Deployment) bool { + if d.Status.UnavailableReplicas > 0 { + return true + } + + for _, dCondition := range d.Status.Conditions { + if dCondition.Type == k8sappsv1.DeploymentProgressing && dCondition.Reason == "ReplicaSetUpdated" { + return true + } + } + + return false +} diff --git a/pkg/helper/deploymentconfig.go b/pkg/helper/deploymentconfig.go deleted file mode 100644 index ae59579ff..000000000 --- a/pkg/helper/deploymentconfig.go +++ /dev/null @@ -1,40 +0,0 @@ -package helper - -import ( - "errors" - - appsv1 "github.com/openshift/api/apps/v1" - corev1 "k8s.io/api/core/v1" -) - -// IsDeploymentConfigAvailable returns true when the provided DeploymentConfig -// has the "Available" condition set to true -func IsDeploymentConfigAvailable(dc *appsv1.DeploymentConfig) bool { - dcConditions := dc.Status.Conditions - for _, dcCondition := range dcConditions { - if dcCondition.Type == appsv1.DeploymentAvailable && dcCondition.Status == corev1.ConditionTrue { - return true - } - } - return false -} - -func FindDeploymentTriggerOnImageChange(triggerPolicies []appsv1.DeploymentTriggerPolicy) (int, error) { - result := -1 - for i := range triggerPolicies { - if triggerPolicies[i].Type == appsv1.DeploymentTriggerOnImageChange { - result = i - break - } - } - - if result == -1 { - return -1, errors.New("no imageChangeParams deployment trigger policy found") - } - - return result, nil -} - -func IsDeploymentConfigDeleting(dc *appsv1.DeploymentConfig) bool { - return dc.GetDeletionTimestamp() != nil -} diff --git a/pkg/helper/envvarutils.go b/pkg/helper/envvarutils.go index d4131de67..e5a086c95 100644 --- a/pkg/helper/envvarutils.go +++ b/pkg/helper/envvarutils.go @@ -123,10 +123,10 @@ func RemoveDuplicateEnvVars(envVars []v1.EnvVar) []v1.EnvVar { return result } -// EnvVarReconciler implements basic env var reconcilliation. +// EnvVarReconciler implements basic env var reconciliation. // Added when in desired and not in existing // Updated when in desired and in existing but not equal -// Removed when not in desired and exists in existing DC +// Removed when not in desired and exists in existing Deployment func EnvVarReconciler(desired []v1.EnvVar, existing *[]v1.EnvVar, envVar string) bool { update := false diff --git a/pkg/helper/job.go b/pkg/helper/job.go index 7067dd6e5..07df6853a 100644 --- a/pkg/helper/job.go +++ b/pkg/helper/job.go @@ -1,11 +1,23 @@ package helper import ( + "context" "fmt" + "strconv" "strings" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + SystemAppGenerationAnnotation = "system-app-deployment-generation" ) // UIDBasedJobName returns a Job name that is compromised of the provided prefix, @@ -26,3 +38,114 @@ func UIDBasedJobName(prefix string, uid types.UID) (string, error) { return jobName, err } + +// HasJobCompleted checks if the provided Job has completed +func HasJobCompleted(jName string, jNamespace string, client k8sclient.Client) bool { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: jNamespace, + Name: jName, + }, job) + + // Return false on error + if err != nil { + return false + } + + // Check if Job has completed + jobConditions := job.Status.Conditions + for _, condition := range jobConditions { + if condition.Type == batchv1.JobComplete && condition.Status == corev1.ConditionTrue { + return true + } + } + + return false +} + +// HasAppGenerationChanged returns true if the system-app Deployment's generation doesn't match the Job's annotation tracking it +func HasAppGenerationChanged(jName string, dName string, namespace string, client k8sclient.Client) (bool, error) { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: namespace, + Name: jName, + }, job) + // Return error if can't get Job + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting job %s: %w", job.Name, err) + } + // Return false if the Job doesn't exist yet + if k8serr.IsNotFound(err) { + return false, nil + } + + deployment := &k8sappsv1.Deployment{} + err = client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: namespace, + Name: dName, + }, deployment) + // Return error if can't get Deployment + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) + } + // Return false if the Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + return false, nil + } + + // Parse the Job's observed Deployment generation from its annotations + var trackedGeneration int64 = 1 + if job.Annotations != nil { + for key, val := range job.Annotations { + if key == SystemAppGenerationAnnotation { + trackedGeneration, err = strconv.ParseInt(val, 10, 64) + if err != nil { + return false, fmt.Errorf("failed to parse system-app Deployment's generation from job %s annotations: %w", job.Name, err) + } + } + } + } + + // Return true if the Deployment's version doesn't match the version tracked in the Job's annotation + if trackedGeneration != deployment.ObjectMeta.Generation { + return true, nil + } + + return false, nil +} + +func DeleteJob(jName string, jNamespace string, client k8sclient.Client) error { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: jNamespace, + Name: jName, + }, job) + + // Breakout if the Job has already been deleted + if k8serr.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("error getting job %s: %v", job.Name, err) + } + + // Return error if Job is currently running + if !HasJobCompleted(job.Name, job.Namespace, client) { + return fmt.Errorf("won't delete job %s because it's still running", job.Name) + } + + deleteOptions := []k8sclient.DeleteOption{ + k8sclient.GracePeriodSeconds(int64(0)), + k8sclient.PropagationPolicy(metav1.DeletePropagationForeground), + } + + // Delete the Job + err = client.Delete(context.TODO(), job, deleteOptions...) + if err != nil { + if !k8serr.IsNotFound(err) { + return fmt.Errorf("error deleting job %s: %v", job.Name, err) + } + } + + return nil +} diff --git a/pkg/reconcilers/deploymentconfig.go b/pkg/reconcilers/deployment.go similarity index 53% rename from pkg/reconcilers/deploymentconfig.go rename to pkg/reconcilers/deployment.go index 4808f5ec8..e3b4e1ab7 100644 --- a/pkg/reconcilers/deploymentconfig.go +++ b/pkg/reconcilers/deployment.go @@ -6,7 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" @@ -14,18 +14,43 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" ) -// DCMutateFn is a function which mutates the existing DeploymentConfig into it's desired state. -type DCMutateFn func(desired, existing *appsv1.DeploymentConfig) (bool, error) +const ( + DeploymentKind = "Deployment" + DeploymentAPIVersion = "apps/v1" + DeploymentLabelSelector = "deployment" +) + +type ContainerImage struct { + Name string + Tag string +} + +type ImageTriggerFrom struct { + Kind string `json:"kind"` + Name string `json:"name"` + // +optional + Namespace *string `json:"namespace,omitempty"` +} + +type ImageTrigger struct { + From ImageTriggerFrom `json:"from"` + FieldPath string `json:"fieldPath"` + // +optional + Paused bool `json:"paused,omitempty"` +} -func DeploymentConfigMutator(opts ...DCMutateFn) MutateFn { +// DMutateFn is a function which mutates the existing Deployment into it's desired state. +type DMutateFn func(desired, existing *k8sappsv1.Deployment) (bool, error) + +func DeploymentMutator(opts ...DMutateFn) MutateFn { return func(existingObj, desiredObj common.KubernetesObject) (bool, error) { - existing, ok := existingObj.(*appsv1.DeploymentConfig) + existing, ok := existingObj.(*k8sappsv1.Deployment) if !ok { - return false, fmt.Errorf("%T is not a *appsv1.DeploymentConfig", existingObj) + return false, fmt.Errorf("%T is not a *k8sappsv1.Deployment", existingObj) } - desired, ok := desiredObj.(*appsv1.DeploymentConfig) + desired, ok := desiredObj.(*k8sappsv1.Deployment) if !ok { - return false, fmt.Errorf("%T is not a *appsv1.DeploymentConfig", desiredObj) + return false, fmt.Errorf("%T is not a *k8sappsv1.Deployment", desiredObj) } update := false @@ -43,23 +68,34 @@ func DeploymentConfigMutator(opts ...DCMutateFn) MutateFn { } } -// GenericBackendMutators returns the generic mutators for backend -func GenericBackendMutators() []DCMutateFn { - return []DCMutateFn{ - DeploymentConfigImageChangeTriggerMutator, - DeploymentConfigContainerResourcesMutator, - DeploymentConfigAffinityMutator, - DeploymentConfigTolerationsMutator, - DeploymentConfigPodTemplateLabelsMutator, - DeploymentConfigPriorityClassMutator, - DeploymentConfigTopologySpreadConstraintsMutator, - DeploymentConfigPodTemplateAnnotationsMutator, - DeploymentConfigArgsMutator, - DeploymentConfigProbesMutator, +// GenericBackendDeploymentMutators returns the generic mutators for backend +func GenericBackendDeploymentMutators() []DMutateFn { + return []DMutateFn{ + DeploymentAnnotationsMutator, + DeploymentContainerResourcesMutator, + DeploymentAffinityMutator, + DeploymentTolerationsMutator, + DeploymentPodTemplateLabelsMutator, + DeploymentPriorityClassMutator, + DeploymentTopologySpreadConstraintsMutator, + DeploymentPodTemplateAnnotationsMutator, + DeploymentArgsMutator, + DeploymentProbesMutator, + DeploymentPodContainerImageMutator, + DeploymentPodInitContainerImageMutator, } } -func DeploymentConfigReplicasMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentAnnotationsMutator ensures Deployment Annotations are reconciled +func DeploymentAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + helper.MergeMapStringString(&updated, &existing.ObjectMeta.Annotations, desired.ObjectMeta.Annotations) + + return updated, nil +} + +func DeploymentReplicasMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { update := false if desired.Spec.Replicas != existing.Spec.Replicas { @@ -70,7 +106,7 @@ func DeploymentConfigReplicasMutator(desired, existing *appsv1.DeploymentConfig) return update, nil } -func DeploymentConfigAffinityMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func DeploymentAffinityMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false if !reflect.DeepEqual(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) { @@ -83,7 +119,7 @@ func DeploymentConfigAffinityMutator(desired, existing *appsv1.DeploymentConfig) return updated, nil } -func DeploymentConfigTolerationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func DeploymentTolerationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false if !reflect.DeepEqual(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) { @@ -96,7 +132,7 @@ func DeploymentConfigTolerationsMutator(desired, existing *appsv1.DeploymentConf return updated, nil } -func DeploymentConfigContainerResourcesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func DeploymentContainerResourcesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { desiredName := common.ObjectInfo(desired) update := false @@ -120,23 +156,23 @@ func DeploymentConfigContainerResourcesMutator(desired, existing *appsv1.Deploym return update, nil } -// DeploymentConfigEnvVarReconciler implements basic env var reconcilliation deployment configs. -// Existing and desired DC must have same number of containers +// DeploymentEnvVarReconciler implements basic env var reconciliation deployments. +// Existing and desired Deployment must have same number of containers // Added when in desired and not in existing // Updated when in desired and in existing but not equal -// Removed when not in desired and exists in existing DC -func DeploymentConfigEnvVarReconciler(desired, existing *appsv1.DeploymentConfig, envVar string) bool { +// Removed when not in desired and exists in existing Deployment +func DeploymentEnvVarReconciler(desired, existing *k8sappsv1.Deployment, envVar string) bool { updated := false if len(desired.Spec.Template.Spec.Containers) != len(existing.Spec.Template.Spec.Containers) { - log.Info("[WARNING] not reconciling deployment config", + log.Info("[WARNING] not reconciling deployment", "name", client.ObjectKeyFromObject(desired), "reason", "existing and desired do not have same number of containers") return false } if len(desired.Spec.Template.Spec.InitContainers) != len(existing.Spec.Template.Spec.InitContainers) { - log.Info("[WARNING] not reconciling deployment config", + log.Info("[WARNING] not reconciling deployment", "name", client.ObjectKeyFromObject(desired), "reason", "existing and desired do not have same number of init containers") return false @@ -160,64 +196,11 @@ func DeploymentConfigEnvVarReconciler(desired, existing *appsv1.DeploymentConfig updated = updated || tmpChanged } - // Pre Hook pod - if existing.Spec.Strategy.RollingParams != nil && - existing.Spec.Strategy.RollingParams.Pre != nil && - existing.Spec.Strategy.RollingParams.Pre.ExecNewPod != nil && - desired.Spec.Strategy.RollingParams != nil && - desired.Spec.Strategy.RollingParams.Pre != nil && - desired.Spec.Strategy.RollingParams.Pre.ExecNewPod != nil { - // reconcile Pre Hook - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, - &existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, - envVar) - updated = updated || tmpChanged - } - - // Post Hook pod - if existing.Spec.Strategy.RollingParams != nil && - existing.Spec.Strategy.RollingParams.Post != nil && - existing.Spec.Strategy.RollingParams.Post.ExecNewPod != nil && - desired.Spec.Strategy.RollingParams != nil && - desired.Spec.Strategy.RollingParams.Post != nil && - desired.Spec.Strategy.RollingParams.Post.ExecNewPod != nil { - // reconcile Pre Hook - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, - &existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, - envVar) - updated = updated || tmpChanged - } - return updated } -// DeploymentConfigImageChangeTriggerMutator ensures image change triggers are reconciled -func DeploymentConfigImageChangeTriggerMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - desiredDeploymentTriggerImageChangePos, err := helper.FindDeploymentTriggerOnImageChange(desired.Spec.Triggers) - if err != nil { - return false, fmt.Errorf("unexpected: '%s' in DeploymentConfig '%s'", err, desired.Name) - - } - existingDeploymentTriggerImageChangePos, err := helper.FindDeploymentTriggerOnImageChange(existing.Spec.Triggers) - if err != nil { - return false, fmt.Errorf("unexpected: '%s' in DeploymentConfig '%s'", err, existing.Name) - } - - desiredDeploymentTriggerImageChangeParams := desired.Spec.Triggers[desiredDeploymentTriggerImageChangePos].ImageChangeParams - existingDeploymentTriggerImageChangeParams := existing.Spec.Triggers[existingDeploymentTriggerImageChangePos].ImageChangeParams - - if !reflect.DeepEqual(existingDeploymentTriggerImageChangeParams.From.Name, desiredDeploymentTriggerImageChangeParams.From.Name) { - existingDeploymentTriggerImageChangeParams.From.Name = desiredDeploymentTriggerImageChangeParams.From.Name - return true, nil - } - - return false, nil -} - -// DeploymentConfigPodTemplateLabelsMutator ensures pod template labels are reconciled -func DeploymentConfigPodTemplateLabelsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentPodTemplateLabelsMutator ensures pod template labels are reconciled +func DeploymentPodTemplateLabelsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false helper.MergeMapStringString(&updated, &existing.Spec.Template.Labels, desired.Spec.Template.Labels) @@ -225,8 +208,8 @@ func DeploymentConfigPodTemplateLabelsMutator(desired, existing *appsv1.Deployme return updated, nil } -// DeploymentConfigRemoveDuplicateEnvVarMutator ensures pod env vars are not duplicated -func DeploymentConfigRemoveDuplicateEnvVarMutator(_, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentRemoveDuplicateEnvVarMutator ensures pod env vars are not duplicated +func DeploymentRemoveDuplicateEnvVarMutator(_, existing *k8sappsv1.Deployment) (bool, error) { updated := false for idx := range existing.Spec.Template.Spec.Containers { prunedEnvs := helper.RemoveDuplicateEnvVars(existing.Spec.Template.Spec.Containers[idx].Env) @@ -239,8 +222,8 @@ func DeploymentConfigRemoveDuplicateEnvVarMutator(_, existing *appsv1.Deployment return updated, nil } -// DeploymentConfigPriorityClassMutator ensures priorityclass is reconciled -func DeploymentConfigPriorityClassMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentPriorityClassMutator ensures priorityclass is reconciled +func DeploymentPriorityClassMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false if existing.Spec.Template.Spec.PriorityClassName != desired.Spec.Template.Spec.PriorityClassName { @@ -251,8 +234,8 @@ func DeploymentConfigPriorityClassMutator(desired, existing *appsv1.DeploymentCo return updated, nil } -// DeploymentConfigStrategyMutator ensures desired strategy -func DeploymentConfigStrategyMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentStrategyMutator ensures desired strategy +func DeploymentStrategyMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false if !reflect.DeepEqual(existing.Spec.Strategy, desired.Spec.Strategy) { @@ -263,8 +246,8 @@ func DeploymentConfigStrategyMutator(desired, existing *appsv1.DeploymentConfig) return updated, nil } -// DeploymentConfigTopologySpreadConstraintsMutator ensures TopologySpreadConstraints is reconciled -func DeploymentConfigTopologySpreadConstraintsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentTopologySpreadConstraintsMutator ensures TopologySpreadConstraints is reconciled +func DeploymentTopologySpreadConstraintsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false if !reflect.DeepEqual(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) { @@ -277,8 +260,8 @@ func DeploymentConfigTopologySpreadConstraintsMutator(desired, existing *appsv1. return updated, nil } -// DeploymentConfigPodTemplateAnnotationsMutator ensures Pod Template Annotations is reconciled -func DeploymentConfigPodTemplateAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentPodTemplateAnnotationsMutator ensures Pod Template Annotations is reconciled +func DeploymentPodTemplateAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false helper.MergeMapStringString(&updated, &existing.Spec.Template.Annotations, desired.Spec.Template.Annotations) @@ -286,8 +269,8 @@ func DeploymentConfigPodTemplateAnnotationsMutator(desired, existing *appsv1.Dep return updated, nil } -// DeploymentConfigArgsMutator ensures Args are reconciled -func DeploymentConfigArgsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentArgsMutator ensures deployment's containers' args are reconciled +func DeploymentArgsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false for i, desiredContainer := range desired.Spec.Template.Spec.Containers { @@ -302,8 +285,8 @@ func DeploymentConfigArgsMutator(desired, existing *appsv1.DeploymentConfig) (bo return updated, nil } -// DeploymentConfigProbesMutator ensures probes are reconciled -func DeploymentConfigProbesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +// DeploymentProbesMutator ensures probes are reconciled +func DeploymentProbesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { updated := false for i, desiredContainer := range desired.Spec.Template.Spec.Containers { @@ -321,4 +304,34 @@ func DeploymentConfigProbesMutator(desired, existing *appsv1.DeploymentConfig) ( } return updated, nil -} \ No newline at end of file +} + +// DeploymentPodContainerImageMutator ensures that the deployment's pod's containers are reconciled +func DeploymentPodContainerImageMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.Containers { + existingContainer := &existing.Spec.Template.Spec.Containers[i] + + if !reflect.DeepEqual(existingContainer.Image, desiredContainer.Image) { + existingContainer.Image = desiredContainer.Image + updated = true + } + } + return updated, nil +} + +// DeploymentPodInitContainerImageMutator ensures that the deployment's pod's containers are reconciled +func DeploymentPodInitContainerImageMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.InitContainers { + existingContainer := &existing.Spec.Template.Spec.InitContainers[i] + + if !reflect.DeepEqual(existingContainer.Image, desiredContainer.Image) { + existingContainer.Image = desiredContainer.Image + updated = true + } + } + return updated, nil +} diff --git a/pkg/reconcilers/deploymentconfig_test.go b/pkg/reconcilers/deployment_test.go similarity index 51% rename from pkg/reconcilers/deploymentconfig_test.go rename to pkg/reconcilers/deployment_test.go index ceed447f8..ae9ddbac9 100644 --- a/pkg/reconcilers/deploymentconfig_test.go +++ b/pkg/reconcilers/deployment_test.go @@ -5,43 +5,45 @@ import ( "testing" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) -func TestDeploymentConfigReplicasMutator(t *testing.T) { - dcFactory := func() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func TestDeploymentReplicasMutator(t *testing.T) { + numReplicas := int32(3) + dFactory := func() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: 3, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &numReplicas, }, } } cases := []struct { testName string - desired func() *appsv1.DeploymentConfig + desired func() *k8sappsv1.Deployment expectedResult bool }{ - {"NothingToReconcile", func() *appsv1.DeploymentConfig { return dcFactory() }, false}, + {"NothingToReconcile", func() *k8sappsv1.Deployment { return dFactory() }, false}, {"ReplicasReconcile", - func() *appsv1.DeploymentConfig { - desired := dcFactory() - desired.Spec.Replicas = desired.Spec.Replicas + 1000 + func() *k8sappsv1.Deployment { + desired := dFactory() + newNumReplicas := *desired.Spec.Replicas + int32(1000) + desired.Spec.Replicas = &newNumReplicas return desired }, true, }, @@ -49,15 +51,15 @@ func TestDeploymentConfigReplicasMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory() - update, err := DeploymentConfigReplicasMutator(tc.desired(), existing) + existing := dFactory() + update, err := DeploymentReplicasMutator(tc.desired(), existing) if err != nil { subT.Fatal(err) } if update != tc.expectedResult { subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) } - if existing.Spec.Replicas != tc.desired().Spec.Replicas { + if *existing.Spec.Replicas != *tc.desired().Spec.Replicas { subT.Fatalf("replica reconciliation failed, existing: %d, desired: %d", existing.Spec.Replicas, tc.desired().Spec.Replicas) } @@ -65,7 +67,7 @@ func TestDeploymentConfigReplicasMutator(t *testing.T) { } } -func TestDeploymentConfigContainerResourcesMutator(t *testing.T) { +func TestDeploymentContainerResourcesMutator(t *testing.T) { emptyResourceRequirements := corev1.ResourceRequirements{ Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}, @@ -80,21 +82,21 @@ func TestDeploymentConfigContainerResourcesMutator(t *testing.T) { corev1.ResourceMemory: resource.MustParse("220Mi"), }, } - dcFactory := func(resources corev1.ResourceRequirements) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ + dFactory := func(resources corev1.ResourceRequirements) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", Resources: resources, }, @@ -119,9 +121,9 @@ func TestDeploymentConfigContainerResourcesMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingResources) - desired := dcFactory(tc.desiredResources) - update, err := DeploymentConfigContainerResourcesMutator(desired, existing) + existing := dFactory(tc.existingResources) + desired := dFactory(tc.desiredResources) + update, err := DeploymentContainerResourcesMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -135,14 +137,14 @@ func TestDeploymentConfigContainerResourcesMutator(t *testing.T) { } } -func TestDeploymentConfigAffinityMutator(t *testing.T) { +func TestDeploymentAffinityMutator(t *testing.T) { testAffinity1 := &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ - corev1.NodeSelectorTerm{ + { MatchFields: []corev1.NodeSelectorRequirement{ - v1.NodeSelectorRequirement{ + { Key: "key1", Operator: corev1.NodeSelectorOpIn, Values: []string{"val1"}, @@ -157,9 +159,9 @@ func TestDeploymentConfigAffinityMutator(t *testing.T) { NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ - corev1.NodeSelectorTerm{ + { MatchFields: []corev1.NodeSelectorRequirement{ - v1.NodeSelectorRequirement{ + { Key: "key2", Operator: corev1.NodeSelectorOpIn, Values: []string{"val2"}, @@ -170,18 +172,18 @@ func TestDeploymentConfigAffinityMutator(t *testing.T) { }, }, } - dcFactory := func(affinity *corev1.Affinity) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ + dFactory := func(affinity *corev1.Affinity) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Affinity: affinity, }, @@ -203,9 +205,9 @@ func TestDeploymentConfigAffinityMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingAffinity) - desired := dcFactory(tc.desiredAffinity) - update, err := DeploymentConfigAffinityMutator(desired, existing) + existing := dFactory(tc.existingAffinity) + desired := dFactory(tc.desiredAffinity) + update, err := DeploymentAffinityMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -219,15 +221,15 @@ func TestDeploymentConfigAffinityMutator(t *testing.T) { } } -func TestDeploymentConfigTolerationsMutator(t *testing.T) { +func TestDeploymentTolerationsMutator(t *testing.T) { testTolerations1 := []corev1.Toleration{ - corev1.Toleration{ + { Key: "key1", Effect: corev1.TaintEffectNoExecute, Operator: corev1.TolerationOpEqual, Value: "val1", }, - corev1.Toleration{ + { Key: "key2", Effect: corev1.TaintEffectNoExecute, Operator: corev1.TolerationOpEqual, @@ -235,31 +237,31 @@ func TestDeploymentConfigTolerationsMutator(t *testing.T) { }, } testTolerations2 := []corev1.Toleration{ - corev1.Toleration{ + { Key: "key3", Effect: corev1.TaintEffectNoExecute, Operator: corev1.TolerationOpEqual, Value: "val3", }, - corev1.Toleration{ + { Key: "key4", Effect: corev1.TaintEffectNoExecute, Operator: corev1.TolerationOpEqual, Value: "val4", }, } - dcFactory := func(toleration []corev1.Toleration) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ + dFactory := func(toleration []corev1.Toleration) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Tolerations: toleration, }, @@ -281,9 +283,9 @@ func TestDeploymentConfigTolerationsMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTolerations) - desired := dcFactory(tc.desiredTolerations) - update, err := DeploymentConfigTolerationsMutator(desired, existing) + existing := dFactory(tc.existingTolerations) + desired := dFactory(tc.desiredTolerations) + update, err := DeploymentTolerationsMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -298,19 +300,19 @@ func TestDeploymentConfigTolerationsMutator(t *testing.T) { } -func TestDeploymentConfigEnvVarReconciler(t *testing.T) { +func TestDeploymentEnvVarReconciler(t *testing.T) { t.Run("DifferentNumberOfContainers", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ + desired := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", }, }, @@ -318,20 +320,20 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, }, } - existing := &appsv1.DeploymentConfig{ + existing := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", }, - corev1.Container{ + { Name: "container2", }, }, @@ -340,29 +342,29 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, } - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") + update := DeploymentEnvVarReconciler(desired, existing, "A") if update { subT.Fatal("expected not to be updated") } }) t.Run("DifferentNumberOfInitContainers", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ + desired := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", }, }, InitContainers: []corev1.Container{ - corev1.Container{ + { Name: "initcontainer1", }, }, @@ -370,25 +372,25 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, }, } - existing := &appsv1.DeploymentConfig{ + existing := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", }, }, InitContainers: []corev1.Container{ - corev1.Container{ + { Name: "initcontainer1", }, - corev1.Container{ + { Name: "initcontainer2", }, }, @@ -397,30 +399,30 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, } - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") + update := DeploymentEnvVarReconciler(desired, existing, "A") if update { subT.Fatal("expected not to be updated") } }) t.Run("ContainersEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ + desired := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", Env: []corev1.EnvVar{ {Name: "A", Value: "valueA"}, }, }, - corev1.Container{ + { Name: "container2", Env: []corev1.EnvVar{}, }, @@ -429,21 +431,21 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, }, } - existing := &appsv1.DeploymentConfig{ + existing := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "container1", Env: []corev1.EnvVar{}, }, - corev1.Container{ + { Name: "container2", Env: []corev1.EnvVar{ {Name: "A", Value: "valueA"}, @@ -455,7 +457,7 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, } - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") + update := DeploymentEnvVarReconciler(desired, existing, "A") if !update { subT.Fatal("expected not be updated") } @@ -469,23 +471,23 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }) t.Run("InitContainersEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ + desired := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ - corev1.Container{ + { Name: "intcontainer1", Env: []corev1.EnvVar{ {Name: "A", Value: "valueA"}, }, }, - corev1.Container{ + { Name: "intcontainer2", Env: []corev1.EnvVar{}, }, @@ -494,21 +496,21 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, }, } - existing := &appsv1.DeploymentConfig{ + existing := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ - corev1.Container{ + { Name: "initcontainer1", Env: []corev1.EnvVar{}, }, - corev1.Container{ + { Name: "initcontainer2", Env: []corev1.EnvVar{ {Name: "A", Value: "valueA"}, @@ -520,7 +522,7 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { }, } - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") + update := DeploymentEnvVarReconciler(desired, existing, "A") if !update { subT.Fatal("expected not be updated") } @@ -531,205 +533,21 @@ func TestDeploymentConfigEnvVarReconciler(t *testing.T) { } } }) - - t.Run("PreHookEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Pre: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Pre: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - if !reflect.DeepEqual(existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env) { - subT.Fatal(cmp.Diff(existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env)) - } - }) - - t.Run("PostHookEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Post: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Post: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - if !reflect.DeepEqual(existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env) { - subT.Fatal(cmp.Diff(existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env)) - } - }) -} - -func TestDeploymentConfigImageChangeTriggerMutator(t *testing.T) { - dcFactory := func(triggers []appsv1.DeploymentTriggerPolicy) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: triggers, - }, - } - } - - sliceCopy := func(a []appsv1.DeploymentTriggerPolicy) []appsv1.DeploymentTriggerPolicy { - return append(a[:0:0], a...) - } - - triggersA := []appsv1.DeploymentTriggerPolicy{ - { - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - From: corev1.ObjectReference{ - Name: "imagestreamA", - }, - }, - }, - } - - triggersB := []appsv1.DeploymentTriggerPolicy{ - { - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - From: corev1.ObjectReference{ - Name: "imagestreamB", - }, - }, - }, - } - - cases := []struct { - testName string - existingTriggers []appsv1.DeploymentTriggerPolicy - desiredTriggers []appsv1.DeploymentTriggerPolicy - expectedResult bool - }{ - {"NothingToReconcile", sliceCopy(triggersA), sliceCopy(triggersA), false}, - {"DifferentName", sliceCopy(triggersA), sliceCopy(triggersB), true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTriggers) - desired := dcFactory(tc.desiredTriggers) - update, err := DeploymentConfigImageChangeTriggerMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - // It should be tested changes in triggers on image change only, but good enough for now - if !reflect.DeepEqual(existing.Spec.Triggers, desired.Spec.Triggers) { - subT.Fatal(cmp.Diff(existing.Spec.Triggers, desired.Spec.Triggers)) - } - }) - } } -func TestDeploymentConfigPodTemplateLabelsMutator(t *testing.T) { - dcFactory := func(labels map[string]string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func TestDeploymentPodTemplateLabelsMutator(t *testing.T) { + dFactory := func(labels map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, }, @@ -768,9 +586,9 @@ func TestDeploymentConfigPodTemplateLabelsMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingLabels) - desired := dcFactory(tc.desiredLabels) - update, err := DeploymentConfigPodTemplateLabelsMutator(desired, existing) + existing := dFactory(tc.existingLabels) + desired := dFactory(tc.desiredLabels) + update, err := DeploymentPodTemplateLabelsMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -784,19 +602,19 @@ func TestDeploymentConfigPodTemplateLabelsMutator(t *testing.T) { } } -func TestDeploymentConfigRemoveDuplicateEnvVarMutator(t *testing.T) { - dcFactory := func(envs []corev1.EnvVar) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func TestDeploymentRemoveDuplicateEnvVarMutator(t *testing.T) { + dFactory := func(envs []corev1.EnvVar) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { @@ -829,8 +647,8 @@ func TestDeploymentConfigRemoveDuplicateEnvVarMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingEnvs) - update, err := DeploymentConfigRemoveDuplicateEnvVarMutator(nil, existing) + existing := dFactory(tc.existingEnvs) + update, err := DeploymentRemoveDuplicateEnvVarMutator(nil, existing) if err != nil { subT.Fatal(err) } @@ -846,9 +664,9 @@ func TestDeploymentConfigRemoveDuplicateEnvVarMutator(t *testing.T) { } } -func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { +func TestDeploymentTopologySpreadConstraintsMutator(t *testing.T) { testTopologySpreadConstraint1 := []corev1.TopologySpreadConstraint{ - corev1.TopologySpreadConstraint{ + { TopologyKey: "topologyKey1", WhenUnsatisfiable: "DoNotSchedule", LabelSelector: &metav1.LabelSelector{ @@ -857,7 +675,7 @@ func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { }, } testTopologySpreadConstraint2 := []corev1.TopologySpreadConstraint{ - corev1.TopologySpreadConstraint{ + { TopologyKey: "topologyKey2", WhenUnsatisfiable: "ScheduleAnyway", LabelSelector: &metav1.LabelSelector{ @@ -866,18 +684,18 @@ func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { }, } - dcFactory := func(topologySpreadConstraint []corev1.TopologySpreadConstraint) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ + dFactory := func(topologySpreadConstraint []corev1.TopologySpreadConstraint) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ TopologySpreadConstraints: topologySpreadConstraint, }, @@ -899,9 +717,9 @@ func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTopologySpreadConstraints) - desired := dcFactory(tc.desiredTopologySpreadConstraints) - update, err := DeploymentConfigTopologySpreadConstraintsMutator(desired, existing) + existing := dFactory(tc.existingTopologySpreadConstraints) + desired := dFactory(tc.desiredTopologySpreadConstraints) + update, err := DeploymentTopologySpreadConstraintsMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -916,19 +734,19 @@ func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { } -func TestDeploymentConfigPodTemplateAnnotationsMutator(t *testing.T) { - dcFactory := func(annotations map[string]string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func TestDeploymentPodTemplateAnnotationsMutator(t *testing.T) { + dFactory := func(annotations map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: "Deployment", APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", + Name: "myDeployment", Namespace: "myNS", }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, }, @@ -967,9 +785,9 @@ func TestDeploymentConfigPodTemplateAnnotationsMutator(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingAnnotations) - desired := dcFactory(tc.desiredAnnotations) - update, err := DeploymentConfigPodTemplateAnnotationsMutator(desired, existing) + existing := dFactory(tc.existingAnnotations) + desired := dFactory(tc.desiredAnnotations) + update, err := DeploymentPodTemplateAnnotationsMutator(desired, existing) if err != nil { subT.Fatal(err) } @@ -983,10 +801,10 @@ func TestDeploymentConfigPodTemplateAnnotationsMutator(t *testing.T) { } } -func TestDeploymentConfigArgsMutator(t *testing.T) { +func TestDeploymentArgsMutator(t *testing.T) { type args struct { - desired *appsv1.DeploymentConfig - existing *appsv1.DeploymentConfig + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment } tests := []struct { name string @@ -997,9 +815,9 @@ func TestDeploymentConfigArgsMutator(t *testing.T) { { name: "No Args Update Required", args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Args: []string{"testArg"}}, @@ -1008,9 +826,9 @@ func TestDeploymentConfigArgsMutator(t *testing.T) { }, }, }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Args: []string{"testArg"}}, @@ -1026,9 +844,9 @@ func TestDeploymentConfigArgsMutator(t *testing.T) { { name: "Args Update Required", args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Args: []string{"testArg1", "testArg2"}}, @@ -1037,9 +855,9 @@ func TestDeploymentConfigArgsMutator(t *testing.T) { }, }, }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Args: []string{"testArg1"}}, @@ -1054,22 +872,22 @@ func TestDeploymentConfigArgsMutator(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentConfigArgsMutator(tt.args.desired, tt.args.existing) + got, err := DeploymentArgsMutator(tt.args.desired, tt.args.existing) if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigArgsMutator() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DeploymentArgsMutator() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("DeploymentConfigArgsMutator() = %v, want %v", got, tt.want) + t.Errorf("DeploymentArgsMutator() = %v, want %v", got, tt.want) } }) } } -func TestDeploymentConfigProbesMutator(t *testing.T) { +func TestDeploymentProbesMutator(t *testing.T) { type args struct { - desired *appsv1.DeploymentConfig - existing *appsv1.DeploymentConfig + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment } tests := []struct { name string @@ -1078,106 +896,106 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { wantErr bool }{ { - name: "Liveness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ + name: "Liveness Probe Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ Port: intstr.FromInt(9306), }, }, InitialDelaySeconds: 60, PeriodSeconds: 10, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, { - name: "Liveness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ + name: "Liveness Probe Not Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ Port: intstr.FromInt(9306), }, }, InitialDelaySeconds: 60, PeriodSeconds: 10, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ Port: intstr.FromInt(9306), }, }, InitialDelaySeconds: 60, PeriodSeconds: 10, - }, + }, }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, { - name: "Readiness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ + name: "Readiness Probe Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ Type: intstr.Type(intstr.Int), @@ -1186,40 +1004,40 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 30, TimeoutSeconds: 5, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, { - name: "Readiness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ + name: "Readiness Probe Not Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ Type: intstr.Type(intstr.Int), @@ -1228,20 +1046,20 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 30, TimeoutSeconds: 5, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ Type: intstr.Type(intstr.Int), @@ -1249,27 +1067,254 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { }, InitialDelaySeconds: 30, TimeoutSeconds: 5, - }, + }, }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentProbesMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentProbesMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentProbesMutator() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeploymentAnnotationsMutator(t *testing.T) { + dFactory := func(annotations map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + Annotations: annotations, + }, + } + } + + mapCopy := func(originalMap map[string]string) map[string]string { + // Create the target map + targetMap := make(map[string]string) + + // Copy from the original map to the target map + for key, value := range originalMap { + targetMap[key] = value + } + + return targetMap + } + + annotationsA := map[string]string{"a": "1", "a2": "2"} + annotationsB := map[string]string{"a": "other", "b": "1"} + + cases := []struct { + testName string + existingAnnotations map[string]string + desiredAnnotations map[string]string + expectedResult bool + expectedNewAnnotations map[string]string + }{ + {"NothingToReconcile", mapCopy(annotationsA), mapCopy(annotationsA), false, mapCopy(annotationsA)}, + {"AnnotationsReconciled", mapCopy(annotationsB), mapCopy(annotationsA), true, map[string]string{ + "a": "1", "a2": "2", "b": "1", + }}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingAnnotations) + desired := dFactory(tc.desiredAnnotations) + update, err := DeploymentAnnotationsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.ObjectMeta.Annotations, tc.expectedNewAnnotations) { + subT.Fatal(cmp.Diff(existing.ObjectMeta.Annotations, tc.expectedNewAnnotations)) + } + }) + } +} + +func TestDeploymentPodContainerImageMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "No Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image-desired"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image-existing"}, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentPodContainerImageMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentPodContainerImageMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentPodContainerImageMutator() = %v, want %v", got, tt.want) + } + }) } +} + +func TestDeploymentPodInitContainerImageMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "No Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Image: "test-image-desired"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Image: "test-image-existing"}, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentConfigProbesMutator(tt.args.desired, tt.args.existing) + got, err := DeploymentPodInitContainerImageMutator(tt.args.desired, tt.args.existing) if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigProbesMutator() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DeploymentPodInitContainerImageMutator() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("DeploymentConfigProbesMutator() = %v, want %v", got, tt.want) + t.Errorf("DeploymentPodInitContainerImageMutator() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/reconcilers/imagestreams.go b/pkg/reconcilers/imagestreams.go deleted file mode 100644 index 0fb721ec5..000000000 --- a/pkg/reconcilers/imagestreams.go +++ /dev/null @@ -1,66 +0,0 @@ -package reconcilers - -import ( - "fmt" - "reflect" - - "github.com/3scale/3scale-operator/pkg/common" - imagev1 "github.com/openshift/api/image/v1" -) - -func GenericImageStreamMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { - existing, ok := existingObj.(*imagev1.ImageStream) - if !ok { - return false, fmt.Errorf("%T is not a *imagev1.ImageStream", existingObj) - } - desired, ok := desiredObj.(*imagev1.ImageStream) - if !ok { - return false, fmt.Errorf("%T is not a *imagev1.ImageStream", desiredObj) - } - - // merging approach will be implemented - // spec.tags tagrefences in desired must exist in existing. - // If element does not exist, append - // If exists (by name), ensure From field and ImportPolicy fields are equal. - updated := false - - findTagReference := func(tagRefName string, tagRefS []imagev1.TagReference) int { - for i := range tagRefS { - if tagRefS[i].Name == tagRefName { - return i - } - } - return -1 - } - - for idx := range desired.Spec.Tags { - if existingIdx := findTagReference(desired.Spec.Tags[idx].Name, existing.Spec.Tags); existingIdx < 0 { - // does not exist, append - existing.Spec.Tags = append(existing.Spec.Tags, desired.Spec.Tags[idx]) - updated = true - } else { - // exists, reconcile - tmpUpdated := imageStreamReconcile(existing.Spec.Tags, existingIdx, desired.Spec.Tags, idx) - updated = updated || tmpUpdated - } - } - - return updated, nil -} - -func imageStreamReconcile(existingTags []imagev1.TagReference, existingIdx int, desiredTags []imagev1.TagReference, desiredIdx int) bool { - // From and ImportPolicy fields are equal. - updated := false - - if !reflect.DeepEqual(existingTags[existingIdx].From, desiredTags[desiredIdx].From) { - existingTags[existingIdx].From = desiredTags[desiredIdx].From - updated = true - } - - if !reflect.DeepEqual(existingTags[existingIdx].ImportPolicy, desiredTags[desiredIdx].ImportPolicy) { - existingTags[existingIdx].ImportPolicy = desiredTags[desiredIdx].ImportPolicy - updated = true - } - - return updated -} diff --git a/pkg/reconcilers/imagestreams_test.go b/pkg/reconcilers/imagestreams_test.go deleted file mode 100644 index 3926518eb..000000000 --- a/pkg/reconcilers/imagestreams_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package reconcilers - -import ( - "testing" - - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestGenericImageStreamMutator(t *testing.T) { - existing := &imagev1.ImageStream{ - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - ObjectMeta: metav1.ObjectMeta{Name: "myIS", Namespace: "MyNS"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: "tag0", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-0"}, - }, - imagev1.TagReference{ - Name: "tag1", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-1"}, - }, - }, - }, - } - - desired := &imagev1.ImageStream{ - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - ObjectMeta: metav1.ObjectMeta{Name: "myIS", Namespace: "MyNS"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - // tag that should be updated - Name: "tag1", - ImportPolicy: imagev1.TagImportPolicy{Insecure: true, Scheduled: false}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-1other"}, - }, - imagev1.TagReference{ - // tag that should be added - Name: "tag2", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-2"}, - }, - }, - }, - } - - update, err := GenericImageStreamMutator(existing, desired) - if err != nil { - t.Fatal(err) - } - if !update { - t.Fatal("when defaults can be applied, reconciler reported no update needed") - } - - if len(existing.Spec.Tags) != 3 { - t.Fatalf("reconciled obj does not have expected number of tags. Expected: 3, got: %d", len(existing.Spec.Tags)) - } - - findTagReference := func(tagRefName string, tagRefS []imagev1.TagReference) int { - for i := range tagRefS { - if tagRefS[i].Name == tagRefName { - return i - } - } - return -1 - } - - // tag0 existed previously in obj, should be left untouched - tag0Index := findTagReference("tag0", existing.Spec.Tags) - if tag0Index < 0 { - t.Fatal("reconciled obj does not have tag0") - } - - // tag1 existed previously in obj, should be updated - tag1Index := findTagReference("tag1", existing.Spec.Tags) - if tag1Index < 0 { - t.Fatal("reconciled obj does not have tag1") - } - tag1 := existing.Spec.Tags[1] - // From and ImportPolicy fields should have been reconciled - if tag1.From.Name != "3scale-1other" { - t.Fatal("reconciled obj tag1 'from' was not reconciled") - } - - if !tag1.ImportPolicy.Insecure { - t.Fatal("reconciled obj tag1 'impoortpolicy.insecure' was not reconciled") - } - - if tag1.ImportPolicy.Scheduled { - t.Fatal("reconciled obj tag1 'impoortpolicy.scheduled' was not reconciled") - } - - // tag2 did not exist previously in obj, should be appended - tag2Index := findTagReference("tag2", existing.Spec.Tags) - if tag2Index < 0 { - t.Fatal("reconciled obj does not have tag2") - } -} diff --git a/pkg/reconcilers/role.go b/pkg/reconcilers/role.go new file mode 100644 index 000000000..d4ed5bf0c --- /dev/null +++ b/pkg/reconcilers/role.go @@ -0,0 +1,30 @@ +package reconcilers + +import ( + "fmt" + "reflect" + + "github.com/3scale/3scale-operator/pkg/common" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func RoleRuleMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*rbacv1.Role) + if !ok { + return false, fmt.Errorf("%T is not a *rbacv1.Role", existingObj) + } + desired, ok := desiredObj.(*rbacv1.Role) + if !ok { + return false, fmt.Errorf("%T is not a *rbacv1.Role", desiredObj) + } + + updated := false + + if !reflect.DeepEqual(existing.Rules, desired.Rules) { + existing.Rules = desired.Rules + updated = true + } + + return updated, nil +} diff --git a/pkg/reconcilers/role_test.go b/pkg/reconcilers/role_test.go new file mode 100644 index 000000000..ea3b22fa3 --- /dev/null +++ b/pkg/reconcilers/role_test.go @@ -0,0 +1,83 @@ +package reconcilers + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestRoleRuleMutator(t *testing.T) { + + testRules1 := []rbacv1.PolicyRule{ + { + APIGroups: []string{"apps"}, + Resources: []string{ + "deployment", + }, + Verbs: []string{ + "get", + "list", + }, + }, + } + + testRules2 := []rbacv1.PolicyRule{ + { + APIGroups: []string{"apps"}, + Resources: []string{ + "deployment", + }, + Verbs: []string{ + "get", + "list", + "create", + "delete", + }, + }, + } + + roleFactory := func(rules []rbacv1.PolicyRule) *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + Kind: "Role", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testRole", + Namespace: "testNamespace", + }, + Rules: rules, + } + } + + cases := []struct { + testName string + existingRules []rbacv1.PolicyRule + desiredRules []rbacv1.PolicyRule + expectedResult bool + }{ + {"NothingToReconcile", nil, nil, false}, + {"EqualRules", testRules1, testRules1, false}, + {"DifferentRules", testRules1, testRules2, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := roleFactory(tc.existingRules) + desired := roleFactory(tc.desiredRules) + update, err := RoleRuleMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Rules, desired.Rules) { + subT.Fatal(cmp.Diff(existing.Rules, desired.Rules)) + } + }) + } +} diff --git a/pkg/reconcilers/secret.go b/pkg/reconcilers/secret.go index fb17c8990..d68dba497 100644 --- a/pkg/reconcilers/secret.go +++ b/pkg/reconcilers/secret.go @@ -9,7 +9,7 @@ import ( ) // DefaultsOnlySecretMutator is useful for secrets pre-created by the user and when not all the fields are created. -// Fields referenced from deployment configs must exist, +// Fields referenced from deployments must exist, // so defaults only reconciliation makes sure they exist with default values when user does doe pre-create them func DefaultsOnlySecretMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { existing, ok := existingObj.(*v1.Secret) diff --git a/pkg/reconcilers/service.go b/pkg/reconcilers/service.go index 9cb12b8d9..68a33e361 100644 --- a/pkg/reconcilers/service.go +++ b/pkg/reconcilers/service.go @@ -5,9 +5,36 @@ import ( "reflect" "github.com/3scale/3scale-operator/pkg/common" + v1 "k8s.io/api/core/v1" ) +func ServiceMutator(opts ...MutateFn) MutateFn { + return func(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", existingObj) + } + desired, ok := desiredObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", desiredObj) + } + + update := false + + // Loop through each option + for _, opt := range opts { + tmpUpdate, err := opt(existing, desired) + if err != nil { + return false, err + } + update = update || tmpUpdate + } + + return update, nil + } +} + func ServicePortMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { existing, ok := existingObj.(*v1.Service) if !ok { @@ -27,3 +54,23 @@ func ServicePortMutator(existingObj, desiredObj common.KubernetesObject) (bool, return updated, nil } + +func ServiceSelectorMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", existingObj) + } + desired, ok := desiredObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", desiredObj) + } + + updated := false + + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + updated = true + existing.Spec.Selector = desired.Spec.Selector + } + + return updated, nil +} diff --git a/pkg/restore/apimanager_restore.go b/pkg/restore/apimanager_restore.go index 68e7849ba..603e78246 100644 --- a/pkg/restore/apimanager_restore.go +++ b/pkg/restore/apimanager_restore.go @@ -395,13 +395,13 @@ func (b *APIManagerRestore) restoreSecretsAndConfigMapsContainerArgs() string { func (b *APIManagerRestore) zyncResyncDomainsContainerArgs() string { return ` - dcname="system-sidekiq" - dcpods=$(oc get pods --ignore-not-found=true -l deploymentconfig=${dcname} --no-headers=true -o custom-columns=:metadata.name) - if [ -z "${dcpods}" ]; then - echo "No pods found for Deployment ${dcname}" + dname="system-sidekiq" + dpods=$(oc get pods --ignore-not-found=true -l deployment=${dname} --no-headers=true -o custom-columns=:metadata.name) + if [ -z "${dpods}" ]; then + echo "No pods found for Deployment ${dname}" exit 1 fi - podname=$(echo -n $dcpods | awk '{print $1}') + podname=$(echo -n $dpods | awk '{print $1}') oc exec ${podname} -- bash -c "bundle exec rake zync:resync:domains" ` } diff --git a/pkg/upgrade/delete_image_streams_2.15.go b/pkg/upgrade/delete_image_streams_2.15.go new file mode 100644 index 000000000..939ecc5c2 --- /dev/null +++ b/pkg/upgrade/delete_image_streams_2.15.go @@ -0,0 +1,61 @@ +package upgrade + +import ( + "context" + "fmt" + + imagev1 "github.com/openshift/api/image/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// DeleteImageStreams deletes the ImageStream objects managed by APIManager as they are no longer needed with Deployments +// 3scale 2.14 -> 2.15 +func DeleteImageStreams(namespace string, client k8sclient.Client) error { + var imageStreams = []string{ + "amp-apicast", + "amp-backend", + "amp-system", + "amp-zync", + "backend-redis", + "system-memcached", + "system-mysql", + "system-redis", + "system-searchd", + "zync-database-postgresql", + } + + // First check if any ImageStreams exist, if they've already been deleted then break out + imageStreamList := &imagev1.ImageStreamList{} + listOps := []k8sclient.ListOption{ + k8sclient.InNamespace(namespace), + } + err := client.List(context.TODO(), imageStreamList, listOps...) + if err != nil { + return fmt.Errorf("failed to list ImageStreams: %v", err) + } + if len(imageStreamList.Items) == 0 { + return nil + } + + // Delete the specified ImageStreams + for _, imageStreamName := range imageStreams { + imageStream := &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageStreamName, + Namespace: namespace, + }, + } + + err := client.Delete(context.TODO(), imageStream) + if err != nil { + if !k8serr.IsNotFound(err) { + return fmt.Errorf("error deleting ImageStream %s: %v", imageStream.Name, err) + } + } + + } + + return nil +} diff --git a/pkg/upgrade/migrate_deployment_config_2.15.go b/pkg/upgrade/migrate_deployment_config_2.15.go new file mode 100644 index 000000000..c6a7ac977 --- /dev/null +++ b/pkg/upgrade/migrate_deployment_config_2.15.go @@ -0,0 +1,111 @@ +package upgrade + +import ( + "context" + "fmt" + + appsv1 "github.com/openshift/api/apps/v1" + routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/helper" +) + +// MigrateDeploymentConfigToDeployment verifies the Deployment is healthy and then deletes the corresponding DeploymentConfig +// 3scale 2.14 -> 2.15 +func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overrideDeploymentHealth bool, client k8sclient.Client, scheme *runtime.Scheme) (bool, error) { + deploymentConfig := &appsv1.DeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: dName, + Namespace: dNamespace, + }, + } + err := client.Get(context.TODO(), k8sclient.ObjectKey{Name: deploymentConfig.Name, Namespace: deploymentConfig.Namespace}, deploymentConfig) + + // Breakout if the DeploymentConfig has already been deleted + if k8serr.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("error getting deploymentconfig %s: %v", deploymentConfig.Name, err) + } + + // Check Deployment health + deployment := &k8sappsv1.Deployment{} + err = client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: dNamespace, + Name: dName, + }, deployment) + + // Return error if can't get Deployment + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) + } + + // Requeue if Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + log.V(1).Info(fmt.Sprintf("deployment %s does not exist", deployment.Name)) + return false, nil + } + + // Requeue if Deployment isn't healthy and override is set to false, otherwise proceed + if !helper.IsDeploymentAvailable(deployment) && !overrideDeploymentHealth { + log.V(1).Info(fmt.Sprintf("deployment %s is not yet available and overrideDeploymentHealth is set to %t", deployment.Name, overrideDeploymentHealth)) + return false, nil + } + + // Transfer zync-que route ownership + if deployment.Name == component.ZyncQueDeploymentName && scheme != nil { + err = transferZyncRoutesOwnership(deployment, client, scheme) + if err != nil { + return false, err + } + } + + // Delete the DeploymentConfig because the Deployment replacing it is healthy or overrideDeploymentHealth is set to true + err = client.Delete(context.TODO(), deploymentConfig) + if err != nil { + if !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error deleting deploymentconfig %s: %v", deploymentConfig.Name, err) + } + } + + log.Info(fmt.Sprintf("%s Deployment has replaced its corresponding DeploymentConfig", deployment.Name)) + return true, nil +} + +func transferZyncRoutesOwnership(deployment *k8sappsv1.Deployment, client k8sclient.Client, scheme *runtime.Scheme) error { + listOps := []k8sclient.ListOption{ + k8sclient.InNamespace(deployment.Namespace), + } + + routeList := &routev1.RouteList{} + err := client.List(context.TODO(), routeList, listOps...) + if err != nil { + return fmt.Errorf("failed to list routes to transfer ownership: %w", err) + } + + for _, rt := range routeList.Items { + for _, ownerRef := range rt.ObjectMeta.OwnerReferences { + if ownerRef.Name == component.ZyncQueDeploymentName { + err = controllerutil.SetOwnerReference(deployment, &rt, scheme) + if err != nil { + return fmt.Errorf("failed to set ownerRef for route %v : %w", rt, err) + } + + err = client.Update(context.TODO(), &rt) + if err != nil { + return fmt.Errorf("failed to update ownerRef for route %v : %w", rt, err) + } + } + } + } + + return nil +} diff --git a/pkg/upgrade/redis_2.14.go b/pkg/upgrade/redis_2.14.go deleted file mode 100644 index f0429bfa2..000000000 --- a/pkg/upgrade/redis_2.14.go +++ /dev/null @@ -1,48 +0,0 @@ -package upgrade - -import ( - "fmt" - "reflect" - - "github.com/3scale/3scale-operator/pkg/common" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" -) - -// Redis6CommandArgsEnv reconciles environment variables, command and args -func Redis6CommandArgsEnv(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var updated bool - - desiredName := common.ObjectInfo(desired) - - if len(desired.Spec.Template.Spec.Containers) != 1 { - return false, fmt.Errorf("%s desired spec.template.spec.containers length changed to '%d', should be 1", desiredName, len(desired.Spec.Template.Spec.Containers)) - } - - if len(existing.Spec.Template.Spec.Containers) != 1 { - log.Info(fmt.Sprintf("%s spec.template.spec.containers length changed to '%d', recreating dc", desiredName, len(existing.Spec.Template.Spec.Containers))) - existing.Spec.Template.Spec.Containers = desired.Spec.Template.Spec.Containers - updated = true - } - - // Env Vars added in 2.14 - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Template.Spec.Containers[0].Env, - &existing.Spec.Template.Spec.Containers[0].Env, - "REDIS_CONF") - updated = updated || tmpChanged - - // Command updated in 2.14 - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Command, desired.Spec.Template.Spec.Containers[0].Command) { - existing.Spec.Template.Spec.Containers[0].Command = desired.Spec.Template.Spec.Containers[0].Command - updated = true - } - - // Args updated in 2.14 - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Args, desired.Spec.Template.Spec.Containers[0].Args) { - existing.Spec.Template.Spec.Containers[0].Args = desired.Spec.Template.Spec.Containers[0].Args - updated = true - } - - return updated, nil -} diff --git a/pkg/upgrade/upgrade_sphinx_reference_2.14.go b/pkg/upgrade/upgrade_sphinx_reference_2.14.go deleted file mode 100644 index a159aabfe..000000000 --- a/pkg/upgrade/upgrade_sphinx_reference_2.14.go +++ /dev/null @@ -1,19 +0,0 @@ -package upgrade - -import ( - appsv1 "github.com/openshift/api/apps/v1" - - "github.com/3scale/3scale-operator/pkg/reconcilers" -) - -// DeploymentConfigPodTemplateLabelsMutator ensures pod template labels are reconciled -func SphinxAddressReference(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "THINKING_SPHINX_ADDRESS") - - // Not in desired, it should be removed from existing - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "THINKING_SPHINX_CONFIGURATION_FILE") - changed = changed || tmpChanged - - return changed, nil -} diff --git a/pkg/upgrade/upgrade_system_backend_urls_2.14.go b/pkg/upgrade/upgrade_system_backend_urls_2.14.go deleted file mode 100644 index 07974662b..000000000 --- a/pkg/upgrade/upgrade_system_backend_urls_2.14.go +++ /dev/null @@ -1,30 +0,0 @@ -package upgrade - -import ( - appsv1 "github.com/openshift/api/apps/v1" - - "github.com/3scale/3scale-operator/pkg/reconcilers" -) - -// SystemBackendUrls reconciles environment variables for Backend URLs on system DC -func SystemBackendUrls(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var changed bool - - // Remove old env var - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_BACKEND_ROOT_ENDPOINT") - changed = changed || tmpChanged - - // Remove old env var - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_ROUTE") - changed = changed || tmpChanged - - // Add new env vars - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_URL") - changed = changed || tmpChanged - - // Add new env vars - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_PUBLIC_URL") - changed = changed || tmpChanged - - return changed, nil -}