Merge pull request #306 from anynines/develop

Merge Develop into Main for Release v1.1.0

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to the a9s Dataservices on Kubernetes will be documented
 here, the format is based on [Keep a
 Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.1.0] - 2024-06-05
+
+### Added
+
+* Added support for installing/uninstalling PostgreSQL extensions via the extensions field,
+  including postgis, uuid-ossp, ltree, pgcrypto, pg_stat_statements, fuzzystrmatch, pg_trgm, and
+  hstore.
+
+### Fixed
+
+* Fix PostgreSQL restart logic after user parameter updates.
+
+### Updated
+
+* The backup_agent has been bumped to the latest version and now includes buildx support.
+
 ## [1.0.0] - 2024-03-15
 
 ### Updated

CODEOWNERS

@@ -1 +1 @@
-* @abdulhaseeb3 @mkfdoherty @lhaendler
+* @abdulhaseeb3 @mkfdoherty @lhaendler @iliasmavridis

deploy/a8s/manifests/a8s-system-namespace.yaml

@@ -1,5 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: a8s-system
-
+  labels:
+    control-plane: controller-manager
+  name: a8s-system

deploy/a8s/manifests/postgresql-images.yaml

@@ -7,5 +7,4 @@ data:
 kind: ConfigMap
 metadata:
   name: postgresql-images
-  namespace: a8s-system
-
+  namespace: a8s-system

docs/application-developers/advanced_configuration.md

@@ -122,7 +122,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       podAntiAffinity:
@@ -248,7 +248,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       podAntiAffinity:
@@ -389,7 +389,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       nodeAffinity:

docs/application-developers/api-documentation/postgresql-operator/v1beta3.md

@@ -95,7 +95,7 @@ _Appears in:_
 | `volumeSize` _Quantity_ | VolumeSize sets the size of the persistent volume of the PostgreSQL instance, the minimum size is 0.5Gi. The size is to be specified as a plain integer or as a fixed-point number using one of these suffixes: E, P, T, G, M, K, corresponding to kilo-, mega-, gigabytes, etc. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki, corresponding to kibi-, mebi-, gibibytes, etc. For example  a value of "0.5Gi" corresponds to an instance with a persistent volume of 0.5 gibibytes. |
 | `parameters` _[PostgresqlParameters](#postgresqlparameters)_ |  |
 | `schedulingConstraints` _[PostgresqlSchedulingConstraints](#postgresqlschedulingconstraints)_ | SchedulingConstraints contains subfields that affect how the Pods of the Postgresql instance will be scheduled onto Kubernetes cluster nodes. The subfields map directly to Kubernetes API primitives such as node taints, tolerations, affinity and (anti)affinity. See the documentation of each subfield for more details. Together, the subfields of SchedulingConstraints allow you to express constraints such as "Pods of this Postgresql instance MUST be scheduled to different availability zones", or "Pods of this Postgresql instance SHOULD preferably (but not mandatorily) be scheduled to nodes that have a SSD", and many more. As a warning, the subfields of SchedulingConstraints can interfere with each other, so when you set one of them you should consider how it will interact with the values that you set for other subfields. |
-| `extensions` _string array_ | Extensions defines a list of PostgreSQL extensions which should be installed. Installing means that the binaries and libraries of the defined extensions are moved to the PostgreSQL extension directory. The extensions are NOT loaded by default (i.e. by using the PostgreSQL "CREATE EXTENSION" command). Updating the list of extensions will cause a rolling update of the PostgreSQL instance. |
+| `extensions` _string array_ | Extensions defines a list of PostgreSQL extensions which should be installed. Extensions can be installed from the supported list of extensions. The currently supported extensions are: postgis, uuid-ossp, ltree, pgcrypto, pg_stat_statements, fuzzystrmatch, pg_trgm, hstore. |
 | `enableReadOnlyService` _boolean_ | EnableReadOnlyService enables the creation of a read-only service. A read-only service is meant for read operations and allows for load-balancing across different PostgreSQL cluster members. Stale reads are possible when reading from this service. By default this feature is disabled. |
 | `readOnlyTargetNodes` _string_ | ReadOnlyTargetNodes specifies the target PostgreSQL cluster members which the read-only service points to. "replicas" means that only the PostgreSQL cluster replicas are used for the read-only service. "all" means that the read-only service points to all PostgreSQL cluster members. By default this feature only points to the PostgreSQL cluster replicas. |
 | `expose` _[ExposeOption](#exposeoption)_ | Expose determines where (and how) the DSI can be accessed from. Currently supported values are - "Internal": the DSI will be accessible only from inside the K8s cluster - "LoadBalancer": the DSI will receive dedicated load balancers with reachable IP addresses 	  that can be used from external locations . This is only supported on K8s clusters that    support external load  balancers. This field applies to all the services backed by the DSI, that is, both the master-only one and the read-only one; the "LoadBalancer" value means that each service will get a dedicated LoadBalancer. |

docs/application-developers/usage_overview.md

@@ -233,6 +233,16 @@ When you want to delete a `Restore`, run:
 kubectl delete restore <restore-name>
 ```
 
+## Install/Uninstall PostgreSQL Extensions
+
+PostgreSQL is designed to be easily extensible. For this reason, extensions loaded into the database can function just like features that are built in. Application developers can enable extensions by adding the desired extensions to the extensions field as a list. Likewise, extensions can be uninstalled by removing them from the extensions list. Extensions can be installed on new instances as well as on running instances.
+
+WARNING: Extensions cannot be removed if they have dependencies on other objects in the database. If you are using features of an extension in your database tables, the removal operation will not succeed. Users must remove these blocking dependencies for the extension to be cleanly removed from the database. The PostgreSQL operator will not forcibly remove extensions with dependencies to avoid unexpected data loss.
+
+The extensions currently supported are: postgis, uuid-ossp, ltree, pgcrypto, pg_stat_statements, fuzzystrmatch, pg_trgm, hstore.
+
+See [examples/postgresql-extensions-instance.yaml](/examples/postgresql-extensions-instance.yaml) for an example manifest containing a list of desired extensions to be installed.
+
 ## Visualize the Logs of the PostgreSQL Instance
 
 Application developers should be aware that all pods with the label field `app`

docs/current_limitations.md

@@ -59,6 +59,10 @@
 
   to force the deletion.
 
+## PostgreSQL Extensions
+
+- Status on extension state, required for informing user of unresolvable dependencies on extensions that block their removal.
+
 ## Service Bindings
 
 - Custom parameters for configuring the permissions are not supported.

examples/postgresql-extensions-instance.yaml

@@ -0,0 +1,22 @@
+apiVersion: postgresql.anynines.com/v1beta3
+kind: Postgresql
+metadata:
+  name: sample-pg-cluster
+spec:
+  extensions:
+    - postgis
+    - uuid-ossp 
+    - ltree 
+    - pgcrypto
+    - pg_stat_statements
+    - fuzzystrmatch
+    - pg_trgm
+    - hstore
+  replicas: 3
+  volumeSize: 1Gi
+  version: 14
+  resources:
+    requests:
+      cpu: 100m
+    limits:
+      memory: 200Mi

examples/postgresql-ha-1-instance.yaml

@@ -9,7 +9,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       podAntiAffinity:

examples/postgresql-ha-2-instance.yaml

@@ -9,7 +9,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       podAntiAffinity:

examples/postgresql-instance-exposed.yaml

@@ -11,4 +11,4 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi

examples/postgresql-instance.yaml

@@ -10,4 +10,4 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi

examples/postgresql-toleration-instance.yaml

@@ -9,7 +9,7 @@ spec:
     requests:
       cpu: 100m
     limits:
-      memory: 100Mi
+      memory: 200Mi
   schedulingConstraints:
     affinity:
       nodeAffinity:

test/e2e/patroni/patroni_test.go

@@ -94,7 +94,7 @@ var _ = Describe("Patroni end-to-end Tests", func() {
 				defaultTempFileLimitKiloBytes = -1
 				defaultTrackIOTiming          = "off"
 				defaultWalWriterDelayMillis   = 200 // Needs ms
-				defaultMaxLocksPerTransaction = 64
+				defaultMaxLocksPerTransaction = 100
 			)
 
 			By("creating a PostgreSQL instance with implicit defaults", func() {
@@ -391,10 +391,26 @@ var _ = Describe("Patroni end-to-end Tests", func() {
 					// still in the process of restarting due to parameters that
 					// require restart.
 					Eventually(func() error {
-						return client.CheckParameter(
+						portForwardStopCh, localPort, err = framework.PortForward(
+							ctx, instancePort, kubeconfigPath, instance, k8sClient)
+						Expect(err).To(BeNil(),
+							fmt.Sprintf("failed to establish portforward to DSI %s/%s",
+								instance.GetNamespace(), instance.GetName()))
+
+						// Create client for interacting with the new instance.
+						client, err = dsi.NewClient(
+							dataservice, strconv.Itoa(localPort), adminSecretData)
+						Expect(err).To(BeNil(), "failed to create new dsi client")
+
+						probeErr := client.CheckParameter(
 							ctx,
 							setting.parameter,
 							setting.value)
+						if probeErr != nil {
+							close(portForwardStopCh)
+							return probeErr
+						}
+						return probeErr
 					}, framework.AsyncOpsTimeoutMins, 1*time.Second).Should(Succeed(),
 						fmt.Sprintf("unable to check custom config is set correctly on update for %s/%s",
 							instance.GetNamespace(), instance.GetName()))
@@ -421,7 +437,9 @@ var _ = Describe("Patroni end-to-end Tests", func() {
 					fmt.Sprintf("failed to list events emitted for the config update of %s/%s",
 						instance.GetNamespace(), instance.GetName()))
 
-				Expect(len(events.Items)).To(Equal(1), "exactly one event should be emitted for "+
+				// We require two events: one to update the PostgreSQL configuration, and another
+				// to confirm a successful restart following the configuration update.
+				Expect(len(events.Items)).To(Equal(2), "exactly two events should be emitted for "+
 					"the config update of a DSI, found more than one")
 
 				event := events.Items[0]

test/e2e/postgresql/exposed_cluster_test.go

@@ -22,7 +22,6 @@ const (
 )
 
 var _ = Describe("end-to-end tests for exposed instances", Label("ExternalLoadbalancer", "KindIncompatible"), func() {
-
 	Context("Instance exposed via Load Balancer", Ordered, func() {
 		AfterAll(func() {
 			Expect(k8sClient.Delete(ctx, instance.GetClientObject())).To(
@@ -82,15 +81,18 @@ var _ = Describe("end-to-end tests for exposed instances", Label("ExternalLoadba
 			Expect(err).To(BeNil(),
 				fmt.Sprintf("failed to parse secret data for service binding %s/%s",
 					sb.GetNamespace(), sb.GetName()))
-
 		})
 
 		var connInfo v1.ConfigMap
 		It("Creates a connection ConfigMap", func() {
-			Expect(k8sClient.Get(ctx, types.NamespacedName{
-				Namespace: instance.GetNamespace(),
-				Name:      dsi.ConnectionInfoName(instance.GetName()),
-			}, &connInfo)).To(Succeed())
+			Eventually(func() error {
+				return k8sClient.Get(ctx, types.NamespacedName{
+					Namespace: instance.GetNamespace(),
+					Name:      dsi.ConnectionInfoName(instance.GetName()),
+				},
+					&connInfo,
+				)
+			}, asyncOpsTimeoutMins, 1*time.Second).Should(Succeed())
 
 			Expect(connInfo.Data).To(HaveKey("primary"),
 				"connInfo does not contain information about primary database")
@@ -100,7 +102,6 @@ var _ = Describe("end-to-end tests for exposed instances", Label("ExternalLoadba
 
 			Expect(connInfo.Data).To(HaveKey("readOnly"),
 				"connInfo does not contain information about read only service")
-
 		})
 
 		It("can write data via public address", func() {