Merge pull request #1529 from Altinity/0.24.0

.dockerignore

@@ -1,10 +1,14 @@
 docs/
-.vagrant/
-.idea/
-tests/
 hack/
-*.log
-./tests/image/cache/
+tests/
 .devspace/
+.vagrant/
+.idea/
+
 devspace.yaml
 Vagrantfile
+
+config/config-dev.yaml
+config/secret.yaml
+
+*.log

.gitattributes

@@ -0,0 +1 @@
+text eol=lf

.github/workflows/build_branch.yaml

@@ -6,16 +6,16 @@ on:
       - "[0-9]+.[0-9]+.[0-9]+"
 
 jobs:
-  build_master:
+  build_version:
     name: Build branch
     runs-on: ubuntu-latest
     steps:
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Build and push Docker images
         env:
           DOCKER_ORG: ${{ secrets.DOCKER_ORG }}

.github/workflows/build_master.yaml

@@ -11,11 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Build and push Docker images
         env:
           DOCKER_ORG: ${{ secrets.DOCKER_ORG }}

.github/workflows/release_chart.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install chart-releaser

.github/workflows/run_tests.yaml

@@ -70,6 +70,11 @@ jobs:
         id: run-tests
         continue-on-error: true
         run: |
+          echo "Test run settings:"
+          echo "  test mode: ${{ github.event.inputs.test_mode }}"
+          echo "  test mask: ${{ github.event.inputs.test_mask }}"
+          echo
+        
           source ~/venv/qa/bin/activate
           set -x
           set +e # disable the "exit on failure"

cmd/metrics_exporter/app/metrics_exporter.go

@@ -18,14 +18,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/altinity/clickhouse-operator/pkg/metrics/clickhouse"
 	"os"
 	"os/signal"
 	"syscall"
 
 	log "github.com/golang/glog"
 	// log "k8s.io/klog"
 
-	"github.com/altinity/clickhouse-operator/pkg/apis/metrics"
 	"github.com/altinity/clickhouse-operator/pkg/chop"
 	"github.com/altinity/clickhouse-operator/pkg/version"
 )
@@ -96,7 +96,7 @@ func Run() {
 	chop.New(kubeClient, chopClient, chopConfigFile)
 	log.Info(chop.Config().String(true))
 
-	exporter := metrics.StartMetricsREST(
+	exporter := clickhouse.StartMetricsREST(
 		metricsEP,
 		metricsPath,
 		chop.Config().ClickHouse.Metrics.Timeouts.Collect,

cmd/operator/app/main.go

@@ -18,12 +18,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	log "github.com/altinity/clickhouse-operator/pkg/announcer"
-	"github.com/altinity/clickhouse-operator/pkg/version"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
+
+	log "github.com/altinity/clickhouse-operator/pkg/announcer"
+	"github.com/altinity/clickhouse-operator/pkg/version"
 )
 
 // CLI parameter variables
@@ -69,23 +70,40 @@ func Run() {
 	ctx, cancelFunc := context.WithCancel(context.Background())
 
 	// Setup notification signals with cancel
-	setupNotification(cancelFunc)
-
-	initClickHouse(ctx)
-	initClickHouseReconcilerMetricsExporter(ctx)
-	keeperErr := initKeeper(ctx)
+	setupSignalsNotification(cancelFunc)
 
 	var wg sync.WaitGroup
-	wg.Add(3)
 
+	launchClickHouse(ctx, &wg)
+	launchClickHouseReconcilerMetricsExporter(ctx, &wg)
+	launchKeeper(ctx, &wg)
+
+	// Wait for completion
+	<-ctx.Done()
+	wg.Wait()
+}
+
+func launchClickHouse(ctx context.Context, wg *sync.WaitGroup) {
+	initClickHouse(ctx)
+	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		runClickHouse(ctx)
 	}()
+}
+
+func launchClickHouseReconcilerMetricsExporter(ctx context.Context, wg *sync.WaitGroup) {
+	initClickHouseReconcilerMetricsExporter(ctx)
+	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		runClickHouseReconcilerMetricsExporter(ctx)
 	}()
+}
+
+func launchKeeper(ctx context.Context, wg *sync.WaitGroup) {
+	keeperErr := initKeeper(ctx)
+	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		if keeperErr == nil {
@@ -100,14 +118,10 @@ func Run() {
 			log.Warning("Starting keeper skipped due to failed initialization with err: %v", keeperErr)
 		}
 	}()
-
-	// Wait for completion
-	<-ctx.Done()
-	wg.Wait()
 }
 
-// setupNotification sets up OS signals
-func setupNotification(cancel context.CancelFunc) {
+// setupSignalsNotification sets up OS signals
+func setupSignalsNotification(cancel context.CancelFunc) {
 	stopChan := make(chan os.Signal, 2)
 	signal.Notify(stopChan, os.Interrupt, syscall.SIGTERM)
 	go func() {

cmd/operator/app/thread_chi_reconciler_metrics.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"flag"
 	log "github.com/altinity/clickhouse-operator/pkg/announcer"
-	"github.com/altinity/clickhouse-operator/pkg/metrics"
+	"github.com/altinity/clickhouse-operator/pkg/metrics/operator"
 )
 
 // Prometheus exporter defaults
@@ -50,5 +50,5 @@ func runClickHouseReconcilerMetricsExporter(ctx context.Context) {
 	defer log.E().P()
 
 	log.V(1).F().Info("Starting operator metrics exporter")
-	metrics.StartMetricsExporter(metricsEP, metricsPath)
+	operator.StartMetricsExporter(metricsEP, metricsPath)
 }

cmd/operator/app/thread_keeper.go

@@ -59,7 +59,7 @@ func initKeeper(ctx context.Context) error {
 		For(&api.ClickHouseKeeperInstallation{}).
 		Owns(&apps.StatefulSet{}).
 		Complete(
-			&controller.ChkReconciler{
+			&controller.Controller{
 				Client: manager.GetClient(),
 				Scheme: manager.GetScheme(),
 			},

config/chk/conf.d/.gitkeep.xml

@@ -0,0 +1,6 @@
+<!-- IMPORTANT -->
+<!-- This file is auto-generated -->
+<!-- Do not edit this file - all changes would be lost -->
+<!-- Edit appropriate template in the following folder: -->
+<!-- deploy/builder/templates-config -->
+<!-- IMPORTANT -->

config/chk/keeper_config.d/.gitkeep.xml

@@ -0,0 +1,6 @@
+<!-- IMPORTANT -->
+<!-- This file is auto-generated -->
+<!-- Do not edit this file - all changes would be lost -->
+<!-- Edit appropriate template in the following folder: -->
+<!-- deploy/builder/templates-config -->
+<!-- IMPORTANT -->

config/chk/keeper_config.d/01-keeper-01-default-config.xml

@@ -0,0 +1,42 @@
+<!-- IMPORTANT -->
+<!-- This file is auto-generated -->
+<!-- Do not edit this file - all changes would be lost -->
+<!-- Edit appropriate template in the following folder: -->
+<!-- deploy/builder/templates-config -->
+<!-- IMPORTANT -->
+<clickhouse>
+    <keeper_server>
+        <coordination_settings>
+            <min_session_timeout_ms>10000</min_session_timeout_ms>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+            <session_timeout_ms>100000</session_timeout_ms>
+        </coordination_settings>
+        <hostname_checks_enabled>true</hostname_checks_enabled>
+        <log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>
+        <storage_path>/var/lib/clickhouse-keeper</storage_path>
+        <tcp_port>2181</tcp_port>
+        <enable_reconfiguration>true</enable_reconfiguration>
+    </keeper_server>
+    <listen_host>::</listen_host>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+    <logger>
+        <console>1</console>
+        <level>information</level>
+    </logger>
+    <max_connections>4096</max_connections>
+    <openSSL>
+        <server>
+            <cacheSessions>true</cacheSessions>
+            <certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile>
+            <dhParamsFile>/etc/clickhouse-keeper/dhparam.pem</dhParamsFile>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <preferServerCiphers>true</preferServerCiphers>
+            <privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile>
+            <verificationMode>none</verificationMode>
+        </server>
+    </openSSL>
+</clickhouse>

config/chk/keeper_config.d/01-keeper-02-readiness.xml

@@ -0,0 +1,16 @@
+<!-- IMPORTANT -->
+<!-- This file is auto-generated -->
+<!-- Do not edit this file - all changes would be lost -->
+<!-- Edit appropriate template in the following folder: -->
+<!-- deploy/builder/templates-config -->
+<!-- IMPORTANT -->
+<clickhouse>
+    <keeper_server>
+        <http_control>
+            <port>9182</port>
+            <readiness>
+                <endpoint>/ready</endpoint>
+            </readiness>
+        </http_control>
+    </keeper_server>
+</clickhouse>

config/chk/users.d/.gitkeep.xml

@@ -0,0 +1,6 @@
+<!-- IMPORTANT -->
+<!-- This file is auto-generated -->
+<!-- Do not edit this file - all changes would be lost -->
+<!-- Edit appropriate template in the following folder: -->
+<!-- deploy/builder/templates-config -->
+<!-- IMPORTANT -->