Fix empty log lines parsing in parse_cassandra_log, add automated tes… (

#1424)

* Fix empty log lines parsing in parse_cassandra_log, add automated tests for Vector

* Fix kustomize paths in tests, remove manual old kustomize install to prevent old versions from being in the path, change vector-test back to Makefile entirely for GHA

.github/workflows/kind_e2e_tests.yaml

@@ -134,12 +134,6 @@ jobs:
         with:
           go-version-file: 'go.mod'
           cache: true
-      - name: Install kustomize
-        uses: imranismail/setup-kustomize@v2
-        with:
-          kustomize-version: 4.x
-      - name: Log kustomize version
-        run: kustomize version
       - name: Install Kind
         run: go get sigs.k8s.io/kind
       - name: Set up Docker Buildx

.github/workflows/kind_multicluster_e2e_tests.yaml

@@ -102,12 +102,6 @@ jobs:
         with:
           go-version-file: 'go.mod'
           cache: true
-      - name: Install kustomize
-        uses: imranismail/setup-kustomize@v2
-        with:
-          kustomize-version: 4.x
-      - name: Log kustomize version
-        run: kustomize version
       - name: Install Kind
         run: go get sigs.k8s.io/kind
       - name: Set up Docker Buildx

.github/workflows/kind_multicluster_isolated_control_plane_e2e_tests.yaml

@@ -80,12 +80,6 @@ jobs:
         with:
           go-version-file: 'go.mod'
           cache: true
-      - name: Install kustomize
-        uses: imranismail/setup-kustomize@v2
-        with:
-          kustomize-version: 4.x
-      - name: Log kustomize version
-        run: kustomize version
       - name: Install Kind
         run: go get sigs.k8s.io/kind
       - name: Set up Docker Buildx

.github/workflows/kuttl_tests.yaml

@@ -86,12 +86,6 @@ jobs:
       - name: Load Docker images
         run: |
           docker load --input /tmp/k8ssandra-k8ssandra-operator.tar
-      - name: Install kustomize
-        uses: imranismail/setup-kustomize@v2
-        with:
-          kustomize-version: 4.x
-      - name: Log kustomize version
-        run: kustomize version
       - name: Install kuttl
         run: |
           make install-kuttl

.github/workflows/test_and_build_image.yaml

@@ -48,6 +48,9 @@ jobs:
             git status
             exit 1
           fi
+      - name: Run Vector tests
+        run: |
+          make vector-install vector-test
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
   build_image:

.github/workflows/version_tests.yaml

@@ -90,12 +90,6 @@ jobs:
       - name: Load Docker images
         run: |
           docker load --input /tmp/k8ssandra-k8ssandra-operator.tar
-      - name: Install kustomize
-        uses: imranismail/setup-kustomize@v2
-        with:
-          kustomize-version: 4.x
-      - name: Log kustomize version
-        run: kustomize version
       - name: Install kuttl
         run: |
           make install-kuttl

CHANGELOG/CHANGELOG-1.20.md

@@ -17,6 +17,9 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 * [DOCS] [#1469](https://github.com/riptano/mission-control/issues/1469) Add docs for Reaper's Control Plane deployment mode
 * [CHANGE] Bump default Medusa version to 0.22.3
+* [BUGFIX] [#1409](https://github.com/k8ssandra/k8ssandra-operator/issues/1409) Vector would crash in the Cassandra log parsing if empty lines were present. Add automated tests for Vector parsing rules.
+* [BUGFIX] [#1425](https://github.com/k8ssandra/k8ssandra-operator/issues/1425) prepare-helm-release.sh requires kustomize to be in the path and that makes make manifests fail.
+
 
 ## v1.20.1 - 2024-09-19
 

Makefile

@@ -130,9 +130,9 @@ help: ## Display this help.
 ##@ Development
 
 .PHONY: manifests
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+manifests: controller-gen kustomize ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=k8ssandra-operator webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	./scripts/prepare-helm-release.sh
+	KUSTOMIZE=$(KUSTOMIZE) ./scripts/prepare-helm-release.sh
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -151,14 +151,23 @@ lint: golangci-lint ## Run golangci-lint against code.
 	$(GOLANGCI_LINT) run ./...
 
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
-test: manifests generate fmt vet lint envtest kustomize ## Run tests.
+test: manifests generate fmt vet lint envtest ## Run tests.
 ifdef TEST
 	@echo Running test $(TEST)
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(GO_FLAGS) ./apis/... ./pkg/... ./test/yq/... ./controllers/... -run="$(TEST)" -covermode=atomic -coverprofile coverage.out
 else
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(GO_FLAGS) ./apis/... ./pkg/... ./test/yq/... ./controllers/... -covermode=atomic -coverprofile coverage.out
 endif
 
+.PHONY: vector-test
+vector-test: ## Run vector tests
+	@echo Generating test files for Vector tests
+	$(eval TMP := $(shell mktemp -d))
+	VECTOR_TEST_FILES=true OUTPUT_PATH=$(TMP) go test -v ./pkg/telemetry -run=TestGenerateTomlTestFiles
+	@echo Running vector test files
+	OUTPUT_PATH=$(TMP) VECTOR=$(VECTOR) scripts/run-vector-tests.sh
+	rm -rf $(TMP)
+
 E2E_TEST_TIMEOUT ?= 3600s
 
 PHONY: e2e-test
@@ -326,6 +335,7 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint # TODO Add linting to the GHA also
+VECTOR ?= $(LOCALBIN)/bin/vector
 
 ## Tool Versions
 CERT_MANAGER_VERSION ?= v1.12.2
@@ -346,6 +356,10 @@ cert-manager-multi: ## Install cert-manager to the clusters
 		make cert-manager;  \
 	done
 
+.PHONY: vector-install
+vector-install:
+	curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev | bash -s -- --prefix $(LOCALBIN) -y
+
 # Install NGINX in the current Kind cluster using Helm and a values file that is suitable for
 # running e2e tests locally with a cluster created with setup-kind-multicluster.sh. Helm must be
 # pre-installed on the system.

pkg/telemetry/vector.go

@@ -140,10 +140,14 @@ timeout_ms = 10000
 		Inputs: []string{"systemlog"},
 		Config: `source = '''
 del(.source_type)
+.message = string!(.message)
+.message = strip_whitespace(.message)
 . |= parse_groks!(.message, patterns: [
-  "%{LOGLEVEL:loglevel}\\s+\\[(?<thread>((.+)))\\]\\s+%{TIMESTAMP_ISO8601:timestamp}\\s+%{JAVACLASS:class}:%{NUMBER:line}\\s+-\\s+(?<message>(.+\\n?)+)",
+  "%{LOGLEVEL:loglevel}\\s+\\[(?<thread>((.+)))\\]\\s+%{TIMESTAMP_ISO8601:timestamp_raw}\\s+%{JAVACLASS:class}:%{NUMBER:line}\\s+-\\s+(?<message>(.+\\n?)+)",
   ]
 )
+.timestamp = parse_timestamp!(.timestamp_raw, format: "%Y-%m-%d %T,%3f")
+del(.timestamp_raw)
 pod_name, err = get_env_var("POD_NAME")
 if err == null {
   .pod_name = pod_name

pkg/telemetry/vector_test.go

@@ -1,6 +1,9 @@
 package telemetry
 
 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/go-logr/logr/testr"
@@ -366,3 +369,95 @@ func TestOverrideSourcePossible(t *testing.T) {
 
 	assert.Equal("stdin", sources[0].Type)
 }
+
+func TestGenerateTomlTestFiles(t *testing.T) {
+	if os.Getenv("VECTOR_TEST_FILES") == "" {
+		t.Skip("Set VECTOR_TEST_FILES to generate vector test files")
+	}
+	outputDir := os.Getenv("OUTPUT_PATH")
+	if outputDir == "" {
+		fmt.Printf("No OUTPUT_PATH env variable set")
+		t.FailNow()
+	}
+	assert := assert.New(t)
+	sources, transformers, sinks := BuildDefaultVectorComponents(vector.VectorConfig{})
+	assert.Equal(2, len(sources))
+	assert.Equal(2, len(transformers))
+	assert.Equal(1, len(sinks))
+
+	telemetrySpec := &telemetry.TelemetrySpec{
+		Vector: &telemetry.VectorSpec{
+			Enabled: ptr.To[bool](true),
+			Components: &telemetry.VectorComponentsSpec{
+				Sources:    sources,
+				Sinks:      sinks,
+				Transforms: transformers,
+			},
+		},
+	}
+
+	// Vector components are provided in the Telemetry spec, build the Vector sink config from them
+	vectorConfigToml := BuildCustomVectorToml(telemetrySpec)
+
+	b := strings.Builder{}
+	fmt.Fprint(&b, vectorConfigToml)
+
+	// Append tests
+
+	fmt.Fprint(&b, `
+[[tests]]
+name = "Test parsing normal Cassandra logs"
+
+[[tests.inputs]]
+insert_at = "parse_cassandra_log"
+value = "WARN  [ScheduledTasks:1] 2024-10-01 12:31:17,694  LeaksDetectorImpl.java:306 - LEAK: RandomAccessReader/RandomAccessReader was not released before it was garbage-collected. This resource is a debugging aid, no negative consequences follow from this leak. However, please report this nonetheless even if auto-cleaning succeeded. Auto cleaning result: not attempted, no cleaner."
+
+[[tests.outputs]]
+extract_from = "parse_cassandra_log"
+
+[[tests.outputs.conditions]]
+type = "vrl"
+source = '''
+assert_eq!(.loglevel, "WARN")
+assert_eq!(.thread, "ScheduledTasks:1")
+assert!(is_timestamp(.timestamp))
+assert!(is_string(.message))
+assert!(is_string(.line))
+assert!(exists(.class))
+'''
+`)
+
+	assert.NoError(os.WriteFile(fmt.Sprintf("%s/vector-simple.toml", outputDir), []byte(b.String()), 0644))
+
+	b.Reset()
+	fmt.Fprint(&b, vectorConfigToml)
+
+	fmt.Fprint(&b, `
+[[tests]]
+name = "Test parsing normal Cassandra logs"
+
+[[tests.inputs]]
+insert_at = "parse_cassandra_log"
+value = '''
+WARN  [ScheduledTasks:1] 2024-10-01 12:31:17,694  LeaksDetectorImpl.java:306 - LEAK: RandomAccessReader/RandomAccessReader was not released before it was garbage-collected. This resource is a debugging aid, no negative consequences follow from this leak. However, please report this nonetheless even if auto-cleaning succeeded. Auto cleaning result: not attempted, no cleaner."
+
+
+'''
+
+[[tests.outputs]]
+extract_from = "parse_cassandra_log"
+
+[[tests.outputs.conditions]]
+type = "vrl"
+source = '''
+assert_eq!(.loglevel, "WARN")
+assert_eq!(.thread, "ScheduledTasks:1")
+assert!(is_timestamp(.timestamp))
+assert!(is_string(.message))
+assert!(is_string(.line))
+assert!(exists(.class))
+'''
+`)
+
+	assert.NoError(os.WriteFile(fmt.Sprintf("%s/vector-emptyline.toml", outputDir), []byte(b.String()), 0644))
+}

scripts/prepare-helm-release.sh

@@ -8,11 +8,15 @@
 
 set -e
 
+if [ -z "$KUSTOMIZE" ]; then
+    KUSTOMIZE=$(command -v kustomize)
+fi
+
 mkdir -p build/helm
 # Generate the CRDs
-kustomize build config/crd > charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml
+$KUSTOMIZE build config/crd > charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml
 # Generate the role.yaml and clusterrole.yaml files using the RBAC generated manifests
-kustomize build config/rbac > build/helm/k8ssandra-operator-rbac.yaml
+$KUSTOMIZE build config/rbac > build/helm/k8ssandra-operator-rbac.yaml
 cat charts/templates/role.tmpl.yaml | tee build/helm/role.yaml > /dev/null
 cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 1).rules' | tee -a build/helm/role.yaml > /dev/null
 echo "{{- end }}" >> build/helm/role.yaml

scripts/run-vector-tests.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+for i in $OUTPUT_PATH/*; do
+   $VECTOR test $i
+   if [ "$?" -ne 0 ]; then
+      exit 1
+   fi
+done

test/kustomize/kustomize.go

@@ -13,20 +13,18 @@ func LogOutput(enabled bool) {
 	logOutput = enabled
 }
 
-func BuildDir(dir string) (*bytes.Buffer, error) {
+func kustomizePath() string {
 	binDir := os.Getenv("LOCALBIN")
-	kustomizeLocation := ""
 	if binDir == "" {
 		fmt.Println("warning: LOCALBIN environment variable not set, attempting to use system kustomize")
-		kustomizeLocation = "kustomize"
-	} else {
-		fmt.Println("LOCALBIN: " + binDir)
-		kustomizeLocation = binDir + "/kustomize"
+		return "kustomize"
 	}
-	cmd := exec.Command(kustomizeLocation, "build")
-	cmd.Dir = dir
+	return binDir + "/kustomize"
+}
 
-	fmt.Println(cmd)
+func BuildDir(dir string) (*bytes.Buffer, error) {
+	cmd := exec.Command(kustomizePath(), "build")
+	cmd.Dir = dir
 
 	output, err := cmd.CombinedOutput()
 	buffer := bytes.NewBuffer(output)
@@ -39,9 +37,7 @@ func BuildDir(dir string) (*bytes.Buffer, error) {
 }
 
 func BuildUrl(url string) (*bytes.Buffer, error) {
-	cmd := exec.Command("kustomize", "build", url)
-
-	fmt.Println(cmd)
+	cmd := exec.Command(kustomizePath(), "build", url)
 
 	output, err := cmd.CombinedOutput()
 	buffer := bytes.NewBuffer(output)
@@ -54,11 +50,9 @@ func BuildUrl(url string) (*bytes.Buffer, error) {
 }
 
 func SetNamespace(dir, namespace string) error {
-	cmd := exec.Command("kustomize", "edit", "set", "namespace", namespace)
+	cmd := exec.Command(kustomizePath(), "edit", "set", "namespace", namespace)
 	cmd.Dir = dir
 
-	fmt.Println(cmd)
-
 	output, err := cmd.CombinedOutput()
 
 	if logOutput {
@@ -69,11 +63,9 @@ func SetNamespace(dir, namespace string) error {
 }
 
 func AddResource(path string) error {
-	cmd := exec.Command("kustomize", "edit", "add", "resource", path)
+	cmd := exec.Command(kustomizePath(), "edit", "add", "resource", path)
 	cmd.Dir = "../testdata/k8ssandra-operator"
 
-	fmt.Println(cmd)
-
 	output, err := cmd.CombinedOutput()
 
 	if logOutput {
@@ -84,11 +76,9 @@ func AddResource(path string) error {
 }
 
 func RemoveResource(path string) error {
-	cmd := exec.Command("kustomize", "edit", "remove", "resource", path)
+	cmd := exec.Command(kustomizePath(), "edit", "remove", "resource", path)
 	cmd.Dir = "../testdata/k8ssandra-operator"
 
-	fmt.Println(cmd)
-
 	output, err := cmd.CombinedOutput()
 
 	if logOutput {