Skip to content

Commit

Permalink
Kubernetes Operator for Nessie
Browse files Browse the repository at this point in the history
  • Loading branch information
adutra committed May 7, 2024
1 parent 1b9c925 commit 99eeb54
Show file tree
Hide file tree
Showing 197 changed files with 12,653 additions and 4 deletions.
74 changes: 70 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,15 @@ jobs:
-x :nessie-quarkus:compileAll \
-x :nessie-quarkus-cli:compileAll \
-x :nessie-events-quarkus:compileAll \
-x :nessie-operator:compileAll \
--scan
- name: Gradle / Compile Quarkus
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll --scan
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-quarkus-cli:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan
- name: Gradle / Checkstyle
run: ./gradlew checkstyle --scan
Expand Down Expand Up @@ -141,7 +142,8 @@ jobs:
java-version: ${{ matrix.java-version }}

- name: Gradle / test
run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-quarkus-cli:test -x :nessie-events-quarkus:test --scan
run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-quarkus-cli:test -x :nessie-events-quarkus:test -x :nessie-operator:test
--scan

- name: Capture Test Reports
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -196,6 +198,13 @@ jobs:
./gradlew :nessie-events-quarkus:test --scan || \
./gradlew :nessie-events-quarkus:test --scan
- name: Gradle / Test Quarkus Operator
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-operator:test --scan || \
./gradlew :nessie-operator:test --scan || \
./gradlew :nessie-operator:test --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
Expand Down Expand Up @@ -263,6 +272,7 @@ jobs:
-x :nessie-quarkus:intTest \
-x :nessie-quarkus-cli:intTest \
-x :nessie-events-quarkus:intTest \
-x :nessie-operator:intTest \
$(cat ../persist-prjs.txt) \
$(cat ../storage-prjs.txt) \
$(cat ../spark-prjs.txt) \
Expand Down Expand Up @@ -446,6 +456,61 @@ jobs:
job-name: 'int-test-quarkus'
java-version: ${{ matrix.java-version }}

int-test-operator:
name: CI intTest Operator
runs-on: ubuntu-22.04
strategy:
max-parallel: 1
matrix:
java-version: ['17']
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
with:
java-version: ${{ matrix.java-version }}

- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
with:
java-version: ${{ matrix.java-version }}

- name: Gradle / intTest Operator
uses: gradle/actions/setup-gradle@v3
with:
arguments: |
:nessie-operator:intTest
--scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: ci-inttest-operator-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7

- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-operator'
java-version: ${{ matrix.java-version }}

determine-jobs:
name: CI Determine jobs
runs-on: ubuntu-22.04
Expand Down Expand Up @@ -911,6 +976,7 @@ jobs:
- int-test-stores
- int-test-integrations
- int-test-quarkus
- int-test-operator
strategy:
max-parallel: 2
matrix:
Expand Down
1 change: 1 addition & 0 deletions bom/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ dependencies {
api(project(":nessie-keycloak-testcontainer"))
api(project(":nessie-minio-testcontainer"))
api(project(":nessie-nessie-testcontainer"))
api(project(":nessie-operator"))
api(project(":nessie-quarkus-auth"))
api(project(":nessie-quarkus-common"))
api(project(":nessie-quarkus-cli"))
Expand Down
1 change: 1 addition & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quark
quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkusPlatform" }
quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkusPlatform" }
quarkus-logging-sentry = { module = "io.quarkiverse.loggingsentry:quarkus-logging-sentry", version = "2.0.7" }
quarkus-operator-sdk-bom = { module = "io.quarkus.platform:quarkus-operator-sdk-bom", version.ref = "quarkusPlatform" }
rest-assured = { module = "io.rest-assured:rest-assured", version = "5.4.0" }
rocksdb-jni = { module = "org.rocksdb:rocksdbjni", version = "9.1.1" }
scala-library-v212 = { module = "org.scala-lang:scala-library", version = { strictly = "[2.12, 2.13[", prefer = "2.12.19" }}
Expand Down
1 change: 1 addition & 0 deletions gradle/projects.main.properties
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ nessie-jaxrs-tests=servers/jax-rs-tests
nessie-keycloak-testcontainer=testing/keycloak-container
nessie-nessie-testcontainer=testing/nessie-container
nessie-object-storage-mock=testing/object-storage-mock
nessie-operator=operator
nessie-quarkus-auth=servers/quarkus-auth
nessie-quarkus-cli=servers/quarkus-cli
nessie-quarkus-common=servers/quarkus-common
Expand Down
114 changes: 114 additions & 0 deletions operator/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@

VERSION ?= $(shell cat ../version.txt | sed -e 's/.*-SNAPSHOT/latest/g')
RELEASE_VERSION ?= $(shell cat ../version.txt | sed -e 's/-SNAPSHOT//g')

# CHANNELS define the bundle channels used in the bundle.
# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
# To re-generate a bundle for other specific channels without changing the standard setup, you can:
# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
ifneq ($(origin CHANNELS), undefined)
BUNDLE_CHANNELS := --channels=$(CHANNELS)
endif

# DEFAULT_CHANNEL defines the default channel used in the bundle.
# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
# To re-generate a bundle for any other default channel without changing the default setup, you can:
# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
ifneq ($(origin DEFAULT_CHANNEL), undefined)
BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
endif
BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)

# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
# This variable is used to construct full image tags for bundle and catalog images.
IMAGE_TAG_BASE ?= ghcr.io/projectnessie/nessie-operator

# BUNDLE_IMG defines the image:tag used for the bundle.
# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION)

# Image URL to use all building/pushing image targets
IMG ?= $(IMAGE_TAG_BASE):$(VERSION)

PULL_POLICY ?= $(shell [ "$(VERSION)" = "latest" ] && echo "Always" || echo "IfNotPresent")
PLATFORM ?= linux/$(shell arch)

all: docker-build

##@ General

# The help target prints out all targets with their descriptions organized
# beneath their categories. The categories are represented by '##@' and the
# target descriptions by '##'. The awk commands is responsible for reading the
# entire set of makefiles included in this invocation, looking for lines of the
# file as xyz: ## something, and then pretty-format the target and help. Then,
# if there's a line with ##@ something, that gets pretty-printed as a category.
# More info on the usage of ANSI control characters for terminal formatting:
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
# More info on the awk command:
# http://linuxcommand.org/lc3_adv_awk.php

help: ## Display this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

##@ Build

docker-build: ## Build docker image with the manager.
../gradlew :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
-Dquarkus.container-image.build=true \
-Dquarkus.container-image.image=${IMG} \
-Dquarkus.jib.platforms=${PLATFORM} \
-Dquarkus.kubernetes.prometheus.generate-service-monitor=false \
-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}

docker-push: ## Build and push docker image with the manager.
../gradlew :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
-Dquarkus.container-image.build=true \
-Dquarkus.container-image.push=true \
-Dquarkus.container-image.image=${IMG} \
-Dquarkus.jib.platforms=${PLATFORM} \
-Dquarkus.kubernetes.prometheus.generate-service-monitor=false \
-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}

##@ Deployment

install: ## Install CRDs into the K8s cluster specified in ~/.kube/config.
@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl apply -f $(file);)

uninstall: ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl delete -f $(file);)

deploy: ## Deploy controller to the K8s cluster specified in ~/.kube/config.
kubectl apply -f build/kubernetes/kubernetes.yml

undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
kubectl delete -f build/kubernetes/kubernetes.yml

##@ Helm

helm-install: ## Install CRDs and the operator using Helm.
helm install nessie-operator build/helm -n nessie-operator

helm-upgrade: ## Upgrade CRDs and the operator using Helm.
helm upgrade nessie-operator build/helm -n nessie-operator

helm-uninstall: ## Uninstall CRDs and the operator using Helm.
helm uninstall nessie-operator -n nessie-operator

##@ Bundle

.PHONY: bundle
bundle: ## Generate bundle manifests and metadata, then validate generated files.
cat build/kubernetes/* | operator-sdk generate bundle -q --overwrite --version $(RELEASE_VERSION) $(BUNDLE_METADATA_OPTS)
operator-sdk bundle validate ./bundle
# TODO use quarkus

.PHONY: bundle-build
bundle-build: ## Build the bundle image.
docker build -f build/bundle/nessie-operator/bundle.Dockerfile -t $(BUNDLE_IMG) build/bundle/nessie-operator

.PHONY: bundle-push
bundle-push: ## Push the bundle image.
docker push $(BUNDLE_IMG)
17 changes: 17 additions & 0 deletions operator/PROJECT
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Code generated by tool. DO NOT EDIT.
# This file is used to track the info used to scaffold your project
# and allow the plugins properly work.
# More info: https://book.kubebuilder.io/reference/project-config.html
domain: projectnessie.org
layout:
- quarkus.javaoperatorsdk.io/v1-alpha
projectName: nessie-operator
resources:
- api:
crdVersion: v1
namespaced: true
domain: projectnessie.org
group: nessie
kind: Nessie
version: v1alpha1
version: "3"
88 changes: 88 additions & 0 deletions operator/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# Kubernetes Operator for Nessie

## Overview

This module is a Kubernetes Operator for Nessie.

**WARNING: This is a work in progress and is not ready for production use.**

The operator is designed to manage the lifecycle of Nessie instances in a Kubernetes cluster. It can
also be used to run GC jobs.

This project was created using [Operator SDK]:

```bash
operator-sdk init --plugins=quarkus --domain=projectnessie.org --project-name=nessie-operator
operator-sdk create api --plugins=quarkus --group nessie --version=v1alpha1 --kind=Nessie
operator-sdk create api --plugins=quarkus --group nessie --version=v1alpha1 --kind=NessieGC
```

[Operator SDK]:https://sdk.operatorframework.io/docs/cli/operator-sdk/

## Development

### Prerequisites

- Operator SDK: https://sdk.operatorframework.io/docs/installation/

### Adhoc testing with Minikube

Install [minikube](https://minikube.sigs.k8s.io/docs/start/).

If you need ingress, install the ingress addon:

```bash
minikube addons enable ingress
minikube tunnel
```

Create the `nessie-operator` and `nessie-ns` namespaces (only needed once):

```bash
kubectl create namespace nessie-operator
kubectl create namespace nessie-ns
```

Grant admin rights to the `nessie-operator` service account (only needed once):

```bash
kubectl apply -f - <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: nessie-operator-admin
subjects:
- kind: ServiceAccount
name: nessie-operator
namespace: nessie-operator
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: ""
EOF
```

Build the operator docker image _inside_ minikube to facilitate testing (doesn't need a registry):

```bash
eval $(minikube docker-env)
cd operator
make docker-build PULL_POLICY=IfNotPresent
```

Note: the `PULL_POLICY=IfNotPresent` is required to avoid pulling the image from a registry.

Install the CRDs and deploy the operator in the `nessie-operator` namespace:

```bash
make install deploy
```

Create a Nessie resource in the `nessie-ns` namespace:

```bash
kubectl apply -n nessie-ns -f examples/nessie-simple.yaml
```

You should see 1 pod, 1 deployment and 1 service running. More example resources can be found in the
`examples` directory.
Loading

0 comments on commit 99eeb54

Please sign in to comment.