diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ff966cc..55a431f33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,12 @@ ### Fixed Bugs -- [PR #109](https://github.com/Orange-OpenSource/nifikop/pull/109) - **[Operator/NifiCluster]** Change namespace watch configuration to manage single namespace deletion. +## v0.6.2 + +### Fixed Bugs +- [PR #107](https://github.com/Orange-OpenSource/nifikop/pull/107) - **[Operator/NiFiCluster]** Correct the way to path PVCs. +- [PR #109](https://github.com/Orange-OpenSource/nifikop/pull/109) - **[Operator/NifiCluster]** Change namespace watch configuration to manage single namespace deletion. ## v0.6.1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 92c81edf5..0be4c7d0c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -29,7 +29,7 @@ spec: - /manager args: - --leader-elect - image: orangeopensource/nifikop:v0.6.1-release + image: orangeopensource/nifikop:v0.6.2-release name: nifikop securityContext: allowPrivilegeEscalation: false diff --git a/helm/nifikop/Chart.yaml b/helm/nifikop/Chart.yaml index 2f43d8680..d5e454caf 100644 --- a/helm/nifikop/Chart.yaml +++ b/helm/nifikop/Chart.yaml @@ -4,8 +4,8 @@ name: nifikop home: https://github.com/Orange-OpenSource/nifikop sources: - https://github.com/Orange-OpenSource/nifikop -version: 0.6.1 -appVersion: 0.6.1-release +version: 0.6.2 +appVersion: 0.6.2-release icon: maintainers: - name: erdrix diff --git a/helm/nifikop/README.md b/helm/nifikop/README.md index e9bf4ff11..943de19b7 100644 --- a/helm/nifikop/README.md +++ b/helm/nifikop/README.md @@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | |----------------------------------|--------------------------------------------------|-------------------------------------------| | `image.repository` | Image | `orangeopensource/nifikop` | -| `image.tag` | Image tag | `v0.6.1-release` | +| `image.tag` | Image tag | `v0.6.2-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/helm/nifikop/values.yaml b/helm/nifikop/values.yaml index 720010d2e..ce679f0b1 100644 --- a/helm/nifikop/values.yaml +++ b/helm/nifikop/values.yaml @@ -2,7 +2,7 @@ ## image: repository: orangeopensource/nifikop - tag: v0.6.1-release + tag: v0.6.2-release pullPolicy: Always imagePullSecrets: enabled: false diff --git a/site/docs/2_setup/1_getting_started.md b/site/docs/2_setup/1_getting_started.md index ef149a1e6..324274d5b 100644 --- a/site/docs/2_setup/1_getting_started.md +++ b/site/docs/2_setup/1_getting_started.md @@ -114,8 +114,8 @@ Now deploy the helm chart : helm install nifikop \ orange-incubator/nifikop \ --namespace=nifi \ - --version 0.6.1 \ - --set image.tag=v0.6.1release \ + --version 0.6.2 \ + --set image.tag=v0.6.2-release \ --set resources.requests.memory=256Mi \ --set resources.requests.cpu=250m \ --set resources.limits.memory=256Mi \ diff --git a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md index dbf356aa2..1e455b8fe 100644 --- a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md +++ b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md @@ -32,7 +32,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | |----------------------------------|--------------------------------------------------|-------------------------------------------| | `image.repository` | Image | `orangeopensource/nifikop` | -| `image.tag` | Image tag | `v0.6.1-release` | +| `image.tag` | Image tag | `v0.6.2-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/site/website/versioned_docs/version-v0.6.1/2_setup/1_getting_started.md b/site/website/versioned_docs/version-v0.6.1/2_setup/1_getting_started.md index ef149a1e6..324274d5b 100644 --- a/site/website/versioned_docs/version-v0.6.1/2_setup/1_getting_started.md +++ b/site/website/versioned_docs/version-v0.6.1/2_setup/1_getting_started.md @@ -114,8 +114,8 @@ Now deploy the helm chart : helm install nifikop \ orange-incubator/nifikop \ --namespace=nifi \ - --version 0.6.1 \ - --set image.tag=v0.6.1release \ + --version 0.6.2 \ + --set image.tag=v0.6.2-release \ --set resources.requests.memory=256Mi \ --set resources.requests.cpu=250m \ --set resources.limits.memory=256Mi \ diff --git a/site/website/versioned_docs/version-v0.6.1/2_setup/3_install/1_customizable_install_with_helm.md b/site/website/versioned_docs/version-v0.6.1/2_setup/3_install/1_customizable_install_with_helm.md index dbf356aa2..1e455b8fe 100644 --- a/site/website/versioned_docs/version-v0.6.1/2_setup/3_install/1_customizable_install_with_helm.md +++ b/site/website/versioned_docs/version-v0.6.1/2_setup/3_install/1_customizable_install_with_helm.md @@ -32,7 +32,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | |----------------------------------|--------------------------------------------------|-------------------------------------------| | `image.repository` | Image | `orangeopensource/nifikop` | -| `image.tag` | Image tag | `v0.6.1-release` | +| `image.tag` | Image tag | `v0.6.2-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/site/website/versioned_docs/version-v0.6.2/1_concepts/1_introduction.md b/site/website/versioned_docs/version-v0.6.2/1_concepts/1_introduction.md new file mode 100644 index 000000000..ecbcc49f7 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/1_concepts/1_introduction.md @@ -0,0 +1,36 @@ +--- +id: 1_introduction +title: Introduction +sidebar_label: Introduction +--- + +The Orange NiFi operator is a Kubernetes operator to automate provisioning, management, autoscaling and operations of [Apache NiFi](https://nifi.apache.org/) clusters deployed to K8s. + +## Overview + +Apache NiFi is an open-source solution that support powerful and scalable directed graphs of data routing, transformation, and system mediation logic. +Some of the high-level capabilities and objectives of Apache NiFi include, and some of the main features of the **NiFiKop** are: + +- **Fine grained** node configuration support +- Graceful rolling upgrade +- graceful NiFi cluster **scaling** +- Encrypted communication using SSL +- the provisioning of secure NiFi clusters +- Advanced Dataflow and user management via CRD + +Some of the roadmap features : + +- Monitoring via **Prometheus** +- Automatic reaction and self healing based on alerts (plugin system, with meaningful default alert plugins) +- graceful NiFi cluster **scaling and rebalancing** + +## Motivation + +At [Orange](https://opensource.orange.com/fr/accueil/) we are building some [Kubernetes operator](https://github.com/Orange-OpenSource?utf8=%E2%9C%93&q=operator&type=&language=), that operate NiFi and Cassandra clusters (among other types) for our business cases. + +There are already some approaches to operating NiFi on Kubernetes, however, we did not find them appropriate for use in a highly dynamic environment, nor capable of meeting our needs. + +- [Helm chart](https://github.com/cetic/helm-nifi) +- [Cloudera Nifi Operator](https://blog.cloudera.com/cloudera-flow-management-goes-cloud-native-with-apache-nifi-on-red-hat-openshift-kubernetes-platform/) + +Finally, our motivation is to build an open source solution and a community which drives the innovation and features of this operator. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/1_concepts/2_design_principes.md b/site/website/versioned_docs/version-v0.6.2/1_concepts/2_design_principes.md new file mode 100644 index 000000000..867de25a9 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/1_concepts/2_design_principes.md @@ -0,0 +1,62 @@ +--- +id: 2_design_principes +title: Design Principes +sidebar_label: Design Principes +--- + +## Pod level management + +NiFi is a stateful application. The first piece of the puzzle is the Node, which is a simple server capable of createing/forming a cluster with other Nodes. Every Node has his own **unique** configuration which differs slightly from all others. + +All NiFi on Kubernetes setup use [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) to create a NiFi Cluster. Just to quickly recap from the K8s docs: + +>StatefulSet manages the deployment and scaling of a set of Pods, and provide guarantees about their ordering and uniqueness. Like a Deployment, a StatefulSet manages Pods that are based on an identical container spec. Unlike a Deployment, a StatefulSet maintains sticky identities for each of its Pods. These pods are created from the same spec, but are not interchangeable: each has a persistent identifier that is maintained across any rescheduling. + +How does this looks from the perspective of Apache NiFi ? + +With StatefulSet we get : +- unique Node IDs generated during Pod startup +- networking between Nodes with headless services +- unique Persistent Volumes for Nodes + +Using StatefulSet we **lose** the ability to : + +- modify the configuration of unique Nodes +- remove a specific Node from a cluster (StatefulSet always removes the most recently created Node) +- use multiple, different Persistent Volumes for each Node + +The Orange NiFi Operator uses `simple` Pods, ConfigMaps, and PersistentVolumeClaims, instead of StatefulSet (based on the design used by [Banzai Cloud Kafka Operator](https://github.com/banzaicloud/kafka-operator)). +Using these resources allows us to build an Operator which is better suited to NiFi. + +With the Orange NiFi operator we can: + +- modify the configuration of unique Nodes +- remove specific Nodes from clusters +- use multiple Persistent Volumes for each Node + +## Dataflow Lifecycle management + +The [Dataflow Lifecycle management feature](./3_features.md#dataflow-lifecycle-management-via-crd) introduces 3 new CRDs : + +- **NiFiRegistryClient :** Allowing you to declare a [NiFi registry client](https://nifi.apache.org/docs/nifi-registry-docs/html/getting-started.html#connect-nifi-to-the-registry). +- **NiFiParameterContext :** Allowing you to create parameter context, with two kinds of parameters, a simple `map[string]string` for non-sensitive parameters and a `list of secrets` which contains sensitive parameters. +- **NiFiDataflow :** Allowing you to declare a Dataflow based on a `NiFiRegistryClient` and optionally a `ParameterContext`, which will be deployed and managed by the operator on the `targeted NiFi cluster`. + +The following diagram shows the interactions between all the components : + +![dataflow lifecycle management schema](/img/1_concepts/2_design_principes/dataflow_lifecycle_management_schema.jpg) + +With each CRD comes a new controller, with a reconcile loop : + +- **NiFiRegistryClient's controller :** + +![NiFi registry client's reconcile loop](/img/1_concepts/2_design_principes/registry_client_reconcile_loop.jpeg) + +- **NiFiParameterContext's controller :** + +![NiFi parameter context's reconcile loop](/img/1_concepts/2_design_principes/parameter_context_reconcile_loop.jpeg) + +- **NiFiDataflow's controller :** + +![NiFi dataflow's reconcile loop](/img/1_concepts/2_design_principes/dataflow_reconcile_loop.jpeg) + diff --git a/site/website/versioned_docs/version-v0.6.2/1_concepts/3_features.md b/site/website/versioned_docs/version-v0.6.2/1_concepts/3_features.md new file mode 100644 index 000000000..6ccd1e78d --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/1_concepts/3_features.md @@ -0,0 +1,57 @@ +--- +id: 3_features +title: Features +sidebar_label: Features +--- + +To highligt some of the features we needed and were not possible with the operators available, please keep reading + +## Fine Grained Node Config Support + +We needed to be able to react to events in a fine-grained way for each Node - and not in the limited way StatefulSet does (which, for example, removes the most recently created Nodes). Some of the available solutions try to overcome these deficits by placing scripts inside the container to generate configs at runtime (a good example is our [Cassandra Operator](https://github.com/Orange-OpenSource/casskop)), whereas the Orange NiFi operator's configurations are deterministically placed in specific Configmaps. + +## Graceful NiFi Cluster Scaling + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +## Graceful Rolling Upgrade + +Operator supports graceful rolling upgrade. It means the operator will check if the cluster is healthy. + +## Dynamic Configuration Support + +NiFi operates with two type of configs: + +- Read-only +- PerNode + +Read only config requires node restart to update all the others may be updated dynamically. +Operator CRD distinguishes these fields, and proceed with the right action. It can be a rolling upgrade, or +a dynamic reconfiguration. + +## Dataflow lifecycle management via CRD + +In a cloud native approach, we are looking for important management features, which we have applied to NiFi Dataflow : + +- **Automated deployment :** Based on the NiFi registry, you can describe your `NiFiDataflow` resource that will be deployed and run on the targeted NiFi cluster. +- **Portability :** On kubernetes everything is a yaml file, so with NiFiKop we give you the ability to describe your clusters but also the `registry clients`, `parameter contexts` and `dataflows` of your NiFi application, so that you can redeploy the same thing in a different namespace or cluster. +- **State management :** With NiFiKop resources, you can describe what you want, and the operator deals with the NiFi Rest API to make sure the resource stays in sync (even if someone manually makes changes directly on NiFi cluster). +- **Configurations :** Based on the `Parameter Contexts`, NiFiKop allows you to associate to your `Dataflow` (= your applications) with a different configuration depending on the environment ! + +## Users and access policies management + +Without the management of users and access policies associated, it was not possible to have a fully automated NiFi cluster setup due to : + +- **Node scaling :** when a new node joins the cluster it needs to have some roles like `proxy user request`, `view data` etc., by managing users and access policies we can easily create a user for this node with the right accesses. +- **Operator admin rigth :** For the operator to manage efficiently the cluster it needs a lot of rights as `deploying process groups`, `empty the queues` etc., these rights are not available by default when you set a user as [InitialAdmin](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#initial-admin-identity). Once again by giving the ability to define users and access policies we go through this. +- **User's access :** as seen just below we need to define the operator as `InitialAdmin`, in this situation there is no more users that can access to the web UI to manually give access to other users. That's why we extend the `InitialAdmin` concept into the operator, giving the ability to define a list of users as admins. + +In addition to these requirements to have a fully automated and managed cluster, we introduced some useful features : + +- **User management :** using `NifiUser` resource, you are able to create (or bind an existing) user in NiFi cluster and apply some access policies that will be managed and continuously synced by the operator. +- **Group management :** using `NifiUserGroup` resource, you can create groups in NiFi cluster and apply access policies and a list of `NifiUser` that will be managed and continuously synced by the operator. +- **Default group :** As the definition of `NifiUser` and `NifiUserGroup` resources could be heavy for some simple use cases, we also decided to define two default groups that you can feed with a list of users that will be created and managed by the operator (no kubernetes resources to create) : + - **Admins :** a group giving access to everything on the NiFi Cluster, + - **Readers :** a group giving access as viewer on the NiFi Cluster. + +By introducing this feature we are giving you the ability to fully automate your deployment, from the NiFi Cluster to your managed NiFi Dataflow. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/1_concepts/4_roadmap.md b/site/website/versioned_docs/version-v0.6.2/1_concepts/4_roadmap.md new file mode 100644 index 000000000..fe28367ea --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/1_concepts/4_roadmap.md @@ -0,0 +1,95 @@ +--- +id: 4_roadmap +title: Roadmap +sidebar_label: Roadmap +--- + +## Available + +### NiFi cluster installation + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +### Graceful NiFi Cluster Scaling + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +### Communication via SSL + +| | | +| --------------------- | -------- | +| Status | Done | +| Priority | High | +| Targeted Start date | May 2020 | + + +The operator fully automates NiFi's SSL support. +The operator can provision the required secrets and certificates for you, or you can provide your own. + +### Dataflow lifecycle management via CRD + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Aug 2020 | + +### Users & access policies management + +| | | +| --------------------- | ----- | +| Status | Done| +| Priority | High | +| Targeted Start date | November 2020 | + +The operator fully automates NiFi's user and access policies management. + +## Backlog + +### Monitoring via Prometheus + +| | | +| --------------------- | -------- | +| Status | To Do | +| Priority | High | +| Targeted Start date | Oct 2020 | + +The NiFi operator exposes NiFi JMX metrics to Prometheus. + +### Reacting on Alerts + +| | | +| --------------------- | ----- | +| Status | To Do | +| Priority | Low | +| Targeted Start date | - | + +The NiFi Operator acts as a **Prometheus Alert Manager**. It receives alerts defined in Prometheus, and creates actions based on Prometheus alert annotations. + +Currently, there are three actions expected : +- upscale cluster (add a new Node) +- downscale cluster (remove a Node) +- add additional disk to a Node + +### Seamless Istio mesh support + +| | | +| --------------------- | ----- | +| Status | To Do | +| Priority | Low | +| Targeted Start date | - | + +- Operator allows to use ClusterIP services instead of Headless, which still works better in case of Service meshes. +- To avoid too early nifi initialization, which might lead to unready sidecar container. The operator will use a small script to +mitigate this behaviour. All NiFi image can be used the only one requirement is an available **wget** command. +- To access a NiFi cluster which runs inside the mesh. Operator will supports creating Istio ingress gateways. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/2_setup/1_getting_started.md b/site/website/versioned_docs/version-v0.6.2/2_setup/1_getting_started.md new file mode 100644 index 000000000..324274d5b --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/2_setup/1_getting_started.md @@ -0,0 +1,157 @@ +--- +id: 1_getting_started +title: Getting Started +sidebar_label: Getting Started +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The operator installs the 1.12.1 version of Apache NiFi, can run on Minikube v0.33.1+ and **Kubernetes 1.16.0+**, and require **Helm 3**. + +:::info +The operator supports NiFi 1.11.0+ +::: + +As a pre-requisite it needs a Kubernetes cluster. Also, NiFi requires Zookeeper so you need to first have a Zookeeper cluster if you don't already have one. + +> We believe in the `separation of concerns` principle, thus the NiFi operator does not install nor manage Zookeeper. + +## Prerequisites + +### Install Zookeeper + +To install Zookeeper we recommend using the [Bitnami's Zookeeper chart](https://github.com/bitnami/charts/tree/master/bitnami/zookeeper). + +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +``` + +```bash +# You have to create the namespace before executing following command +helm install zookeeper bitnami/zookeeper \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set global.storageClass=standard \ + --set networkPolicy.enabled=true \ + --set replicaCount=3 +``` + +:::warning +Replace the `storageClass` parameter value with your own. +::: + +### Install cert-manager + +The NiFiKop operator uses `cert-manager` for issuing certificates to users and and nodes, so you'll need to have it setup in case you want to deploy a secured cluster with authentication enabled. + + + + +```bash +# Install the CustomResourceDefinitions and cert-manager itself +kubectl apply -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.2.0/cert-manager.yaml +``` + + + +```bash +# Install CustomResourceDefinitions first +kubectl apply --validate=false -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.2.0/cert-manager.crds.yaml + +# Add the jetstack helm repo +helm repo add jetstack https://charts.jetstack.io +helm repo update + +# You have to create the namespace before executing following command +helm install cert-manager \ + --namespace cert-manager \ + --version v1.2.0 jetstack/cert-manager +``` + + + + +## Installation + +## Installing with Helm + +You can deploy the operator using a Helm chart [Helm chart](https://github.com/Orange-OpenSource/nifikop/tree/master/helm): + +> To install an other version of the operator use `helm install --name=nifikop --namespace=nifi --set operator.image.tag=x.y.z orange-incubator/nifikop` + +In the case where you don't want to deploy the crds using helm (`--skip-crds`), you have to deploy manually the crds : + +```bash +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiregistryclients.yaml +``` + +Add the orange incubator repository : + +```bash + +helm repo add orange-incubator https://orange-kubernetes-charts-incubator.storage.googleapis.com/ +``` + +Now deploy the helm chart : + +```bash +# You have to create the namespace before executing following command +helm install nifikop \ + orange-incubator/nifikop \ + --namespace=nifi \ + --version 0.6.2 \ + --set image.tag=v0.6.2-release \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set namespaces={"nifi"} +``` + +:::note +Add the following parameter if you are using this instance to only deploy unsecured clusters : `--set certManager.enabled=false` +::: + +## Create custom storage class + +We recommend to use a **custom StorageClass** to leverage the volume binding mode `WaitForFirstConsumer` + +```bash +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: exampleStorageclass +parameters: + type: pd-standard +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +``` + +:::tip +Remember to set your NiFiCluster CR properly to use the newly created StorageClass. +::: + +## Deploy NiFi cluster + +And after you can deploy a simple NiFi cluster. + +```bash +# Add your zookeeper svc name to the configuration +kubectl create -n nifi -f config/samples/simplenificluster.yaml +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/1_gke.md b/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/1_gke.md new file mode 100644 index 000000000..c798f7443 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/1_gke.md @@ -0,0 +1,42 @@ +--- +id: 1_gke +title: Google Kubernetes Engine +sidebar_label: Google Kubernetes Engine +--- + +Follow these instructions to prepare a GKE cluster for NiFiKop + +1. Setup environment variables. + +```sh +export GCP_PROJECT= +export GCP_ZONE= +export CLUSTER_NAME= +``` + +2. Create a new cluster. + +```sh +gcloud container clusters create $CLUSTER_NAME \ + --cluster-version latest \ + --machine-type=n1-standard-1 \ + --num-nodes 4 \ + --zone $GCP_ZONE \ + --project $GCP_PROJECT +``` + +3. Retrieve your credentials for `kubectl`. + +```sh +cloud container clusters get-credentials $CLUSTER_NAME \ + --zone $GCP_ZONE \ + --project $GCP_PROJECT +``` + +4. Grant cluster administrator (admin) permissions to the current user. To create the necessary RBAC rules for NiFiKop, the current user requires admin permissions. + +```sh +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole=cluster-admin \ + --user=$(gcloud config get-value core/account) +``` diff --git a/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/2_minikube.md b/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/2_minikube.md new file mode 100644 index 000000000..a75e68998 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/2_setup/2_platform_setup/2_minikube.md @@ -0,0 +1,43 @@ +--- +id: 2_minikube +title: MiniKube +sidebar_label: MiniKube +--- + +Follow these instructions to prepare minikube for NiFiKop installation with sufficient resources to run NiFiKop and some basic applications. + +## Prerequisites + +- Administrative privileges are required to run minikube. + +## Installation steps + +1. Install the latest version of [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/), version 1.1.1 or later, and a [minikube hypervisor driver](https://kubernetes.io/docs/tasks/tools/install-minikube/#install-a-hypervisor). +2. If you’re not using the default driver, set your minikube hypervisor driver. + For example, if you installed the KVM hypervisor, set the vm-driver within the minikube configuration using the following command: + + ```sh + minikube config set vm-driver kvm2 + ``` +3. Start minikube with 16384 MB of memory and 4 CPUs. This example uses Kubernetes version 1.14.2. You can change the version to any Kubernetes version supported by NiFiKop by altering the --kubernetes-version value: + + ```sh + $ minikube start --memory=16384 --cpus=4 --kubernetes-version=v1.14.2 + ``` + +Depending on the hypervisor you use and the platform on which the hypervisor is run, minimum memory requirements vary. 16384 MB is sufficent to run NiFiKop. + +:::tip +If you don’t have enough RAM allocated to the minikube virtual machine, the following errors could occur: + - Image pull failures +- Healthcheck timeout failures +- Kubectl failures on the host +- General network instability of the virtual machine and the host +- Complete lock-up of the virtual machine +- Host NMI watchdog reboots +- One effective way to monitor memory usage in minikube: +```sh +minikube ssh +top +``` +::: diff --git a/site/website/versioned_docs/version-v0.6.2/2_setup/3_install/1_customizable_install_with_helm.md b/site/website/versioned_docs/version-v0.6.2/2_setup/3_install/1_customizable_install_with_helm.md new file mode 100644 index 000000000..1e455b8fe --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/2_setup/3_install/1_customizable_install_with_helm.md @@ -0,0 +1,193 @@ +--- +id: 1_customizable_install_with_helm +title: Customizable install with Helm +sidebar_label: Customizable install with Helm +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Prerequisites + +- Perform any necessary [plateform-specific setup](../2_platform_setup/1_gke.md) +- [Install a Helm client](https://github.com/helm/helm#install) with a version higher than 3 + +## Introduction + +This Helm chart install NiFiKop the Orange's Nifi Kubernetes operator to create/configure/manage NiFi +clusters in a Kubernetes Namespace. + +It will use Custom Ressources Definition CRDs: + +- `nificlusters.nifi.orange.com`, +- `nifiusers.nifi.orange.com`, +- `nifiusergroups.nifi.orange.com`, +- `nifiregistryclients.nifi.orange.com`, +- `nifiparametercontexts.nifi.orange.com`, +- `nifidataflows.nifi.orange.com`, + +### Configuration + +The following tables lists the configurable parameters of the NiFi Operator Helm chart and their default values. + +| Parameter | Description | Default | +|----------------------------------|--------------------------------------------------|-------------------------------------------| +| `image.repository` | Image | `orangeopensource/nifikop` | +| `image.tag` | Image tag | `v0.6.2-release` | +| `image.pullPolicy` | Image pull policy | `Always` | +| `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | +| `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | +| `certManager.enabled` | Enable cert-manager integration | `true` | +| `rbacEnable` | If true, create & use RBAC resources | `true` | +| `resources` | Pod resource requests & limits | `{}` | +| `metricService` | deploy service for metrics | `false` | +| `debug.enabled` | activate DEBUG log level | `false` | +| `certManager.clusterScoped` | If true setup cluster scoped resources | `false` | +| `namespaces` | List of namespaces where Operator watches for custom resources. Make sure the operator ServiceAccount is granted `get` permissions on this `Node` resource when using limited RBACs.| `""` i.e. all namespaces | +| `nodeSelector` | Node selector configuration for operator pod | `{}` | +| `affinity` | Node affinity configuration for operator pod | `{}` | +| `tolerations` | Toleration configuration for operator pod | `{}` | +| `serviceAccount.create` | Whether the SA creation is delegated to the chart or not | `true` | +| `serviceAccount.name` | Name of the SA used for NiFiKop deployment | release name | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install nifikop \ + orange-incubator/nifikop \ + -f values.yaml +``` + +### Installing the Chart + +:::important Skip CRDs +In the case where you don't want to deploy the crds using helm (`--skip-crds`) you need to deploy manually the crds beforehand: + +```bash +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/nifikop/master/config/crd/bases/nifi.orange.com_nifiregistryclients.yaml +``` + +::: + + + + +```bash +helm install nifikop orange-incubator/nifikop \ + --dry-run \ + --debug.enabled \ + --set debug.enabled=true \ + --set namespaces={"nifikop"} +``` + + + +```bash +helm install orange-incubator/nifikop +``` + + + + +```bash +helm install nifikop orange-incubator/nifikop --set namespaces={"nifikop"} +``` + + + +> the `--replace` flag allow you to reuses a charts release name + +### Listing deployed charts + +``` +helm list +``` + +### Get Status for the helm deployment + +``` +helm status nifikop +``` + +## Uninstaling the Charts + +If you want to delete the operator from your Kubernetes cluster, the operator deployment +should be deleted. + +``` +helm del nifikop +``` + +The command removes all the Kubernetes components associated with the chart and deletes the helm release. + +:::tip +The CRD created by the chart are not removed by default and should be manually cleaned up (if required) +::: + +Manually delete the CRD: + +``` +kubectl delete crd nificlusters.nifi.orange.com +kubectl delete crd nifiusers.nifi.orange.com +kubectl delete crd nifiusergroups.nifi.orange.com +kubectl delete crd nifiregistryclients.nifi.orange.com +kubectl delete crd nifiparametercontexts.nifi.orange.com +kubectl delete crd nifidataflows.nifi.orange.com +``` + +:::warning +If you delete the CRD then +It will delete **ALL** Clusters that has been created using this CRD!!! +Please never delete a CRD without very good care +::: + +Helm always keeps records of what releases happened. Need to see the deleted releases ? + +```bash +helm list --deleted +``` + +Need to see all of the releases (deleted and currently deployed, as well as releases that +failed) ? + +```bash +helm list --all +``` + +Because Helm keeps records of deleted releases, a release name cannot be re-used. (If you really need to re-use a +release name, you can use the `--replace` flag, but it will simply re-use the existing release and replace its +resources.) + +Note that because releases are preserved in this way, you can rollback a deleted resource, and have it re-activate. + +To purge a release + +```bash +helm delete --purge nifikop +``` + +## Troubleshooting + +### Install of the CRD + +By default, the chart will install the CRDs, but this installation is global for the whole +cluster, and you may want to not modify the already deployed CRDs. + +In this case there is a parameter to say to not install the CRDs : + +``` +$ helm install --name nifikop ./helm/nifikop --set namespaces={"nifikop"} --skip-crds +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/1_nodes_configuration.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/1_nodes_configuration.md new file mode 100644 index 000000000..0260901b1 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/1_nodes_configuration.md @@ -0,0 +1,9 @@ +--- +id: 1_nodes_configuration +title: Nodes configuration +sidebar_label: Nodes configuration +--- + +:::warning +WIP +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/2_cluster_scaling.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/2_cluster_scaling.md new file mode 100644 index 000000000..ead57409a --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/2_cluster_scaling.md @@ -0,0 +1,237 @@ +--- +id: 2_cluster_scaling +title: Cluster Scaling +sidebar_label: Cluster Scaling +--- + +This tasks shows you how to perform a gracefull cluster scale up and scale down. + +## Before you begin + +- Setup NiFiKop by following the instructions in the [Installation guide](../../2_setup/1_getting_started.md). +- Deploy the [Simple NiFi](../../2_setup/1_getting_started.md#easy-way-installing-with-helm) sample cluster. +- Review the [Node](../../5_references/1_nifi_cluster/4_node.md) references doc. + +## About this task + +The [Simple NiFi](../../2_setup/1_getting_started.md#easy-way-installing-with-helm) example consists of a three nodes NiFi cluster. +A node decommission must follow a strict procedure, described in the [NiFi documentation](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes) : + +1. Disconnect the node +2. Once disconnect completes, offload the node. +3. Once offload completes, delete the node. +4. Once the delete request has finished, stop/remove the NiFi service on the host. + + +For the moment, we have implemented it as follows in the operator : + + 1. Disconnect the node + 2. Once disconnect completes, offload the node. + 3. Once offload completes, delete the pod. + 4. Once the pod deletion completes, delete the node. + 5. Once the delete request has finished, remove the node from the NifiCluster status. + +In addition, we have a regular check that ensure that all nodes have been removed. + +In this task, you will first perform a scale up, in adding an new node. Then, you will remove another node that the one created, and observe the decommission's steps. + +## Scale up : Add a new node + +For this task, we will simply add a node with the same configuration than the other ones, if you want to know more about how to add a node with an other configuration let's have a look to the [Node configuration](./1_nodes_configuration.md) documentation page. + +1. Add and run a dataflow as the example : + +![Scaling dataflow](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaling_dataflow.png) + +2. Add a new node to the list of `NifiCluster.Spec.Nodes` field, by following the [Node object definition](../../5_references/1_nifi_cluster/4_node.md) documentation: + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + zkAddress: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.12.1" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" + - id: 2 + nodeConfigGroup: "default_group" +# >>>> START: The new node + - id: 25 + nodeConfigGroup: "default_group" +# <<<< END + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +:::important +**Note :** The `Node.Id` field must be unique in the `NifiCluster.Spec.Nodes` list. +::: + +3. Apply the new `NifiCluster` configuration : + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +4. You should now have the following resources into kubernetes : + +```console +kubectl get pods,configmap,pvc -l nodeId=25 +NAME READY STATUS RESTARTS AGE +pod/simplenifi-25-nodem5jh4 1/1 Running 0 11m + +NAME DATA AGE +configmap/simplenifi-config-25 7 11m + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/simplenifi-25-storagehwn24 Bound pvc-7da86076-728e-11ea-846d-42010a8400f2 10Gi RWO standard 11m +``` + +And if you go on the NiFi UI, in the cluster administration page : + +![Scale up, cluster list](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_cluster_list.png) + +5. You now have data on the new node : + +![Scale up, cluster distribution](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_distribution.png) + +## Scaledown : Gracefully remove node + +For this task, we will simply remove a node and look at that the decommission's steps. + +1. Remove the node from the list of `NifiCluster.Spec.Nodes` field : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + headlessServiceEnabled: true + zkAddresse: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.11.3" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" +# >>>> START: node removed +# - id: 2 +# nodeConfigGroup: "default_group" +# <<<< END + - id: 25 + nodeConfigGroup: "default_group" + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +2. Apply the new `NifiCluster` configuration : + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +3. You can follow the node's action step status in the `NifiCluster.Status` description : + +```console +kubectl describe nificluster simplenifi + +... +Status: + Nodes State: + ... + 2: + Configuration State: ConfigInSync + Graceful Action State: + Action State: GracefulDownscaleRequired + Error Message: + ... +... +``` + +:::tip +The list of decommision's step and their corresponding value for the `Nifi Cluster.Status.Node State.Graceful ActionState.ActionStep` field is described into the [Node State page](../../5_references/1_nifi_cluster/5_node_state.md#actionstep) +::: + +4. Once the scaledown successfully performed, you should have the data offloaded on the other nodes, and the node state removed from the `NifiCluster.Status.NodesState` list : + +:::warning +Keep in mind that the [`NifiCluster.Spec.nifiClusterTaskSpec.retryDurationMinutes`](../../5_references/1_nifi_cluster/1_nifi_cluster.md#nificlustertaskspec) should be long enough to perform the whole procedure, or you will have some rollback and retry loop. +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/3_external_dns.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/3_external_dns.md new file mode 100644 index 000000000..27c33dc5e --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/1_nifi_cluster/3_external_dns.md @@ -0,0 +1,9 @@ +--- +id: 3_external_dns +title: External DNS +sidebar_label: External DNS +--- + +:::warning +WIP +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/1_ssl.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/1_ssl.md new file mode 100644 index 000000000..2056a8fdf --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/1_ssl.md @@ -0,0 +1,159 @@ +--- +id: 1_ssl +title: Securing NiFi with SSL +sidebar_label: SSL +--- + +The `NiFi operator` makes securing your NiFi cluster with SSL. You may provide your own certificates, or instruct the operator to create them for from your cluster configuration. + +Below this is an example configuration required to secure your cluster with SSL : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + managedAdminUsers: + - identity : "alexandre.guitton@orange.com" + name: "aguitton" + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + ... + ... + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +- `managedAdminUsers` : list of users account which will be configured as admin into NiFi cluster, please check [](../4_nifi_user_group#managed-groups-for-simple-setup) for more information. +- `readOnlyConfig.nifiProperties.webProxyHosts` : A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties) + +If `listenersConfig.sslSecrets.create` is set to `false`, the operator will look for the secret at `listenersConfig.sslSecrets.tlsSecretName` and expect these values : + +| key | value | +|-----|-------| +| caCert | The CA certificate | +| caKey | The CA private key | +| clientCert | A client certificate (this will be used by operator for NiFI operations) | +| clientKey | The private key for clientCert | + +## Using an existing Issuer + +As described in the [Reference section](../../5_references/1_nifi_cluster/6_listeners_config.md#sslsecrets), instead of using a self-signed certificate as CA, you can use an existing one. +In order to do so, you only have to refer it into your `Spec.ListenerConfig.SslSecrets.IssuerRef` field. + +### Example : Let's encrypt + +Let's say you have an existing DNS server, with [external dns](https://github.com/kubernetes-sigs/external-dns) deployed into your cluster's namespace. +You can easily use Let's encrypt as authority for your certificate. + +To do this, you have to : + +1. Create an issuer : + +```yaml +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: letsencrypt-staging +spec: + acme: + # You must replace this email address with your own. + # Let's Encrypt will use this to contact you about expiring + # certificates, and issues related to your account. + email: + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource used to store the account's private key. + name: example-issuer-account-key + # Add a single challenge solver, HTTP01 using nginx + solvers: + - http01: + ingress: + ingressTemplate: + metadata: + annotations: + "external-dns.alpha.kubernetes.io/ttl": "5" +``` + +2. Setup External dns and correctly create your issuer into your cluster configuration : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + clusterSecure: true + siteToSiteSecure: true + ... + listenersConfig: + clusterDomain: + useExternalDNS: true + ... + sslSecrets: + tlsSecretName: "test-nifikop" + create: true + issuerRef: + name: letsencrypt-staging + kind: Issuer +``` + +## Create SSL credentials + +You may use `NifiUser` resource to create new certificates for your applications, allowing them to query your Nifi cluster. + +To create a new client you will need to generate new certificates sign by the CA. The operator can automate this for you using the `NifiUser` CRD : + +```console +cat << EOF | kubectl apply -n nifi -f - +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiUser +metadata: + name: example-client + namespace: nifi +spec: + clusterRef: + name: nifi + secretName: example-client-secret +EOF +``` + +This will create a user and store its credentials in the secret `example-client-secret`. The secret contains these fields : + +| key | value | +|-----|-------| +| ca.crt | The CA certificate | +| tls.crt | The user certificate | +| tls.key | The user private key | + +You can then mount these secret to your pod. Alternatively, you can write them to your local machine by running: + +```console +kubectl get secret example-client-secret -o jsonpath="{['data']['ca\.crt']}" | base64 -d > ca.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.crt']}" | base64 -d > tls.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.key']}" | base64 -d > tls.key +``` + +The operator can also include a Java keystore format (JKS) with your user secret if you'd like. Add `includeJKS`: `true` to the `spec` like shown above, and then the user-secret will gain these additional fields : + +| key | value | +|-----|-------| +| tls.jks | The java keystore containing both the user keys and the CA (use this for your keystore AND truststore) | +| pass.txt | The password to decrypt the JKS (this will be randomly generated) | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/2_authentication/1_oidc.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/2_authentication/1_oidc.md new file mode 100644 index 000000000..0c9f8826c --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/2_security/2_authentication/1_oidc.md @@ -0,0 +1,42 @@ +--- +id: 1_oidc +title: OpenId Connect +sidebar_label: OpenId Connect +--- + +To enable authentication via OpenId Connect refering to [NiFi Administration guide](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html) required some configuration into `nifi.properties`. + +In addition and to ensure multiple identity provider support, we recommended to add the following configuration to your `nifi.properties` : + +```sh +nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? +nifi.security.identity.mapping.value.dn=$1 +nifi.security.identity.mapping.transform.dn=NONE +``` + +To perform this with `NiFiKop` you just have to configure the `Spec.NifiProperties.OverrideConfigs` field with your OIDC configuration, for example : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + # Additionnals nifi.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + nifi.security.user.oidc.discovery.url= + nifi.security.user.oidc.client.id= + nifi.security.user.oidc.client.secret= + nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? + nifi.security.identity.mapping.value.dn=$1 + nifi.security.identity.mapping.transform.dn=NONE + ... + ... +... +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/3_nifi_dataflow.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/3_nifi_dataflow.md new file mode 100644 index 000000000..0cba5e550 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/3_nifi_dataflow.md @@ -0,0 +1,124 @@ +--- +id: 3_nifi_dataflow +title: Provisioning NiFi Dataflows +sidebar_label: NiFi Dataflows +--- + +You can create NiFi dataflows either : + +* directly against the cluster through its REST API (using UI or some home made scripts), or +* via the `NifiDataflow` CRD. + +If you want more details about the design, just have a look on the [design page](../1_concepts/2_design_principes.md#dataflow-lifecycle-management) + +To deploy a [NifiDataflow] you have to start by deploying a [NifiRegistryClient] because **NiFiKop** manages dataflow using the [NiFi Registry feature](https://nifi.apache.org/registry). + +Below is an example of [NifiRegistryClient] : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiRegistryClient +metadata: + name: registry-client-example + namespace: nifikop +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Registry client managed by NiFiKop" + uri: "http://nifi.hostname.com:18080" +``` + +Once you have deployed your [NifiRegistryClient], you have the possibility of defining a configuration that you will apply to your [NifiDataflow]. + +This configuration is defined using the [NifiParameterContext] CRD, which NiFiKop will convert into a [Parameter context](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#parameter-contexts). + + +Below is an example of [NifiParameterContext] : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle + namespace: demo +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + value: toto + description: toto +``` + +As you can see, in the [NifiParameterContext] you can refer to some secrets that will be converted into [sensitive parameter](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#using-parameters-with-sensitive-properties). + +Here is an example of secret that you can create that will be used by the configuration above : + +```console +kubectl create secret generic secret-params \ + --from-literal=secret1=yop \ + --from-literal=secret2=yep \ + -n nifikop +``` + +:::warning +As a sensitive value cannot be retrieved through the Rest API, to update the value of a sensitive parameter, you have to : + +- remove it from the secret +- wait for the next loop +- insert the parameter with the new value inside the secret + +or you can simply create a new [NifiParameterContext] and refer it into your [NifiDataflow]. +::: + +You can now deploy your [NifiDataflow] by referencing the previous objects : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + runOnce: false + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: registry-client-example + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: demo + updateStrategy: drain +``` + +To find details about the versioned flow information required check the [official documentation](https://nifi.apache.org/docs/nifi-registry-docs/index.html) + +You have two modes of control from your dataflow by the operator : + +1 - `Spec.RunOnce == true` : The operator will deploy the dataflow as described in the resource, run it once, and never control it again (unless you change the field to `false`). It is useful when you want to deploy your dataflow in a dev environment, and you want to update the dataflow. + +2 - `Spec.RunOnce == false` : The operator will deploy and ensure the dataflow lifecycle, it will avoid all manual modification directly from the Cluster (e.g remove the process group, remove the versioning, update the parent process group, make some local changes ...). If you want to perform update, rollback or stuff like this, you have to simply update the [NifiDataflow] resource. + +:::important +More information about `Spec.UpdateStrategy` [here](../5_references/5_nifi_dataflow.md#dataflowupdatestrategy) +::: + +[NifiDataflow]: ../5_references/5_nifi_dataflow.md +[NifiRegistryClient]: ../5_references/3_nifi_registry_client.md +[NifiParameterContext]: ../5_references/4_nifi_parameter_context.md \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/3_tasks/4_nifi_user_group.md b/site/website/versioned_docs/version-v0.6.2/3_tasks/4_nifi_user_group.md new file mode 100644 index 000000000..0ecaa7da0 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/3_tasks/4_nifi_user_group.md @@ -0,0 +1,168 @@ +--- +id: 4_nifi_user_group +title: Provisioning NiFi Users and Groups +sidebar_label: NiFi Users and Groups +--- + +## User management + +The `NifiUser` resource was already introduced for the [SSL credentials](./2_security/1_ssl.md#create-ssl-credentials) concerns. +What we are covering here is the NiFi user management part introduced in this resource. + +When you create a `NifiUser` resource the operator will : + +1. Try to check if a user already exists with the same name on the NiFi cluster, if it does, the operator will set [NifiUser.Status.Id](./2_security/1_ssl.md#create-ssl-credentials) to bind it with the kubernetes resource. +2. If no user is found, the operator will create and manage it (i.e it will ensure the synchronisation with the NiFi Cluster). + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiUser +metadata: + name: aguitton +spec: + # identity field is use to define the user identity on NiFi cluster side, + # it use full when the user's name doesn't suite with Kubernetes resource name. + identity: alexandre.guitton@orange.com + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # Whether or not the the operator also include a Java keystore format (JKS) with you secret + includeJKS: false + # Whether or not a certificate will be created for this user. + createCert: false + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: component + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page : + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /data + # componentType is used if the type is "component", it's allow to define the kind of component on which is the + # access policy + componentType: "process-groups" + # componentId is used if the type is "component", it's allow to define the id of the component on which is the + # access policy + componentId: "" +``` + +By default the user name that will be used is the name of the resource. + +But as there are some constraints on this name (e.g [RFC 1123](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names)) that doesn't match with those applied on NiFi, you can override it with the `NifiUser.Spec.Identity` field which is more permissive. +In the example above the kubernetes resource name will be `aguitton` but the NiFi use created on the cluster will be `alexandre.guitton@orange.com`. + +In the case the user will not authenticate himself using TLS authentication, the operator doesn't have to create a certificate, so just set `NifiUser.Spec.CreateCert` to false. + +For each user, you have the ability to define a list of [AccessPolicies](../5_references/2_nifi_user.md#accesspolicy) to give a list of access to your user. +In the example above we are giving to user `alexandre.guitton@orange.com` the right to view metadata et content for the root process group in flowfile queues in outbound connections and through provenance events. + +## UserGroup management + +To simplify the access management Apache NiFi allows to define groups containing a list of users, on which we apply a list of access policies. +This part is supported by the operator using the `NifiUserGroup` resource : + + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # contains the list of reference to NifiUsers that are part to the group. + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local +# namespace: nifikop + - name: nc-controller.nifikop.mgt.cluster.local + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: global + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page : + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /counters +# # componentType is used if the type is "component", it's allow to define the kind of component on which is the +# # access policy +# componentType: "process-groups" +# # componentId is used if the type is "component", it's allow to define the id of the component on which is the +# # access policy +# componentId: "" +``` + +When you create a `NifiUserGroup` resource, the operator will create and manage a group named `${resource namespace}-${resource name}` in Nifi. +To declare the users that are part of this group, you just have to declare them in the [NifiUserGroup.UsersRef](../5_references/6_nifi_usergroup.md#userreference) field. + +:::important +The [NifiUserGroup.UsersRef](../5_references/6_nifi_usergroup.md#userreference) requires to declare the name and namespace of a `NifiUser` resource, so it is previously required to declare the resource. + +It's required to create the resource even if the user is already declared in NiFi Cluster (In that case the operator will just sync the kubernetes resource). +::: + +Like for `NifiUser` you can declare a list of [AccessPolicies](../5_references/2_nifi_user.md#accesspolicy) to give a list of access to your user. + +In the example above we are giving to users `nc-0-node.nc-headless.nifikop.svc.cluster.local` and `nc-controller.nifikop.mgt.cluster.local` the right to view the counters informations. + +## Managed groups for simple setup + +In some case these two features could be heavy to define, for example when you have 10 dataflows with one cluster for each of them, it will lead in a lot of `.yaml` files ... +To simplify this, we implement in the operator 2 `managed groups` : + +- **Admins :** a group giving access to everything on the NiFi Cluster, +- **Readers :** a group giving access as viewer on the NiFi Cluster. + +You can directly define the list of users who belong to each of them in the `NifiCluster.Spec` field : + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +metadata: + name: mynifi +spec: + ... + oneNifiNodePerNode: false + # + propagateLabels: true + managedAdminUsers: + - identity : "alexandre.guitton@orange.com" + name: "aguitton" + - identity : "nifiuser@orange.com" + name: "nifiuser" + managedReaderUsers: + - identity : "toto@orange.com" + name: "toto" + ... +``` + +In this example the operator will create and manage 3 `NifiUsers` : + +- **aguitton**, with the identity : `alexandre.guitton@orange.com` +- **nifiuser**, with the identity : `nifiuser@orange.com` +- **toto**, with the identity : `toto@orange.com` + +And create and manage two groups : + +- **managed-admins :** that will contain 3 users (**aguitton**, **nifiuser**, **nc-controller.nifikop.mgt.cluster.local** which is the controller user). +- **managed-readers :** that will contain 1 user (**toto**) + +And the rest of the stuff will be reconciled and managed as described for `NifiUsers` and `NifiUserGroups`. + +:::note +There is one more group that is created and managed by the operator, this is the **managed-nodes** group, for each node a `NifiUser` is created, and we automatically add them to this group to give them the right list of accesses. + +To get the list of managed groups just check the list of `NifiUserGroup` : + +```console +kubectl get -n nifikop nifiusergroups.nifi.orange.com +NAME AGE +managed-admins 6d7h +managed-nodes 6d7h +managed-readers 6d7h +``` +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/4_examples/1_simple_nifi_cluster.md b/site/website/versioned_docs/version-v0.6.2/4_examples/1_simple_nifi_cluster.md new file mode 100644 index 000000000..224511cdb --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/4_examples/1_simple_nifi_cluster.md @@ -0,0 +1,5 @@ +--- +id: 1_simple_nifi_cluster +title: Simple NiFi cluster +sidebar_label: Simple NiFi cluster +--- \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/1_nifi_cluster.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/1_nifi_cluster.md new file mode 100644 index 000000000..b632e8eb9 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/1_nifi_cluster.md @@ -0,0 +1,163 @@ +--- +id: 1_nifi_cluster +title: NiFi cluster +sidebar_label: NiFi cluster +--- + +`NifiCluster` describes the desired state of the NiFi cluster we want to setup through the operator. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + zkAddress: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.11.3" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 1 + nodeConfigGroup: "default_group" + - id: 2 + nodeConfigGroup: "default_group" + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + externalServices: + - name: "clusterip" + spec: + type: ClusterIP + portConfigs: + - port: 8080 + internalListenerName: "http" + serviceAnnotations: + toto: tata +``` + +## NifiCluster + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects users must create.|No|nil| +|spec|[NifiClusterSpec](#nificlusterspec)|defines the desired state of NifiCluster.|No|nil| +|status|[NifiClusterStatus](#nificlusterstatus)|defines the observed state of NifiCluster.|No|nil| + +## NifiClusterSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|service|[ServicePolicy](#servicepolicy)| defines the policy for services owned by NiFiKop operator. |No| - | +|pod|[PodPolicy](#podpolicy)| defines the policy for pod owned by NiFiKop operator. |No| - | +|zkAddress|string| specifies the ZooKeeper connection string in the form hostname:port where host and port are those of a Zookeeper server.|Yes|""| +|zkPath|string| specifies the Zookeeper chroot path as part of its Zookeeper connection string which puts its data under same path in the global ZooKeeper namespace.|Yes|"/"| +|initContainerImage|string| can override the default image used into the init container to check if ZoooKeeper server is reachable.. |Yes|"busybox"| +|initContainers|\[ \]string| defines additional initContainers configurations. |No|\[ \]| +|clusterImage|string| can specify the whole nificluster image in one place. |No|""| +|oneNifiNodePerNode|boolean|if set to true every nifi node is started on a new node, if there is not enough node to do that it will stay in pending state. If set to false the operator also tries to schedule the nifi node to a unique node but if the node number is insufficient the nifi node will be scheduled to a node where a nifi node is already running.|Yes| nil | +|propagateLabels|boolean| - |Yes|false| +|managedAdminUsers|\[ \][ManagedUser](#managedusers)| contains the list of users that will be added to the managed admin group (with all rights). |No|[]| +|managedReaderUsers|\[ \][ManagedUser](#managedusers)| contains the list of users that will be added to the managed admin group (with all rights). |No|[]| +|readOnlyConfig|[ReadOnlyConfig](./2_read_only_config.md)| specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node.|No| nil | +|nodeConfigGroups|map\[string\][NodeConfig](./3_node_config.md)| specifies multiple node configs with unique name|No| nil | +|nodes|\[ \][Node](./3_node_config.md)| specifies the list of cluster nodes, all node requires an image, unique id, and storageConfigs settings|Yes| nil +|disruptionBudget|[DisruptionBudget](#disruptionbudget)| defines the configuration for PodDisruptionBudget.|No| nil | +|ldapConfiguration|[LdapConfiguration](#ldapconfiguration)| specifies the configuration if you want to use LDAP.|No| nil | +|nifiClusterTaskSpec|[NifiClusterTaskSpec](#nificlustertaskspec)| specifies the configuration of the nifi cluster Tasks.|No| nil | +|listenersConfig|[ListenersConfig](./6_listeners_config.md)| specifies nifi's listener specifig configs.|Yes| - | +|sidecarConfigs|\[ \][Container](https://godoc.org/k8s.io/api/core/v1#Container)|Defines additional sidecar configurations. [Check documentation for more informations]| +|externalServices|\[ \][ExternalServiceConfigs](./7_external_service_config.md)| specifies settings required to access nifi externally.|No| - | + +## NifiClusterStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|nodesState|map\[string\][NodeState](./5_node_state.md)|Store the state of each nifi node.|No| - | +|State|[ClusterState](#clusterstate)|Store the state of each nifi node.|Yes| - | +|rootProcessGroupId|string|contains the uuid of the root process group for this cluster.|No| - | + +## ServicePolicy + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|headlessEnabled|boolean| specifies if the cluster should use headlessService for Nifi or individual services using service per nodes may come an handy case of service mesh.|Yes|false| +|annotations|map\[string\]string|Annotations specifies the annotations to attach to services the NiFiKop operator creates|No|-| + +## PodPolicy + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|annotations|map\[string\]string|Annotations specifies the annotations to attach to pods the NiFiKop operator creates|No|-| + +## ManagedUsers + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|create|bool| if set to true, will create a podDisruptionBudget.|No| - | +|name|string|name field is use to name the NifiUser resource, if not identity is provided it will be used to name the user on NiFi cluster side.|Yes| - | + +## DisruptionBudget + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|identity|string|identity field is use to define the user identity on NiFi cluster side, it use full when the user's name doesn't suite with Kubernetes resource name.|No| - | +|budget|string| the budget to set for the PDB, can either be static number or a percentage.|Yes| - | + +## LdapConfiguration + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|enabled|boolean| if set to true, we will enable ldap usage into nifi.properties configuration.|No| false | +|url|string| space-separated list of URLs of the LDAP servers (i.e. ldap://${hostname}:${port}).|No| "" | +|searchBase|string| base DN for searching for users (i.e. CN=Users,DC=example,DC=com).|No| "" | +|searchFilter|string| Filter for searching for users against the 'User Search Base'. (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.|No| "" | + +## NifiClusterTaskSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|retryDurationMinutes|int| describes the amount of time the Operator waits for the task.|Yes| 5 | + +## ClusterState + +|Name|Value|Description| +|-----|----|------------| +|NifiClusterInitializing|ClusterInitializing|states that the cluster is still in initializing stage| +|NifiClusterInitialized|ClusterInitialized|states that the cluster is initialized| +|NifiClusterReconciling|ClusterReconciling|states that the cluster is still in reconciling stage| +|NifiClusterRollingUpgrading|ClusterRollingUpgrading|states that the cluster is rolling upgrading| +|NifiClusterRunning|ClusterRunning|states that the cluster is in running state| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/2_read_only_config.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/2_read_only_config.md new file mode 100644 index 000000000..0dfbeebdc --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/2_read_only_config.md @@ -0,0 +1,190 @@ +--- +id: 2_read_only_config +title: Read only configurations +sidebar_label: Read only configurations +--- + +ReadOnlyConfig object specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node. + +```yaml +readOnlyConfig: + # MaximumTimerDrivenThreadCount define the maximum number of threads for timer driven processors available to the system. + maximumTimerDrivenThreadCount: 30 + # Logback configuration that will be applied to the node + logbackConfig: + # logback.xml configuration that will replace the one produced based on template + replaceConfigMap: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # logback.xml configuration that will replace the one produced based on template and overrideConfigMap + replaceSecretConfig: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # NifiProperties configuration that will be applied to the node. + nifiProperties: + # Additionnals nifi.properties configuration that will override the one produced based on template and + # configuration + overrideConfigMap: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop. + # Additionnals nifi.properties configuration that will override the one produced based + # on template, configurations, overrideConfigMap and overrideConfigs. + overrideSecretConfig: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Additionnals nifi.properties configuration that will override the one produced based + # on template, configurations and overrideConfigMap + overrideConfigs: | + nifi.ui.banner.text=NiFiKop by Orange + # A comma separated list of allowed HTTP Host header values to consider when NiFi + # is running securely and will be receiving requests to a different host[:port] than it is bound to. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties + # webProxyHosts: + # Nifi security client auth + needClientAuth: false + # Indicates which of the configured authorizers in the authorizers.xml file to use + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration + # authorizer: + # ZookeeperProperties configuration that will be applied to the node. + zookeeperProperties: + # # Additionnals zookeeeper.properties configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnals zookeeeper.properties configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # Additionnals zookeeper.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + initLimit=15 + autopurge.purgeInterval=24 + syncLimit=5 + tickTime=2000 + dataDir=./state/zookeeper + autopurge.snapRetainCount=30 + # BootstrapProperties configuration that will be applied to the node. + bootstrapProperties: + # # Additionnals bootstrap.properties configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: bootstrap.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnals bootstrap.properties configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: bootstrap.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # JVM memory settings + nifiJvmMemory: "512m" + # Additionnals bootstrap.properties configuration that will override the one produced based + # on template and configurations. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties + overrideConfigs: | + # java.arg.4=-Djava.net.preferIPv4Stack=true +``` + +## ReadOnlyConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|maximumTimerDrivenThreadCount|int32|define the maximum number of threads for timer driven processors available to the system.|No|nil| +|nifiProperties|[NifiProperties](#nifiproperties)|nifi.properties configuration that will be applied to the node.|No|nil| +|zookeeperProperties|[ZookeeperProperties](#zookeeperproperties)|zookeeper.properties configuration that will be applied to the node.|No|nil| +|bootstrapProperties|[BootstrapProperties](#bootstrapproperties)|bootstrap.conf configuration that will be applied to the node.|No|nil| +|logbackConfig|[LogbackConfig](#logbackconfig)|logback.xml configuration that will be applied to the node.|No|nil| +|bootstrapNotificationServicesConfig|[BootstrapNotificationServices](#bootstrapnotificationservices)|bootstrap_notification_services.xml configuration that will be applied to the node.|No|nil| + +## NifiProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals nifi.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals nifi.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals nifi.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|webProxyHosts|\[ \]string| A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties)|No|""| +|needClientAuth|boolean|Nifi security client auth.|No|false| +|authorizer|string|Indicates which of the configured authorizers in the authorizers.xml file to use [authorizer-configuration](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration)|No|"managed-authorizer"| + + +## ZookeeperProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals zookeeper.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals zookeeper.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals zookeeper.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| + +## BootstrapProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals bootstrap.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals bootstrap.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals bootstrap.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|NifiJvmMemory|string|JVM memory settings.|No|"512m"| + +## LogbackConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|logback.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|logback.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## BootstrapNotificationServicesConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## ConfigmapReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the configmap that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the configmap that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| + +## SecretConfigReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the secret that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the secret that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/3_node_config.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/3_node_config.md new file mode 100644 index 000000000..065ea4d58 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/3_node_config.md @@ -0,0 +1,79 @@ +--- +id: 3_node_config +title: Node configuration +sidebar_label: Node configuration +--- + +NodeConfig defines the node configuration + +```yaml + default_group: + # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties + provenanceStorage: "10 GB" + #RunAsUser define the id of the user to run in the Nifi image + # +kubebuilder:validation:Minimum=1 + runAsUser: 1000 + # Set this to true if the instance is a node in a cluster. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup + isNode: true + # Docker image used by the operator to create the node associated + # https://hub.docker.com/r/apache/nifi/ +# image: "apache/nifi:1.11.2" + # nodeAffinity can be specified, operator populates this value if new pvc added later to node + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity +# nodeAffinity: + # imagePullPolicy define the pull policy for NiFi cluster docker image + imagePullPolicy: IfNotPresent + # storageConfigs specifies the node related configs + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi +``` + +## NodeConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|provenanceStorage|string|provenanceStorage allow to specify the maximum amount of data provenance information to store at a time: [write-ahead-provenance-repository-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties)|No|"8 GB"| +|runAsUser|int64|define the id of the user to run in the Nifi image|No|1000| +|fsGroup|int64|define the id of the group for each volumes in Nifi image|No|1000| +|isNode|boolean|Set this to true if the instance is a node in a cluster: [basic-cluster-setup](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup)|No|true| +|image|string| Docker image used by the operator to create the node associated. [Nifi docker registry](https://hub.docker.com/r/apache/nifi/)|No|""| +|imagePullPolicy|[PullPolicy](https://godoc.org/k8s.io/api/core/v1#PullPolicy)| define the pull policy for NiFi cluster docker image.)|No|""| +|nodeAffinity|string| operator populates this value if new pvc added later to node [node-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity)|No|nil| +|storageConfigs|\[ \][StorageConfig](#storageconfig)|specifies the node related configs.|No|nil| +|serviceAccountName|string|specifies the serviceAccount used for this specific node.|No|"default"| +|resourcesRequirements|[ResourceRequirements](https://godoc.org/k8s.io/api/core/v1#ResourceRequirements)| works exactly like Container resources, the user can specify the limit and the requests through this property [manage-compute-resources-container](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/).|No|nil| +|imagePullSecrets|\[ \][LocalObjectReference](https://godoc.org/k8s.io/api/core/v1#TypedLocalObjectReference)|specifies the secret to use when using private registry.|No|nil| +|nodeSelector|map\[string\]string|nodeSelector can be specified, which set the pod to fit on a node [nodeselector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)|No|nil| +|tolerations|\[ \][Toleration](https://godoc.org/k8s.io/api/core/v1#Toleration)|tolerations can be specified, which set the pod's tolerations [taint-and-toleration](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts).|No|nil| +|nodeAnnotations|map\[string\]string|Additionnal annotation to attach to the pod associated [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set).|No|nil| + +## StorageConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the storage config, used to name PV to reuse into sidecars for example.|Yes| - | +|mountPath|string|Path where the volume will be mount into the main nifi container inside the pod.|Yes| - | +|pvcSpec|[PersistentVolumeClaimSpec](https://godoc.org/k8s.io/api/core/v1#PersistentVolumeClaimSpec)|Kubernetes PVC spec. [create-a-persistentvolumeclaim](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim).|Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/4_node.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/4_node.md new file mode 100644 index 000000000..419bbb4a1 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/4_node.md @@ -0,0 +1,60 @@ +--- +id: 4_node +title: Node +sidebar_label: Node +--- + +Node defines the nifi node basic configuration + +```yaml + - id: 0 + # nodeConfigGroup can be used to ease the node configuration, if set only the id is required + nodeConfigGroup: "default_group" + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop by Orange - Node 0 + # node configuration +# nodeConfig: + - id: 2 + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop by Orange - Node 2 + # node configuration + nodeConfig: + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 8Gi +``` + +## Node + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|int32| unique Node id. |Yes| - | +|nodeConfigGroup|string| can be used to ease the node configuration, if set only the id is required |No| "" | +|readOnlyConfig|[ReadOnlyConfig](./2_read_only_config.md)| readOnlyConfig can be used to pass Nifi node config which has type read-only these config changes will trigger rolling upgrade.| No | nil | +|nodeConfig|[NodeConfig](./3_node_config.md)| node configuration. |No| nil | + diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/5_node_state.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/5_node_state.md new file mode 100644 index 000000000..6b4b8cdb1 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/5_node_state.md @@ -0,0 +1,71 @@ +--- +id: 5_node_state +title: Node state +sidebar_label: Node state +--- + +Holds information about nifi state + +## NodeState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|gracefulActionState|[GracefulActionState](#gracefulactionstate)| holds info about nifi cluster action status.| - | - | +|configurationState|[ConfigurationState](#configurationstate)| holds info about the config.| - | - | +|initClusterNode|[InitClusterNode](#initclusternode)| contains if this nodes was part of the initial cluster.| - | - | + + +## GracefulActionState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|errorMessage|string| holds the information what happened with Nifi Cluster. | - | "" | +|actionStep|[ActionStep](#actionstep)| holds info about the action step ran.| No | nil | +|taskStarted|string| hold the time when the execution started.| No | "" | +|actionState|[State](#state)| holds the information about Action state.| No | nil | + +## ConfigurationState + +|Name|Value|Description| +|-----|----|------------| +|ConfigInSync|ConfigInSync|states that the generated nodeConfig is in sync with the Node| +|ConfigOutOfSync|ConfigOutOfSync|states that the generated nodeConfig is out of sync with the Node| + +## InitClusterNode + +|Name|Value|Description| +|-----|----|------------| +|IsInitClusterNode|true|states the node is part of initial cluster setup| +|NotInitClusterNode|false|states the node is not part of initial cluster setup| + +## State + +### Upscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulUpscaleRequired|GracefulUpscaleRequired|states that a node upscale is required.| +|GracefulUpscaleRunning|GracefulUpscaleRunning|states that the node upscale task is still running.| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully.| + +### Downscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulDownscaleRequired|GracefulDownscaleRequired|states that a node downscale is required| +|GracefulDownscaleRunning|GracefulDownscaleRunning|states that the node downscale is still running in| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully| + +## ActionStep +|Name|Value|Description| +|-----|----|------------| +|DisconnectNodeAction|DISCONNECTING|states that the NiFi node is disconnecting from NiFi Cluster.| +|DisconnectStatus|DISCONNECTED|states that the NiFi node is disconnected from NiFi Cluster.| +|OffloadNodeAction|OFFLOADING|states that the NiFi node is offloading data to NiFi Cluster.| +|OffloadStatus|OFFLOADED|states that the NiFi node offloaded data to NiFi Cluster.| +|RemovePodAction|POD_REMOVING|states that the NiFi node pod and object related are removing by operator.| +|RemovePodStatus|POD_REMOVED|states that the NiFi node pod and object related have been removed by operator.| +|RemoveNodeAction|REMOVING|states that the NiFi node is removing from NiFi Cluster.| +|RemoveStatus|REMOVED|states that the NiFi node is removed from NiFi Cluster.| +|ConnectNodeAction|CONNECTING|states that the NiFi node is connecting to the NiFi Cluster.| +|ConnectStatus|CONNECTED|states that the NiFi node is connected to the NiFi Cluster.| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/6_listeners_config.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/6_listeners_config.md new file mode 100644 index 000000000..571d350b7 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/6_listeners_config.md @@ -0,0 +1,56 @@ +--- +id: 6_listeners_config +title: Listeners Config +sidebar_label: Listeners Config +--- + +ListenersConfig defines the Nifi listener types : + +```yaml + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + - type: "prometheus" + name: "prometheus" + containerPort: 9090 + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +## ListenersConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|internalListeners|\[ \][InternalListener](#internallistener)| specifies settings required to access nifi internally.| Yes | - | +|sslSecrets|[SSLSecrets](#sslsecrets)| contains information about ssl related kubernetes secrets if one of the listener setting type set to ssl these fields must be populated to.| Yes | nil | +|clusterDomain|string| allow to override the default cluster domain which is "cluster.local".| Yes | `cluster.local` | +|useExternalDNS|string| allow to manage externalDNS usage by limiting the DNS names associated to each nodes and load balancer: `-node-...`| Yes | false | + +## InternalListener + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|enum{ "cluster", "http", "https", "s2s", "prometheus"}| allow to specify if we are in a specific nifi listener it's allowing to define some required information such as Cluster Port, Http Port, Https Port, S2S or Prometheus port| Yes | - | +|name|string| an identifier for the port which will be configured. | Yes | - | +|containerPort|int32| the containerPort. | Yes | - | + + +## SSLSecrets + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|tlsSecretName|string| should contain all ssl certs required by nifi including: caCert, caKey, clientCert, clientKey serverCert, serverKey, peerCert, peerKey. | Yes | - | +|create|boolean| tells the installed cert manager to create the required certs keys. | Yes | - | +|clusterScoped|boolean| defines if the Issuer created is cluster or namespace scoped. | Yes | - | +|issuerRef|[ObjectReference](https://docs.cert-manager.io/en/release-0.9/reference/api-docs/index.html#objectreference-v1alpha1)| cIssuerRef allow to use an existing issuer to act as CA: https://cert-manager.io/docs/concepts/issuer/ | No | - | +|pkiBackend|enum{"cert-manager"}| | Yes | - | + diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/7_external_service_config.md b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/7_external_service_config.md new file mode 100644 index 000000000..e7369762e --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/1_nifi_cluster/7_external_service_config.md @@ -0,0 +1,47 @@ +--- +id: 7_external_service_config +title: External Service Config +sidebar_label: External Service Config +--- + +ListenersConfig defines the Nifi listener types : + +```yaml + externalServices: + - name: "clusterip" + spec: + type: ClusterIP + portConfigs: + - port: 8080 + internalListenerName: "http" + serviceAnnotations: + toto: tata +``` + +## ExternalServiceConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| must be unique within a namespace. Name is primarily intended for creation idempotence and configuration.| Yes | - | +|serviceAnnotations|map\[string\]string| Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadat| No | - | +|spec|[ExternalServiceSpec](#externalservicespec)| defines the behavior of a service.| Yes | | + +## ExternalServiceSpec + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|portConfigs||\[ \][PortConfig](#portconfig)| Contains the list port for the service and the associated listener| Yes | - | +|clusterIP|string| More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies | No | - | +|type|[ServiceType](https://godoc.org/k8s.io/api/core/v1#ServiceType)| type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. | No | - | +|externalIPs|\[ \]string| externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes | No | - | +|loadBalancerIP|string| Only applies to Service Type: LoadBalancer. LoadBalancer will get created with the IP specified in this field. | No | - | +|loadBalancerSourceRanges|\[ \]string| If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs | No | - | +|externalName|string| externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. | No | - | + +## PortConfig + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|port|int32| The port that will be exposed by this service. | Yes | - | +|internalListenerName| string| The name of the listener which will be used as target container. | Yes | - | + diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/2_nifi_user.md b/site/website/versioned_docs/version-v0.6.2/5_references/2_nifi_user.md new file mode 100644 index 000000000..0c1535181 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/2_nifi_user.md @@ -0,0 +1,101 @@ +--- +id: 2_nifi_user +title: NiFi User +sidebar_label: NiFi User +--- + +`NifiUser` is the Schema for the nifi users API. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiUser +metadata: + name: aguitton +spec: + identity: alexandre.guitton@orange.com + clusterRef: + name: nc + namespace: nifikop + createCert: false +``` + +## NifiUser +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects users must create.|No|nil| +|spec|[NifiUserSpec](#nifiuserspec)|defines the desired state of NifiUser.|No|nil| +|status|[NifiUserStatus](#nifiuserstatus)|defines the observed state of NifiUser.|No|nil| + +## NifiUserSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|identity|string| used to define the user identity on NiFi cluster side, when the user's name doesn't suit with Kubernetes resource name. |No| - | +|secretName|string| name of the secret where all cert resources will be stored. |No| - | +|clusterRef|[ClusterReference](#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|DNSNames|\[ \]string| list of DNSNames that the user will used to request the NifiCluster (allowing to create the right certificates associated). |Yes| - | +|includeJKS|boolean| whether or not the the operator also include a Java keystore format (JKS) with you secret. |Yes| - | +|createCert|boolean| whether or not a certificate will be created for this user. |No| - | +|accessPolicies|\[ \][AccessPolicy](#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + + +## NifiUserStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi user's node id.|Yes| - | +|version|string| the last nifi user's node revision version catched.|Yes| - | + +## ClusterReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiCluster. |Yes| - | +|namespace|string| the NifiCluster namespace location. |Yes| - | + +## AccessPolicy + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[AccessPolicyType](#accesspolicytype)| defines the kind of access policy, could be "global" or "component". |Yes| - | +|action|[AccessPolicyAction](#accesspolicyaction)| defines the kind of action that will be granted, could be "read" or "write". |Yes| - | +|resource|[AccessPolicyResource](#accesspolicyresource)| defines the kind of resource targeted by this access policies, please refer to the following page : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies |Yes| - | +|componentType|string| used if the type is "component", it allows to define the kind of component on which is the access policy. |No| - | +|componentId|string| used if the type is "component", it allows to define the id of the component on which is the access policy. |No| - | + +## AccessPolicyType + +|Name|Value|Description| +|-----|----|------------| +|GlobalAccessPolicyType|global|Global access policies govern the following system level authorizations| +|ComponentAccessPolicyType|component|Component level access policies govern the following component level authorizations| + +## AccessPolicyAction + +|Name|Value|Description| +|-----|----|------------| +|ReadAccessPolicyAction|read|Allows users to view| +|WriteAccessPolicyAction|write|Allows users to modify| + +## AccessPolicyResource + +|Name|Value|Description| +|-----|----|------------| +|FlowAccessPolicyResource|/flow|About the UI| +|ControllerAccessPolicyResource|/controller| about the controller including Reporting Tasks, Controller Services, Parameter Contexts and Nodes in the Cluster| +|ParameterContextAccessPolicyResource|/parameter-context|About the Parameter Contexts. Access to Parameter Contexts are inherited from the "access the controller" policies unless overridden.| +|ProvenanceAccessPolicyResource|/provenance|Allows users to submit a Provenance Search and request Event Lineage| +|RestrictedComponentsAccessPolicyResource|/restricted-components|About the restricted components assuming other permissions are sufficient. The restricted components may indicate which specific permissions are required. Permissions can be granted for specific restrictions or be granted regardless of restrictions. If permission is granted regardless of restrictions, the user can create/modify all restricted components.| +|PoliciesAccessPolicyResource|/policies|About the policies for all components| +|TenantsAccessPolicyResource|/tenants| About the users and user groups| +|SiteToSiteAccessPolicyResource|/site-to-site|Allows other NiFi instances to retrieve Site-To-Site details| +|SystemAccessPolicyResource|/system|Allows users to view System Diagnostics| +|ProxyAccessPolicyResource|/proxy|Allows proxy machines to send requests on the behalf of others| +|CountersAccessPolicyResource|/counters|About counters| +|ComponentsAccessPolicyResource|/| About the component configuration details| +|OperationAccessPolicyResource|/operation|to operate components by changing component run status (start/stop/enable/disable), remote port transmission status, or terminating processor threads| +|ProvenanceDataAccessPolicyResource|/provenance-data|to view provenance events generated by this component| +|DataAccessPolicyResource|/data|About metadata and content for this component in flowfile queues in outbound connections and through provenance events| +|PoliciesComponentAccessPolicyResource|/policies|-| +|DataTransferAccessPolicyResource|/data-transfer|Allows a port to receive data from NiFi instances| + diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/3_nifi_registry_client.md b/site/website/versioned_docs/version-v0.6.2/5_references/3_nifi_registry_client.md new file mode 100644 index 000000000..e05c54479 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/3_nifi_registry_client.md @@ -0,0 +1,42 @@ +--- +id: 3_nifi_registry_client +title: NiFi Registry Client +sidebar_label: NiFi Registry Client +--- + +`NifiRegistryClient` is the Schema for the NiFi registry client API. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiRegistryClient +metadata: + name: squidflow +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Squidflow demo" + uri: "http://nifi-registry:18080" +``` + +## NifiRegistryClient +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects registry clients must create.|No|nil| +|spec|[NifiRegistryClientSpec](#nifiregistryclientspec)|defines the desired state of NifiRegistryClient.|No|nil| +|status|[NifiRegistryClientStatus](#nifiregistryclientstatus)|defines the observed state of NifiRegistryClient.|No|nil| + +## NifiRegistryClientsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Registry client. |No| - | +|uri|string| URI of the NiFi registry that should be used for pulling the flow. |Yes| - | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | + +## NifiRegistryClientStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi registry client's id. |Yes| - | +|version|int64| the last nifi registry client revision version catched. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/4_nifi_parameter_context.md b/site/website/versioned_docs/version-v0.6.2/5_references/4_nifi_parameter_context.md new file mode 100644 index 000000000..005aa9313 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/4_nifi_parameter_context.md @@ -0,0 +1,84 @@ +--- +id: 4_nifi_parameter_context +title: NiFi Parameter Context +sidebar_label: NiFi Parameter Context +--- + +`NifiParameterContext` is the Schema for the NiFi parameter context API. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + value: toto + description: toto +``` + +## NifiParameterContext + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects parameter contexts must create.|No|nil| +|spec|[NifiParameterContextSpec](#NifiParameterContextspec)|defines the desired state of NifiParameterContext.|No|nil| +|status|[NifiParameterContextStatus](#NifiParameterContextstatus)|defines the observed state of NifiParameterContext.|No|nil| + +## NifiParameterContextsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Parameter Context. |No| - | +|parameters|\[ \][Parameter](#parameter)| a list of non-sensitive Parameters. |No| - | +|secretRefs|\[ \][SecretReference](#secretreference)| a list of secret containing sensitive parameters (the key will name of the parameter) |No| - | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | + +## NifiParameterContextStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi parameter context's id. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | +|latestUpdateRequest|[ParameterContextUpdateRequest](#parametercontextupdaterequest)|the latest update request. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | + +## Parameter + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| the name of the Parameter. |Yes| - | +|value|string| the value of the Parameter. |Yes| - | +|description|string| the description of the Parameter. |No| - | + +## SecretReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the secret. |Yes| - | +|namespace|string| the secret namespace location. |Yes| - | + + +## ParameterContextUpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the id of the update request. |Yes| - | +|uri|string| the uri for this request. |Yes| - | +|submissionTime|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|lastUpdated|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| - | +|state|string| the state of the request. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/5_nifi_dataflow.md b/site/website/versioned_docs/version-v0.6.2/5_references/5_nifi_dataflow.md new file mode 100644 index 000000000..0a5261a50 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/5_nifi_dataflow.md @@ -0,0 +1,126 @@ +--- +id: 5_nifi_dataflow +title: NiFi Dataflow +sidebar_label: NiFi Dataflow +--- + +`NifiDataflow` is the Schema for the NiFi dataflow API. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + runOnce: false + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: squidflow + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: nifikop + updateStrategy: drain +``` + +## NifiDataflow + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects dataflows must create.|No|nil| +|spec|[NifiDataflowSpec](#NifiDataflowspec)|defines the desired state of NifiDataflow.|No|nil| +|status|[NifiDataflowStatus](#NifiDataflowstatus)|defines the observed state of NifiDataflow.|No|nil| + + +## NifiDataflowsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|parentProcessGroupID|string|the UUID of the parent process group where you want to deploy your dataflow, if not set deploy at root level. |No| - | +|bucketId|string|the UUID of the Bucket containing the flow. |Yes| - | +|flowId|string|the UUID of the flow to run. |Yes| - | +|flowVersion|*int32|the version of the flow to run, if not present or equals to -1, then the latest version of flow will be used. |Yes| - | +|runOnce|bool|if the flow will be ran once or continuously checked. |Yes| true | +|skipInvalidControllerService|bool|whether the flow is considered as ran if some controller services are still invalid or not. |Yes| false | +|skipInvalidComponent|bool|whether the flow is considered as ran if some components are still invalid or not. |Yes| false | +|updateStrategy|[DataflowUpdateStrategy](#dataflowupdatestrategy)|describes the way the operator will deal with data when a dataflow will be updated : Drop or Drain |Yes| drain | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|parameterContextRef|[ParameterContextReference](./4_nifi_parameter_context.md#parametercontextreference)| contains the reference to the ParameterContext with the one the dataflow is linked. |No| - | +|clusterRef|[RegistryClientReference](./3_nifi_registry_client.md#registryclientreference)| contains the reference to the NifiRegistry with the one the dataflow is linked. |Yes| - | + +## NifiDataflowStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|processGroupID|string| process Group ID. |Yes| - | +|state|[DataflowState](#dataflowstate))| the dataflow current state. |Yes| - | +|latestUpdateRequest|[UpdateRequest](#updaterequest)|the latest update request sent. |Yes| - | +|latestDropRequest|[DropRequest](#droprequest)|the latest queue drop request sent. |Yes| - | + +## DataflowUpdateStrategy + +|Name|Value|Description| +|-----|----|------------| +|DrainStrategy|drain|leads to shutting down only input components (Input processors, remote input process group) and dropping all flowfiles from the flow.| +|DropStrategy|drop|leads to shutting down all components and dropping all flowfiles from the flow.| + +## DataflowState + +|Name|Value|Description| +|-----|----|------------| +|DataflowStateCreated|Created|describes the status of a NifiDataflow as created.| +|DataflowStateStarting|Starting|describes the status of a NifiDataflow as starting.| +|DataflowStateRan|Ran|describes the status of a NifiDataflow as running.| +|DataflowStateOutOfSync|OutOfSync|ddescribes the status of a NifiDataflow as out of sync.| +|DataflowStateInSync|InSync|describes the status of a NifiDataflow as in sync.| + +## UpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[DataflowUpdateRequestType](#dataflowupdaterequesttype)|defines the type of versioned flow update request. |Yes| - | +|id|string|the id of the update request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| 0 | +|state|string| the state of the request. |Yes| - | + +## DropRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|connectionId|string|the connection id. |Yes| - | +|id|string|the id for this drop request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|finished|bool|whether the request has finished. |Yes| false | +|failureReason|string|an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32|the percentage complete of the request, between 0 and 100. |Yes| 0 | +|currentCount|int32|the number of flow files currently queued. |Yes| 0 | +|currentSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|current|string|the count and size of flow files currently queued. |Yes| - | +|originalCount|int32|the number of flow files to be dropped as a result of this request. |Yes| 0 | +|originalSize|int64| the size of flow files to be dropped as a result of this request in bytes. |Yes| 0 | +|original|string|the count and size of flow files to be dropped as a result of this request. |Yes| - | +|droppedCount|int32|the number of flow files that have been dropped thus far. |Yes| 0 | +|droppedSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|Dropped|string|the count and size of flow files that have been dropped thus far. |Yes| - | +|state|string|the state of the request. |Yes| - | + + +## DataflowUpdateRequestType + +|Name|Value|Description| +|-----|----|------------| +|RevertRequestType|Revert|defines a revert changes request.| +|UpdateRequestType|Update|defines an update version request.| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/5_references/6_nifi_usergroup.md b/site/website/versioned_docs/version-v0.6.2/5_references/6_nifi_usergroup.md new file mode 100644 index 000000000..3b5881c07 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/5_references/6_nifi_usergroup.md @@ -0,0 +1,55 @@ +--- +id: 6_nifi_usergroup +title: NiFi UserGroup +sidebar_label: NiFi UserGroup +--- + +`NifiUserGroup` is the Schema for the nifi user groups API. + +```yaml +apiVersion: nifi.orange.com/v1alpha1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + clusterRef: + name: nc + namespace: nifikop + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local + - name: nc-controller.nifikop.mgt.cluster.local + accessPolicies: + - type: global + action: read + resource: /counters +``` + +## NifiUser +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects usergroups must create.|No|nil| +|spec|[NifiUserGroupSpec](#nifiusergroupspec)|defines the desired state of NifiUserGroup.|No|nil| +|status|[NifiUserGroupStatus](#nifiusergroupstatus)|defines the observed state of NifiUserGroup.|No|nil| + +## NifiUserGroupSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|usersRef|\[ \][UserReference](#userref)| contains the list of reference to NifiUsers that are part to the group. |No| [] | +|accessPolicies|\[ \][AccessPolicy](./2_nifi_user.md#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + +## NifiUserGroupStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi usergroup's node id.|Yes| - | +|version|string| the last nifi usergroup's node revision version catched.|Yes| - | + +## UserReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiUser. |Yes| - | +|namespace|string| the NifiUser namespace location. |Yes| - | + diff --git a/site/website/versioned_docs/version-v0.6.2/6_contributing/1_developer_guide.md b/site/website/versioned_docs/version-v0.6.2/6_contributing/1_developer_guide.md new file mode 100644 index 000000000..5482ed87f --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/6_contributing/1_developer_guide.md @@ -0,0 +1,143 @@ +--- +id: 1_developer_guide +title: Developer guide +sidebar_label: Developer guide +--- + +## Operator SDK + +### Prerequisites + +NiFiKop has been validated with : + +- [go](https://golang.org/doc/install) version v1.15+. +- [docker](https://docs.docker.com/get-docker/) version 18.09+ +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.16+ +- [Helm](https://helm.sh/) version v3.4.2 +- [Operator sdk](https://github.com/operator-framework/operator-sdk) version v1.5.0 + +### Initial setup + +Checkout the project. + +```bash +git clone https://github.com/Orange-OpenSource/nifikop.git +cd nifikop +``` + +### Operator sdk + +The full list of command is available here : https://sdk.operatorframework.io/docs/upgrading-sdk-version/v1.0.0/#cli-changes + +### Build NiFiKop + +#### Local environment + +If you prefer working directly with your local go environment you can simply uses : + +```bash +make build +``` + +### Run NiFiKop + +We can quickly run NiFiKop in development mode (on your local host), then it will use your kubectl configuration file to connect to your kubernetes cluster. + +There are several ways to execute your operator : + +- Using your IDE directly +- Executing directly the Go binary +- deploying using the Helm charts + +If you want to configure your development IDE, you need to give it environment variables so that it will uses to connect to kubernetes. + +```bash +KUBECONFIG={path/to/your/kubeconfig} +WATCH_NAMESPACE={namespace_to_watch} +POD_NAME={name for operator pod} +LOG_LEVEL=Debug +OPERATOR_NAME=ide +``` + +#### Run the Operator Locally with the Go Binary + +This method can be used to run the operator locally outside of the cluster. This method may be preferred during development as it facilitates faster deployment and testing. + +Set the name of the operator in an environment variable + +```bash + export OPERATOR_NAME=nifi-operator +``` + +Deploy the CRDs. + +```bash +kubectl apply -f config/crd/bases/nifi.orange.com_nificlusters.yaml +kubectl apply -f config/crd/bases/nifi.orange.com_nifidataflows.yaml +kubectl apply -f config/crd/bases/nifi.orange.com_nifiparametercontexts.yaml +kubectl apply -f config/crd/bases/nifi.orange.com_nifiregistryclients.yaml +kubectl apply -f config/crd/bases/nifi.orange.com_nifiusers.yaml +``` + +And deploy the operator. + +```bash +make run +``` + +This will run the operator in the `default` namespace using the default Kubernetes config file at `$HOME/.kube/config`. + +#### Deploy using the Helm Charts + +This section provides an instructions for running the operator Helm charts with an image that is built from the local branch. + +Build the image from the current branch. + +```bash +export DOCKER_REPO_BASE={your-docker-repo} +make docker-build +``` + +Push the image to docker hub (or to whichever repo you want to use) + +```bash +$ make docker-push +``` + +:::info +The image tag is a combination of the version as defined in `verion/version.go` and the branch name. +::: + +Install the Helm chart. + +```bash +helm install skeleton ./helm/nifikop \ + --set image.tag=v0.5.1-release \ + --namespace-{"nifikop"} +``` + +:::important +The `image.repository` and `image.tag` template variables have to match the names from the image that we pushed in the previous step. +::: + +:::info +We set the chart name to the branch, but it can be anything. +::: + +Lastly, verify that the operator is running. + +```console +$ kubectl get pods -n nifikop +NAME READY STATUS RESTARTS AGE +skeleton-nifikop-8946b89dc-4cfs9 1/1 Running 0 7m45s +``` + +## Helm + +The NiFiKop operator is released in the `orange-incubator` helm repository. + +In order to package the chart you need to run the following command. + +```bash +make helm-package +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/6_contributing/2_reporting_bugs.md b/site/website/versioned_docs/version-v0.6.2/6_contributing/2_reporting_bugs.md new file mode 100644 index 000000000..4c3869ad4 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/6_contributing/2_reporting_bugs.md @@ -0,0 +1,25 @@ +--- +id: 2_reporting_bugs +title: Reporting bugs +sidebar_label: Reporting bugs +--- + +If any part of the NiFiKop project has bugs or documentation mistakes, please let us know by [opening an issue](https://github.com/Orange-OpenSource/nifikop/issues/new). We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist. + +To make the bug report accurate and easy to understand, please try to create bug reports that are: + +- Specific. Include as much details as possible: which version, what environment, what configuration, etc. + +- Reproducible. Include the steps to reproduce the problem. We understand some issues might be hard to reproduce, please include the steps that might lead to the problem. + +- Isolated. Please try to isolate and reproduce the bug with minimum dependencies. It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. Debugging external systems that rely on operator-sdk is out of scope, but we are happy to provide guidance in the right direction or help with using operator-sdk itself. + +- Unique. Do not duplicate existing bug report. + +- Scoped. One bug per report. Do not follow up with another bug inside one report. + +It may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report. + +We might ask for further information to locate a bug. A duplicated bug report will be closed. + +[filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/ \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.6.2/6_contributing/3_credits.md b/site/website/versioned_docs/version-v0.6.2/6_contributing/3_credits.md new file mode 100644 index 000000000..e04adf7f4 --- /dev/null +++ b/site/website/versioned_docs/version-v0.6.2/6_contributing/3_credits.md @@ -0,0 +1,11 @@ +--- +id: 3_credits +title: Credits +sidebar_label: Credits +--- + +This implementation is based on other Open-Source project, and lot of the community ideas. Particular thanks to : + +- Operator implementation based on [banzaicloud/kafka-operator](https://github.com/banzaicloud/kafka-operator) +- NiFi kubernetes setup configuration inspired from [cetic/helm-nifi](https://github.com/cetic/helm-nifi) +- Implementation is based on [Operator SDK](https://github.com/operator-framework/operator-sdk) \ No newline at end of file diff --git a/site/website/versioned_sidebars/version-v0.6.2-sidebars.json b/site/website/versioned_sidebars/version-v0.6.2-sidebars.json new file mode 100644 index 000000000..0f7099689 --- /dev/null +++ b/site/website/versioned_sidebars/version-v0.6.2-sidebars.json @@ -0,0 +1,193 @@ +{ + "version-v0.6.2/docs": [ + { + "collapsed": true, + "type": "category", + "label": "Concepts", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/1_concepts/1_introduction" + }, + { + "type": "doc", + "id": "version-v0.6.2/1_concepts/2_design_principes" + }, + { + "type": "doc", + "id": "version-v0.6.2/1_concepts/3_features" + }, + { + "type": "doc", + "id": "version-v0.6.2/1_concepts/4_roadmap" + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Setup", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/2_setup/1_getting_started" + }, + { + "collapsed": true, + "type": "category", + "label": "Platform Setup", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/2_setup/2_platform_setup/1_gke" + }, + { + "type": "doc", + "id": "version-v0.6.2/2_setup/2_platform_setup/2_minikube" + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Install", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/2_setup/3_install/1_customizable_install_with_helm" + } + ] + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Tasks", + "items": [ + { + "collapsed": true, + "type": "category", + "label": "NiFi Cluster", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/3_tasks/1_nifi_cluster/2_cluster_scaling" + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Security", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/3_tasks/2_security/1_ssl" + }, + { + "collapsed": true, + "type": "category", + "label": "Authentication", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/3_tasks/2_security/2_authentication/1_oidc" + } + ] + } + ] + }, + { + "type": "doc", + "id": "version-v0.6.2/3_tasks/3_nifi_dataflow" + }, + { + "type": "doc", + "id": "version-v0.6.2/3_tasks/4_nifi_user_group" + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Reference", + "items": [ + { + "collapsed": true, + "type": "category", + "label": "NiFi Cluster", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/1_nifi_cluster" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/2_read_only_config" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/3_node_config" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/4_node" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/5_node_state" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/6_listeners_config" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/1_nifi_cluster/7_external_service_config" + } + ] + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/2_nifi_user" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/3_nifi_registry_client" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/4_nifi_parameter_context" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/5_nifi_dataflow" + }, + { + "type": "doc", + "id": "version-v0.6.2/5_references/6_nifi_usergroup" + } + ] + }, + { + "collapsed": true, + "type": "category", + "label": "Contributing", + "items": [ + { + "type": "doc", + "id": "version-v0.6.2/6_contributing/1_developer_guide" + }, + { + "type": "doc", + "id": "version-v0.6.2/6_contributing/2_reporting_bugs" + }, + { + "type": "doc", + "id": "version-v0.6.2/6_contributing/3_credits" + } + ] + } + ] +} diff --git a/site/website/versions.json b/site/website/versions.json index 3e798228e..816f57f45 100644 --- a/site/website/versions.json +++ b/site/website/versions.json @@ -1,4 +1,5 @@ [ + "v0.6.2", "v0.6.1", "v0.6.0", "v0.5.3", diff --git a/version/version.go b/version/version.go index 049535a6f..c31aa426b 100644 --- a/version/version.go +++ b/version/version.go @@ -1,5 +1,5 @@ package version var ( - Version = "0.6.1" + Version = "0.6.2" )