task/DEP-3321: Upgrade OpenDistro to version 1.11.0

src/opendistro-es/Chart.yaml

@@ -1,11 +1,29 @@
+# Copyright 2019 Viasat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
 apiVersion: v1
-appVersion: 1.0.0
-description: 'Opendistro Elasticsearch'
+# Open Distro for Elasticsearch version
+appVersion: 1.11.0
+description: 'Open Distro for Elasticsearch'
 engine: gotpl
 kubeVersion: ^1.10.0-0
 maintainers:
-   - name: OpenDistro ES Maintainers
+- email: [email protected]
+  name: Derek Heldt-Werle
+- email: [email protected]
+  name: Kalvin Chau
 name: opendistro-es
 sources:
 - https://pages.git.viasat.com/ATG/charts
-version: 1.0.7
+# Chart version
+version: 1.11.0

src/opendistro-es/README.md

@@ -0,0 +1,6 @@
+# Custom eHA OpenDistro Chart
+
+This custom chart is cloned from [here](https://opendistro.github.io/for-elasticsearch-docs/docs/install/helm/).
+The `values.yaml` and `Chart.yaml` are major files that were edited. Others remain mostly a clone of the src code.
+
+Current Version is `1.11.0`.

src/opendistro-es/templates/_helpers.tpl

@@ -1,4 +1,19 @@
 {{/* vim: set filetype=mustache: */}}
+{{/*
+Copyright 2019 Viasat, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License").
+You may not use this file except in compliance with the License.
+A copy of the License is located at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+or in the "license" file accompanying this file. This file is distributed
+on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+express or implied. See the License for the specific language governing
+permissions and limitations under the License.
+*/}}
+
 {{/*
 Expand the name of the chart.
 */}}
@@ -29,11 +44,22 @@ If release name contains chart name it will be used as a full name.
 Define standard labels for frequently used metadata.
 */}}
 {{- define "opendistro-es.labels.standard" -}}
+app: {{ template "opendistro-es.fullname" . }}
 chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 release: "{{ .Release.Name }}"
 heritage: "{{ .Release.Service }}"
 {{- end -}}
 
+{{/*
+Define labels for deployment/statefulset selectors.
+We cannot have the chart label here as it will prevent upgrades.
+*/}}
+{{- define "opendistro-es.labels.selector" -}}
+app: {{ template "opendistro-es.fullname" . }}
+release: "{{ .Release.Name }}"
+heritage: "{{ .Release.Service }}"
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}

src/opendistro-es/templates/elasticsearch/elasticsearch-serviceaccount.yaml

@@ -1,3 +1,18 @@
+# Copyright 2019 Viasat, Inc.
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# @formatter:off
 {{ if .Values.elasticsearch.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount

src/opendistro-es/templates/elasticsearch/es-client-deploy.yaml

@@ -1,7 +1,19 @@
+# Copyright 2019 Viasat, Inc.
 # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# SPDX-License-Identifier: MIT-0
-{{- if .Values.elasticsearch.client.enabled }}
----
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# @formatter:off
+{{- if and .Values.elasticsearch.client.enabled .Values.elasticsearch.client.dedicatedPod.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -14,7 +26,8 @@ spec:
   replicas: {{ .Values.elasticsearch.client.replicas }}
   selector:
     matchLabels:
-{{ include "opendistro-es.labels.standard" . | indent 8 }}
+{{ include "opendistro-es.labels.selector" . | indent 6 }}
+      role: client
   template:
     metadata:
       labels:
@@ -23,8 +36,11 @@ spec:
       annotations:
         {{/* This forces a restart if the secret config has changed */}}
         {{- if .Values.elasticsearch.config }}
-        configchecksum: {{ include (print .Template.BasePath "/elasticsearch/es-config.yaml") . | sha256sum | trunc 63 }}
+        configchecksum: {{ include (print .Template.BasePath "/elasticsearch/es-config-secret.yaml") . | sha256sum | trunc 63 }}
         {{- end }}
+{{- if .Values.elasticsearch.client.podAnnotations }}
+{{ toYaml .Values.elasticsearch.client.podAnnotations | indent 8 }}
+{{- end }}
     spec:
 {{- include "opendistro-es.imagePullSecrets" . | indent 6 }}
       serviceAccountName: {{ template "opendistro-es.elasticsearch.serviceAccountName" . }}
@@ -36,31 +52,29 @@ spec:
       nodeSelector:
 {{ toYaml . | indent 8 }}
     {{- end }}
-      # Weighted anti-affinity to disallow deploying client node to the same worker node as master node
+    {{- with .Values.elasticsearch.client.affinity }}
       affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 1
-              podAffinityTerm:
-                topologyKey: "kubernetes.io/hostname"
-                labelSelector:
-                  matchLabels:
-                    role: client
-      {{- with .Values.elasticsearch.client.nodeAffinity }}
-        nodeAffinity:
-{{ toYaml . | indent 10 }}
-      {{- end }}
+{{ toYaml . | indent 8 }}
+    {{- end }}
       initContainers:
+{{- if .Values.elasticsearch.sysctl.enabled }}
       - name: init-sysctl
-        image: {{ .Values.elasticsearch.initContainer.image }}:{{ .Values.elasticsearch.initContainer.imageTag }}
+        image: {{ .Values.global.registry }}/{{ .Values.elasticsearch.initContainer.image }}:{{ .Values.elasticsearch.initContainer.imageTag }}
         command:
         - sysctl
         - -w
         - vm.max_map_count={{ .Values.elasticsearch.maxMapCount }}
         securityContext:
           privileged: true
+{{- end }}
+{{- if .Values.elasticsearch.extraInitContainers }}
+{{ toYaml .Values.elasticsearch.extraInitContainers| indent 6 }}
+{{- end }}
       containers:
       - name: elasticsearch
+        securityContext:
+          capabilities:
+            add: ["SYS_CHROOT"]
         env:
         - name: cluster.name
           value: {{ .Values.global.clusterName }}
@@ -94,21 +108,27 @@ spec:
         resources:
 {{ toYaml .Values.elasticsearch.client.resources | indent 12 }}
         # Official Image from Open Distro Team
-        image: {{ .Values.elasticsearch.image }}:{{ .Values.elasticsearch.imageTag }}
-        imagePullPolicy: Always
+        image: {{ .Values.global.registry }}/{{ .Values.elasticsearch.image }}:{{ .Values.elasticsearch.imageTag }}
+        imagePullPolicy: {{ .Values.elasticsearch.imagePullPolicy | default "Always" | quote }}
         ports:
         - containerPort: 9200
           name: http
         - containerPort: 9300
           name: transport
         - containerPort: 9600
           name: metrics
+        - containerPort: 9650
+          name: rca
     {{- with .Values.elasticsearch.client.readinessProbe}}
         readinessProbe:
 {{ toYaml . | indent 10 }}
     {{- end }}
     {{- with .Values.elasticsearch.client.livenessProbe}}
         livenessProbe:
+{{ toYaml . | indent 10 }}
+    {{- end }}
+    {{- with .Values.elasticsearch.client.startupProbe}}
+        startupProbe:
 {{ toYaml . | indent 10 }}
     {{- end }}
         volumeMounts:
@@ -117,9 +137,6 @@ spec:
           name: config
           subPath: elasticsearch.yml
         {{- end }}
-        - mountPath: /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/config.yml
-          name: security-config
-          subPath: config.yml
         {{- if .Values.elasticsearch.log4jConfig }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/log4j2.properties
           name: config
@@ -128,44 +145,47 @@ spec:
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/logging.yml
           name: config
           subPath: logging.yml
-        {{- if and .Values.elasticsearch.ssl.transport.enabled .Values.elasticsearch.ssl.transport.existingCertSecret }}
-        - mountPath: {{ .Values.elasticsearch.configDirectory }}/transport-crt.pem
+        {{- if .Values.elasticsearch.ssl.transport.existingCertSecret }}
+        - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-transport-crt.pem
           name: transport-certs
-          subPath: transport-crt.pem
-        - mountPath: {{ .Values.elasticsearch.configDirectory }}/transport-key.pem
+          subPath: {{ .Values.elasticsearch.ssl.transport.existingCertSecretCertSubPath }}
+        - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-transport-key.pem
           name: transport-certs
-          subPath: transport-key.pem
-        - mountPath: {{ .Values.elasticsearch.configDirectory }}/transport-root-ca.pem
+          subPath: {{ .Values.elasticsearch.ssl.transport.existingCertSecretKeySubPath }}
+        - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-transport-root-ca.pem
           name: transport-certs
-          subPath: transport-root-ca.pem
+          subPath: {{ .Values.elasticsearch.ssl.transport.existingCertSecretRootCASubPath }}
          {{- end }}
          {{- if and .Values.elasticsearch.ssl.rest.enabled .Values.elasticsearch.ssl.rest.existingCertSecret }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-rest-crt.pem
           name: rest-certs
-          subPath: elk-rest-crt.pem
+          subPath: {{ .Values.elasticsearch.ssl.rest.existingCertSecretCertSubPath }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-rest-key.pem
           name: rest-certs
-          subPath: elk-rest-key.pem
+          subPath: {{ .Values.elasticsearch.ssl.rest.existingCertSecretKeySubPath }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/elk-rest-root-ca.pem
           name: rest-certs
-          subPath: elk-rest-root-ca.pem
+          subPath: {{ .Values.elasticsearch.ssl.rest.existingCertSecretRootCASubPath }}
         {{- end }}
         {{- if and .Values.elasticsearch.ssl.admin.enabled .Values.elasticsearch.ssl.admin.existingCertSecret }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/admin-crt.pem
           name: admin-certs
-          subPath: admin-crt.pem
+          subPath: {{ .Values.elasticsearch.ssl.admin.existingCertSecretCertSubPath }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/admin-key.pem
           name: admin-certs
-          subPath: admin-key.pem
+          subPath: {{ .Values.elasticsearch.ssl.admin.existingCertSecretKeySubPath }}
         - mountPath: {{ .Values.elasticsearch.configDirectory }}/admin-root-ca.pem
           name: admin-certs
-          subPath: admin-root-ca.pem
+          subPath: {{ .Values.elasticsearch.ssl.admin.existingCertSecretRootCASubPath }}
         {{- end }}
+{{- if .Values.elasticsearch.extraVolumeMounts }}
+{{ toYaml .Values.elasticsearch.extraVolumeMounts | indent 8 }}
+{{- end }}
       volumes:
       - name: config
-        configMap:
-          name: {{ template "opendistro-es.fullname" . }}-config
-      {{- if and .Values.elasticsearch.ssl.transport.enabled .Values.elasticsearch.ssl.transport.existingCertSecret }}
+        secret:
+          secretName: {{ template "opendistro-es.fullname" . }}-es-config
+      {{- if .Values.elasticsearch.ssl.transport.existingCertSecret }}
       - name: transport-certs
         secret:
           secretName: {{ .Values.elasticsearch.ssl.transport.existingCertSecret }}
@@ -180,7 +200,7 @@ spec:
         secret:
           secretName: {{ .Values.elasticsearch.ssl.admin.existingCertSecret }}
       {{- end }}
-      - name: security-config
-        configMap:
-          name: {{ template "opendistro-es.fullname" . }}-security-config
+{{- if .Values.elasticsearch.extraVolumes }}
+{{ toYaml .Values.elasticsearch.extraVolumes | indent 6 }}
+{{- end }}
 {{- end }}

src/opendistro-es/templates/elasticsearch/es-client-ingress.yaml

@@ -1,8 +1,22 @@
-{{- if and .Values.elasticsearch.client.ingress.enabled .Values.elasticsearch.client.enabled -}}
-{{- $fullName := printf "%s-%s"  (include "opendistro-es.fullname" .) "client-service" }}
-{{- $ingressPath := .Values.elasticsearch.client.ingress.path -}}
+# Copyright 2019 Viasat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# @formatter:off
+{{- if and .Values.elasticsearch.client.ingress.enabled .Values.elasticsearch.client.enabled }}
+{{ $fullName := printf "%s-%s"  (include "opendistro-es.fullname" .) "client-service" }}
+{{ $ingressPath := .Values.elasticsearch.client.ingress.path }}
 kind: Ingress
-apiVersion: apps/v1
+apiVersion: extensions/v1beta1
 metadata:
   name: {{ $fullName }}
   labels:
@@ -31,5 +45,5 @@ spec:
             backend:
               serviceName: {{ $fullName }}
               servicePort: http
-  {{- end -}}
-{{- end -}}
+  {{- end }}
+{{- end }}

src/opendistro-es/templates/elasticsearch/es-client-pdb.yaml

@@ -1,7 +1,18 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# SPDX-License-Identifier: MIT-0
-{{- if and .Values.elasticsearch.client.podDisruptionBudget.enabled .Values.elasticsearch.client.enabled }}
----
+# Copyright 2019 Viasat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# @formatter:off
+{{- if and .Values.elasticsearch.client.podDisruptionBudget.enabled .Values.elasticsearch.client.enabled .Values.elasticsearch.client.dedicatedPod.enabled }}
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
@@ -11,7 +22,7 @@ metadata:
 {{ include "opendistro-es.labels.standard" . | indent 4 }}
 spec:
 {{- if .Values.elasticsearch.client.podDisruptionBudget.minAvailable }}
-  minAvailable: {{ .Values.client.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.elasticsearch.client.podDisruptionBudget.minAvailable }}
 {{- end }}
 {{- if .Values.elasticsearch.client.podDisruptionBudget.maxUnavailable }}
   maxUnavailable: {{ .Values.elasticsearch.client.podDisruptionBudget.maxUnavailable }}