Merge pull request #116 from StephenSorriaux/fix/zk-tls

Add tests on ZK mTLS & fix zk tls load config
StephenSorriaux · Nov 16, 2021 · 012f62a · 012f62a
2 parents 291ba00 + 7ed6c05
commit 012f62a
Show file tree

Hide file tree

Showing 47 changed files with 771 additions and 383 deletions.
diff --git a/library/kafka_acl.py b/library/kafka_acl.py
@@ -26,6 +26,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 

diff --git a/library/kafka_acls.py b/library/kafka_acls.py
@@ -26,6 +26,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 

diff --git a/library/kafka_lib.py b/library/kafka_lib.py
@@ -25,6 +25,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 
@@ -109,6 +113,10 @@
     description:
       - 'when zookeeper is configured to use authentication, value used to '
       - 'connect.'
+  zookeeper_use_ssl:
+    description:
+      - 'force using ssl for zookeeper connection.'
+    default: False
   zookeeper_ssl_check_hostname:
     description:
       - 'when using ssl for zookeeper, check if certificate for hostname is '

diff --git a/library/kafka_quotas.py b/library/kafka_quotas.py
@@ -24,6 +24,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 
@@ -58,6 +62,10 @@
     description:
       - 'when zookeeper is configured to use authentication, value used to '
       - 'connect.'
+  zookeeper_use_ssl:
+    description:
+      - 'force using ssl for zookeeper connection.'
+    default: False
   zookeeper_ssl_check_hostname:
     description:
       - 'when using ssl for zookeeper, check if certificate for hostname is '

diff --git a/library/kafka_topic.py b/library/kafka_topic.py
@@ -24,6 +24,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 
@@ -73,6 +77,10 @@
     description:
       - 'when zookeeper is configured to use authentication, value used to '
       - 'connect.'
+  zookeeper_use_ssl:
+    description:
+      - 'force using ssl for zookeeper connection.'
+    default: False
   zookeeper_ssl_check_hostname:
     description:
       - 'when using ssl for zookeeper, check if certificate for hostname is '

diff --git a/library/kafka_topics.py b/library/kafka_topics.py
@@ -24,6 +24,10 @@
 log.addHandler(logging.StreamHandler(sys.stdout))
 log.setLevel(logging.INFO)
 
+log = logging.getLogger('kazoo.client')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
 ANSIBLE_METADATA = {'metadata_version': '1.0'}
 
 
@@ -58,6 +62,10 @@
     description:
       - 'when zookeeper is configured to use authentication, value used to '
       - 'connect.'
+  zookeeper_use_ssl:
+    description:
+      - 'force using ssl for zookeeper connection.'
+    default: False
   zookeeper_ssl_check_hostname:
     description:
       - 'when using ssl for zookeeper, check if certificate for hostname is '

diff --git a/module_utils/kafka_lib_commons.py b/module_utils/kafka_lib_commons.py
@@ -158,6 +158,8 @@
     zookeeper_sleep_time=dict(type='int', required=False, default=5),
 
     zookeeper_max_retries=dict(type='int', required=False, default=5),
+
+    zookeeper_use_ssl=dict(type='bool', required=False, default=False),
 )
 
 module_commons = dict(
@@ -323,10 +325,12 @@ def get_zookeeper_configuration(params):
           zookeeper_ssl_cafile, zookeeper_ssl_certfile,
           zookeeper_ssl_keyfile
         )
-        zookeeper_use_ssl = bool(
-            zookeeper_ssl_files['keyfile']['path'] is not None and
-            zookeeper_ssl_files['certfile']['path'] is not None
-        )
+        zookeeper_use_ssl = params.get('zookeeper_use_ssl', bool(
+            (
+                zookeeper_ssl_files['keyfile']['path'] is not None and
+                zookeeper_ssl_files['certfile']['path'] is not None
+            ) or zookeeper_ssl_files['cafile']['path'] is not None
+        ))
 
         zookeeper_auth = []
         if zookeeper_auth_value != '':

diff --git a/module_utils/kafka_manager.py b/module_utils/kafka_manager.py
@@ -100,7 +100,7 @@ def init_zk_client(self):
             self.zk_client.start()
         except Exception as e:
             raise ZookeeperBroken(
-                msg='Error while initializing Zookeeper client : '
+                'Error while initializing Zookeeper client : '
                 '%s. Is your Zookeeper server available and '
                 'running on \'%s\'?' % (e, self.zk_configuration['hosts'])
             )

diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
@@ -16,7 +16,7 @@ platforms:
       /bin/bash -c '/usr/local/bin/python -m pip install -r /src/requirements.txt && trap : TERM INT; sleep infinity & wait'
     volumes:
       - ${MOLECULE_PROJECT_DIRECTORY}:/src
-      - ${MOLECULE_SCENARIO_DIRECTORY}/tls:/opt/kafka/tls
+      - ${MOLECULE_SCENARIO_DIRECTORY}/tls:/opt/tls
     groups:
       - executors
     networks:
@@ -53,10 +53,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -89,10 +89,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -136,10 +136,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -172,10 +172,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -192,16 +192,21 @@ platforms:
       - kafka
   # 2.6.0
   - name: zookeeper-260
-    image: zookeeper:3.4
+    image: zookeeper:3.6
     command: "bin/zkServer.sh start-foreground"
     published_ports:
       - "2181"
+      - "2281"
     networks:
       - name: molecule
         aliases:
           - zookeeper-260
+    volumes:
+      - ${MOLECULE_SCENARIO_DIRECTORY}/zoo-tls.cfg:/conf/zoo.cfg
+      - ${MOLECULE_SCENARIO_DIRECTORY}/tls:/opt/zookeeper/tls
     groups:
       - zookeeper
+      - zookeeper-tls
   - name: kafka1-260
     hostname: kafka1-260
     image: wurstmeister/kafka:2.13-2.6.0
@@ -219,10 +224,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -238,6 +243,7 @@ platforms:
     groups:
       - kafka
       - kafka1
+      - kafka1_zk_tls
   - name: kafka2-260
     hostname: kafka2-260
     image: wurstmeister/kafka:2.13-2.6.0
@@ -255,10 +261,10 @@ platforms:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_OPTS: -Djava.security.auth.login.config=/opt/kafka/jaas/kafka_server_jaas.conf
-      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.p12
+      KAFKA_SSL_KEYSTORE_LOCATION: /opt/kafka/tls/keystore/server-keystore.jks
       KAFKA_SSL_KEYSTORE_PASSWORD: password
       KAFKA_SSL_KEY_PASSWORD: password
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.p12
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /opt/kafka/tls/keystore/server-truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
       KAFKA_SSL_CLIENT_AUTH: required
     published_ports:
@@ -293,6 +299,7 @@ provisioner:
             instance_suffix: "111"
           - protocol_version: "2.6.0"
             instance_suffix: "260"
+            zk_tls: true
         topic_defaut_configuration:
           state: 'present'
           replica_factor: 1