From 0c97d257745657300d8810d09a6ce85a4d10c28f Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 6 Sep 2023 17:36:04 +0200 Subject: [PATCH 01/18] Configure Galaxy in Molecule (#255) This will force Molecule to run Ansible Galaxy twice. Once for roles, once for collection. fixes #254 --- molecule/elasticsearch_cluster-oss/molecule.yml | 2 ++ molecule/elasticsearch_default/molecule.yml | 2 ++ molecule/elasticsearch_no-security/molecule.yml | 2 ++ molecule/elasticsearch_roles_calculation/molecule.yml | 2 ++ molecule/elasticstack_default/molecule.yml | 2 ++ molecule/logstash_full_stack-oss/molecule.yml | 2 ++ molecule/logstash_pipelines/molecule.yml | 2 ++ molecule/logstash_specific_version/molecule.yml | 2 ++ molecule/repos_default/molecule.yml | 2 ++ molecule/repos_oss/molecule.yml | 2 ++ 10 files changed, 20 insertions(+) diff --git a/molecule/elasticsearch_cluster-oss/molecule.yml b/molecule/elasticsearch_cluster-oss/molecule.yml index 56dd07f2..50f31e05 100644 --- a/molecule/elasticsearch_cluster-oss/molecule.yml +++ b/molecule/elasticsearch_cluster-oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_default/molecule.yml b/molecule/elasticsearch_default/molecule.yml index 0b2fca9d..25c5022b 100644 --- a/molecule/elasticsearch_default/molecule.yml +++ b/molecule/elasticsearch_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_no-security/molecule.yml b/molecule/elasticsearch_no-security/molecule.yml index 83ba94da..9855d8e2 100644 --- a/molecule/elasticsearch_no-security/molecule.yml +++ b/molecule/elasticsearch_no-security/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml index 7f5a962d..6e7d35a3 100644 --- a/molecule/elasticsearch_roles_calculation/molecule.yml +++ b/molecule/elasticsearch_roles_calculation/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index f92ae58b..d658c84d 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_full_stack-oss/molecule.yml b/molecule/logstash_full_stack-oss/molecule.yml index e053a2f7..ebd40da8 100644 --- a/molecule/logstash_full_stack-oss/molecule.yml +++ b/molecule/logstash_full_stack-oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_pipelines/molecule.yml b/molecule/logstash_pipelines/molecule.yml index 204e1b92..e2627f02 100644 --- a/molecule/logstash_pipelines/molecule.yml +++ b/molecule/logstash_pipelines/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/logstash_specific_version/molecule.yml b/molecule/logstash_specific_version/molecule.yml index a4024619..5b4cf3c0 100644 --- a/molecule/logstash_specific_version/molecule.yml +++ b/molecule/logstash_specific_version/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/repos_default/molecule.yml b/molecule/repos_default/molecule.yml index dbbe9ea0..3c857a61 100644 --- a/molecule/repos_default/molecule.yml +++ b/molecule/repos_default/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: diff --git a/molecule/repos_oss/molecule.yml b/molecule/repos_oss/molecule.yml index a083fcd5..9527212b 100644 --- a/molecule/repos_oss/molecule.yml +++ b/molecule/repos_oss/molecule.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.yml driver: name: docker platforms: From c2dc5295f3c3ac793d94bfefbecb069e159ff160 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 7 Sep 2023 09:01:01 +0200 Subject: [PATCH 02/18] Install cryptography Python library (#213) We had this as a requirement. But it makes sense to have the role install the library without any further interaction with the user. fixes #212 --- docs/role-beats.md | 2 -- docs/role-elasticsearch.md | 5 ----- docs/role-kibana.md | 5 ----- docs/role-logstash.md | 1 - roles/beats/defaults/main.yml | 1 - roles/beats/tasks/beats-security.yml | 10 +++++++--- roles/elasticsearch/tasks/elasticsearch-security.yml | 12 ++++++++++++ roles/kibana/tasks/kibana-security.yml | 7 +++++-- roles/logstash/tasks/logstash-security.yml | 9 ++++++--- 9 files changed, 30 insertions(+), 22 deletions(-) diff --git a/docs/role-beats.md b/docs/role-beats.md index 2f45e14f..73722a6c 100644 --- a/docs/role-beats.md +++ b/docs/role-beats.md @@ -10,7 +10,6 @@ Requirements You need to have the beats you want to install available in your software repositories. We provide a [role](./role-repos.md) for just that but if you have other ways of managing software, just make sure it's available. Alternatively you can install the Beats yourself. -* `cryptography` >= 2.5 * `community.crypto` collection: ansible-galaxy collection install community.crypto Role Variables @@ -87,7 +86,6 @@ beats_filebeat_journald_inputs: * *beats_loglevel*: Level of logging (for all beats) (Default: `info`) * *beats_logpath*: If logging to file, where to put logfiles (Default: `/var/log/beats`) * *beats_fields*: Fields that are added to every input in the configuration -* *beats_manage_unzip*: Install `unzip` via package manager (Default: `true`) The following variables only apply if you use this role together with our other Elastic Stack roles. diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md index 2db6c39b..bfcdf3d1 100644 --- a/docs/role-elasticsearch.md +++ b/docs/role-elasticsearch.md @@ -9,11 +9,6 @@ If you use the role to set up security you, can use its CA to create certificate Please note that setting `elasticsearch_bootstrap_pw` as variable will only take effect when initialising Elasticsearch. Changes after starting elasticsearch for the first time will not change the bootstrap password for the instance and will lead to breaking tests. -Requirements ------------- - -* `cryptography` >= 2.5 - Role Variables -------------- diff --git a/docs/role-kibana.md b/docs/role-kibana.md index 28abf012..07d17653 100644 --- a/docs/role-kibana.md +++ b/docs/role-kibana.md @@ -5,11 +5,6 @@ Ansible Role: Kibana This roles installs and configures Kibana. -Requirements ------------- - -* `cryptography` >= 2.5 - Role Variables -------------- diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 010dff1a..cf8f331e 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -19,7 +19,6 @@ Requirements ------------ * `community.general` collection -* `cryptography` >= 2.5 You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) diff --git a/roles/beats/defaults/main.yml b/roles/beats/defaults/main.yml index 7f9550fc..5708617b 100644 --- a/roles/beats/defaults/main.yml +++ b/roles/beats/defaults/main.yml @@ -10,7 +10,6 @@ elasticstack_beats_port: 5044 beats_logging: file beats_logpath: /var/log/beats beats_loglevel: info -beats_manage_unzip: true # Use TLS without Elastic X-Pack # diff --git a/roles/beats/tasks/beats-security.yml b/roles/beats/tasks/beats-security.yml index 0a19489c..ef034ee3 100644 --- a/roles/beats/tasks/beats-security.yml +++ b/roles/beats/tasks/beats-security.yml @@ -1,11 +1,15 @@ --- -- name: Install unzip for certificate handling +- name: Install packages for security tasks ansible.builtin.package: - name: unzip - when: beats_manage_unzip | bool + name: + - unzip + - python3-cryptography + - openssl tags: + - certificates - renew_ca + - renew_kibana_cert - renew_beats_cert - name: Ensure beats certificate exists diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 65483fc3..7b3ac565 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -1,5 +1,17 @@ --- +- name: Install packages for security tasks + ansible.builtin.package: + name: + - unzip + - python3-cryptography + - openssl + tags: + - certificates + - renew_ca + - renew_kibana_cert + - renew_es_cert + - name: Set elasticstack_ca variable if not already done by user ansible.builtin.set_fact: elasticstack_ca: "{{ groups['elasticsearch'][0] }}" diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index d7058afb..4bb14fbd 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -1,8 +1,11 @@ --- -- name: Make sure openssl is installed +- name: Install packages for security tasks ansible.builtin.package: - name: openssl + name: + - unzip + - python3-cryptography + - openssl tags: - certificates - renew_ca diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index f0cae832..330d6a20 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -1,8 +1,11 @@ --- -- name: Install unzip for certificate handling +- name: Install packages for security tasks ansible.builtin.package: - name: unzip + name: + - unzip + - python3-cryptography + - openssl tags: - certificates - renew_ca @@ -383,7 +386,7 @@ - name: Create logstash password hash salt ansible.builtin.copy: - content: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed)}}" + content: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed) }}" dest: /root/logstash_password_hash_salt owner: root group: root From 028a3ec565b4582106d2c7d64fe9ba75e64f6b7e Mon Sep 17 00:00:00 2001 From: Afeef Ghannam <39904920+afeefghannam89@users.noreply.github.com> Date: Thu, 7 Sep 2023 10:24:52 +0200 Subject: [PATCH 03/18] Fix/debian pipeline (#269) Close #258 This PR will fix the problem on debian the problem on rocky is already fixed in another branch, but GitHub did not close the issue that time, I do not know why. --- roles/elasticsearch/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 7677f040..6de5b3f1 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -175,7 +175,7 @@ - name: Force systemd to reread configs on container ansible.builtin.systemd: daemon_reload: true - when: ansible_virtualization_type == "container" + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" # Free up some space to let elsticsearch allocate replica in GitHub Action - name: Remove cache # noqa: risky-shell-pipe @@ -184,7 +184,7 @@ rm -rf /var/cache/* failed_when: false changed_when: false - when: ansible_virtualization_type == "container" + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" - name: Import Tasks elasticsearch-security.yml ansible.builtin.import_tasks: elasticsearch-security.yml From d6d2e75928bd251f19dadf9f9a9962f91b08bc4d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 7 Sep 2023 11:59:51 +0200 Subject: [PATCH 04/18] Remove redundant Beats include in test (#268) We don't need to run the beats role twice. fixes #267 --- molecule/elasticstack_default/converge.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 8cde64e3..718dc256 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -40,6 +40,9 @@ - name: Include logstash ansible.builtin.include_role: name: logstash + - name: Include kibana + ansible.builtin.include_role: + name: kibana - name: Include Beats ansible.builtin.include_role: name: beats @@ -54,9 +57,3 @@ ansible.builtin.service: name: rsyslog state: started - - name: Include kibana - ansible.builtin.include_role: - name: kibana - - name: Include Beats - ansible.builtin.include_role: - name: beats From dc2f8777089effb3391345318a7b19e419223166 Mon Sep 17 00:00:00 2001 From: Lucinda Siemer <118279543+dnssmr@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:24:16 +0200 Subject: [PATCH 05/18] Feature/use uri not curl 123 (#186) Close #123 Close #266 Close #265 Close #11 --------- Co-authored-by: Afeef Ghannam Co-authored-by: Afeef Ghannam <39904920+afeefghannam89@users.noreply.github.com> Co-authored-by: Thomas Widhalm Co-authored-by: denise --- README.md | 17 +- docs/role-logstash.md | 6 +- requirements-test.txt | 2 +- .../tasks/elasticsearch-keystore.yml | 184 +++++++++++++++ .../tasks/elasticsearch-security.yml | 214 ++---------------- roles/elasticsearch/tasks/main.yml | 17 +- .../templates/elasticsearch.yml.j2 | 3 + roles/logstash/defaults/main.yml | 1 + roles/logstash/tasks/logstash-security.yml | 177 ++++++--------- .../templates/logstash_writer_user.j2 | 6 +- 10 files changed, 293 insertions(+), 334 deletions(-) create mode 100644 roles/elasticsearch/tasks/elasticsearch-keystore.yml diff --git a/README.md b/README.md index 4665f5ab..b401cb14 100644 --- a/README.md +++ b/README.md @@ -35,21 +35,27 @@ collections: You will need the following Ansible collections installed -* community.general (probably already present) +* `community.general` (probably already present) + +You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. + +* `passlib` Python library if you do not disable password hashing for logstash user and you want to use logstash role from this collection. It should be installed with pip on the Ansible controller. You may want the following Ansible roles installed. There other ways to achieve what they are doing but using them is easy and convenient. -* geerlingguy.redis -* openssl if you want to use Elastic Security +* `geerlingguy.redis` if you want to use logstash role ### Supported systems We test the collection on the following Linux distributions. Each one with Elastic Stack 7 and 8. +* Rocky Linux 9 * Rocky Linux 8 -* Ubuntu 20.04 LTS * Ubuntu 22.04 LTS +* Ubuntu 20.04 LTS * Debian 11 +* Debian 10 +* CentOS 8 We know from personal experience, that the collections work in following combinations. Missing tests mostly come from incompatibilties between the distribution and our testing environment, not from problems with the collection itself. @@ -57,9 +63,6 @@ We know from personal experience, that the collections work in following combina ### Known Issues -There are known issues with the following Linux distributions. - -* Rocky Linux 9: The GnuPG key used by Elastic seems to be incompatible with this version of Rocky. ## Usage diff --git a/docs/role-logstash.md b/docs/role-logstash.md index cf8f331e..f489a25d 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -20,6 +20,10 @@ Requirements * `community.general` collection +You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. + +* `passlib` Python library if you do not disable password hashing for logstash user. It should be installed with pip on the Ansible controller. + You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) If you want to use the default pipeline configuration you need to have `git` available. @@ -68,7 +72,7 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_password_hash*: Generate and use a hash from your `logstash_password` (default: `true`) * *logstash_password_hash_algorithm*: Password hashing algorithms. Value must be same as `xpack.security.authc.password_hashing.algorithm` (default: `bcrypt`) * *logstash_password_salt_length*: base64 encoded Salt character lenght. This value must be integer and must be compatible to the selected password hashing algorithms (default: `22`) -**logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) +* *logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) * *logstash_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) * *logstash_user_indices*: Indices the user has access to (default: `'"ecs-logstash*", "logstash*", "logs*"'`) * *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) diff --git a/requirements-test.txt b/requirements-test.txt index c67fdccb..ac24d2cf 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,4 @@ ansible-lint molecule molecule-plugins[docker] pytest -passlib \ No newline at end of file +passlib diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml new file mode 100644 index 00000000..31481fc8 --- /dev/null +++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml @@ -0,0 +1,184 @@ +--- + +- name: Create keystore + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore create + args: + creates: /etc/elasticsearch/elasticsearch.keystore + +- name: Check for bootstrap password + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list + changed_when: false + register: elasticsearch_keystore + +- name: Set bootstrap password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_bootstrap_pw }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -x 'bootstrap.password' + when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines" + changed_when: false + no_log: true + notify: + - Restart Elasticsearch + ignore_errors: "{{ ansible_check_mode }}" + +- name: Get xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.http.ssl.keystore.secure_password' + when: + - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_http_security + register: elasticsearch_http_ssl_keystore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.http.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout + - elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.http.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.http.ssl.truststore.secure_password' + when: + - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_http_security + register: elasticsearch_http_ssl_truststore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.http.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout + - elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.http.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_http_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.transport.ssl.keystore.secure_password' + when: + - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_security + register: elasticsearch_transport_ssl_keystore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.transport.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout + - elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.transport.ssl.keystore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Get xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + show 'xpack.security.transport.ssl.truststore.secure_password' + when: + - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - elasticsearch_security + register: elasticsearch_transport_ssl_truststore_secure_password + ignore_errors: "{{ ansible_check_mode }}" + no_log: true + changed_when: false + +- name: Set xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + echo "{{ elasticsearch_tls_key_passphrase }}" | + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x 'xpack.security.transport.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout + - elasticsearch_security + notify: + - Restart Elasticsearch + +- name: Remove xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove 'xpack.security.transport.ssl.truststore.secure_password' + changed_when: false + no_log: true + when: + - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" + - not elasticsearch_security + notify: + - Restart Elasticsearch diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 7b3ac565..0b1a95ac 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -234,188 +234,8 @@ - renew_ca - renew_es_cert -- name: Create keystore - ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore create - args: - creates: /etc/elasticsearch/elasticsearch.keystore - -- name: Check for bootstrap password - ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list - changed_when: false - register: elasticsearch_keystore - -- name: Set bootstrap password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_bootstrap_pw }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -x 'bootstrap.password' - when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines" - changed_when: true - no_log: "{{ elasticstack_no_log }}" - notify: - - Restart Elasticsearch - ignore_errors: "{{ ansible_check_mode }}" - -- name: Get xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.http.ssl.keystore.secure_password' - when: - - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_http_security - register: elasticsearch_http_ssl_keystore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.http.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout - - elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.http.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.http.ssl.truststore.secure_password' - when: - - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_http_security - register: elasticsearch_http_ssl_truststore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.http.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout - - elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.http.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_http_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.transport.ssl.keystore.secure_password' - when: - - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_security - register: elasticsearch_transport_ssl_keystore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.transport.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout - - elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.transport.ssl.keystore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Get xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - show 'xpack.security.transport.ssl.truststore.secure_password' - when: - - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - elasticsearch_security - register: elasticsearch_transport_ssl_truststore_secure_password - ignore_errors: "{{ ansible_check_mode }}" - no_log: "{{ elasticstack_no_log }}" - changed_when: false - -- name: Set xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - echo "{{ elasticsearch_tls_key_passphrase }}" | - /usr/share/elasticsearch/bin/elasticsearch-keystore - add -f -x 'xpack.security.transport.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout - - elasticsearch_security - notify: - - Restart Elasticsearch - -- name: Remove xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - /usr/share/elasticsearch/bin/elasticsearch-keystore - remove 'xpack.security.transport.ssl.truststore.secure_password' - changed_when: true - no_log: "{{ elasticstack_no_log }}" - when: - - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines" - - not elasticsearch_security - notify: - - Restart Elasticsearch +- name: Import Tasks elasticsearch-keystore.yml + ansible.builtin.import_tasks: elasticsearch-keystore.yml - name: Create ca and certificates on elasticstack_ca host when: inventory_hostname == elasticstack_ca @@ -563,20 +383,19 @@ elasticsearch_http_protocol: "https" when: elasticsearch_http_security -- name: Check for cluster status with bootstrap password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -ks - {{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 +- name: Check for cluster status with bootstrap password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + user: elastic + password: "{{ elasticsearch_bootstrap_pw }}" + validate_certs: false register: elasticsearch_cluster_status_bootstrap changed_when: false no_log: "{{ elasticstack_no_log }}" when: - not elasticsearch_passwords_file.stat.exists | bool - groups['elasticsearch'] | length > 1 - until: elasticsearch_cluster_status_bootstrap.stdout == "green" + until: elasticsearch_cluster_status_bootstrap.json.status == "green" retries: 5 delay: 10 @@ -591,20 +410,19 @@ delegate_to: "{{ elasticstack_ca }}" when: elasticsearch_passwords_file.stat.exists | bool -- name: Check for cluster status with elastic password # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -ks - {{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 +- name: Check for cluster status with elastic password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: false register: elasticsearch_cluster_status changed_when: false no_log: "{{ elasticstack_no_log }}" when: - elasticsearch_passwords_file.stat.exists | bool - groups['elasticsearch'] | length > 1 - until: elasticsearch_cluster_status.stdout == "green" + until: elasticsearch_cluster_status.json.status == "green" retries: 20 delay: 10 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 6de5b3f1..85332673 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -205,20 +205,15 @@ - name: Handle cluster setup without security when: not elasticsearch_security | bool block: - - name: Check for cluster status without security # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl -s - http://localhost:{{ elastic_elasticsearch_http_port }}/_cluster/health?pretty | - grep status | - cut -d\" -f4 - register: es_cluster_status - changed_when: false - no_log: "{{ elasticstack_no_log }}" + - name: Check for cluster status without security + ansible.builtin.uri: + url: "http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" + register: elasticsearch_cluster_status ignore_errors: "{{ ansible_check_mode }}" - until: es_cluster_status.stdout == "green" + until: elasticsearch_cluster_status.json.status == "green" retries: 5 delay: 10 + no_log: "{{ elasticstack_no_log }}" - name: Leave a file showing that the cluster is set up ansible.builtin.template: diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 index b71c3ee8..c809f309 100644 --- a/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -62,6 +62,9 @@ xpack.security.http.ssl.keystore.path: certs/{{ ansible_hostname }}.p12 xpack.security.http.ssl.truststore.path: certs/{{ ansible_hostname }}.p12 {% endif %} {% endif %} +{% if not elasticsearch_security | bool %} +xpack.security.enabled: false +{% endif %} {% endif %} {% if elasticsearch_fs_repo is defined %} diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 27f8a16d..8ff97804 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -47,6 +47,7 @@ logstash_sniffing: false # logstash security logstash_password_hash: true logstash_password_hash_algorithm: bcrypt +logstash_password_salt_length: 22 logstash_user: logstash_writer logstash_password: password logstash_password_hash_salt_length: 22 diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 330d6a20..5a93e9d9 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -369,54 +369,16 @@ - renew_ca - renew_logstash_cert -- name: Place logstash_writer role configuration on ca node - ansible.builtin.template: - dest: /root/logstash_writer_role - src: logstash_writer_role.j2 - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - run_once: true - - name: Check the length of logstash user password ansible.builtin.fail: msg: logstash user password must be at least 6 characters long. when: logstash_password | length < 6 -- name: Create logstash password hash salt - ansible.builtin.copy: - content: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed) }}" - dest: /root/logstash_password_hash_salt - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca - -- name: Read password hash salt - ansible.builtin.slurp: - src: /root/logstash_password_hash_salt - register: logstash_password_hash_salt_file - delegate_to: "{{ elasticstack_ca }}" - when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca - - name: Set password hash salt as a fact ansible.builtin.set_fact: - logstash_password_hash_salt: "{{ logstash_password_hash_salt_file['content'] | b64decode }}" - delegate_to: "{{ elasticstack_ca }}" + logstash_password_hash_salt: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed) }}" when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca -- name: Place logstash_writer user configuration on ca node - ansible.builtin.template: - dest: /root/logstash_writer_user - src: logstash_writer_user.j2 - owner: root - group: root - mode: 0600 - delegate_to: "{{ elasticstack_ca }}" - run_once: true - - name: Fetch Elastic password # noqa: risky-shell-pipe ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; @@ -430,94 +392,81 @@ - configuration - logstash_configuration -- name: Check for logstash_writer role # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/role/logstash_writer - | grep "enabled" - delegate_to: "{{ elasticstack_ca }}" - failed_when: false - changed_when: false - register: logstash_writer_role_present - no_log: "{{ elasticstack_no_log }}" - run_once: true - -# we doubled the task and didn't use a more sophisticated way to just change -# the URI because we expect this task to be removed when ES 7 is out of -# support +- name: Set elasticsearch security-api base url for elasticsearch > 7 + ansible.builtin.set_fact: + security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/" + when: elasticstack_release | int > 7 -- name: Put logstash_writer role into Elasticsearch < 8 - ansible.builtin.command: > - curl -T /root/logstash_writer_role --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/role/logstash_writer +- name: Set elasticsearch security-api base url for elasticsearch < 8 + ansible.builtin.set_fact: + security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/" + when: elasticstack_release | int < 8 + +- name: Check for logstash_writer role + ansible.builtin.uri: + url: "{{ security_api_base_url }}role/logstash_writer" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + register: check_logstash_writer_role_response delegate_to: "{{ elasticstack_ca }}" + failed_when: false changed_when: false no_log: "{{ elasticstack_no_log }}" run_once: true - when: - - logstash_writer_role_present.rc > 0 or logstash_reset_writer_role | bool - - elasticstack_release | int < 8 -- name: Put logstash_writer role into Elasticsearch > 7 - ansible.builtin.command: > - curl -T /root/logstash_writer_role --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/role/logstash_writer +- name: Set logstash_writer_role_present + ansible.builtin.set_fact: + logstash_writer_role_present: true + when: check_logstash_writer_role_response.json.logstash_writer is defined + +- name: Put logstash_writer role into Elasticsearch if not present + ansible.builtin.uri: + url: "{{ security_api_base_url }}role/logstash_writer" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + method: PUT + headers: + Content-Type: application/json + body: "{{ lookup('template', 'logstash_writer_role.j2') }}" + body_format: json + register: put_logstash_writer_role_response + when: logstash_writer_role_present is not defined delegate_to: "{{ elasticstack_ca }}" - changed_when: false + failed_when: not put_logstash_writer_role_response.json.role.created | bool run_once: true - no_log: "{{ elasticstack_no_log }}" - when: - - logstash_writer_role_present.rc > 0 or logstash_reset_writer_role | bool - - elasticstack_release | int > 7 -- name: Check for logstash_writer user # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/{{ logstash_user }} - | grep "enabled" +- name: Check for logstash_writer user + ansible.builtin.uri: + url: "{{ security_api_base_url }}user/{{ logstash_user }}" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + register: check_logstash_writer_user_response delegate_to: "{{ elasticstack_ca }}" failed_when: false changed_when: false - no_log: "{{ elasticstack_no_log }}" - register: logstash_writer_user_present run_once: true -# we doubled the task and didn't use a more sophisticated way to just change -# the URI because we expect this task to be removed when ES 7 is out of -# support - -- name: Put logstash_writer user into Elasticsearch < 8 - ansible.builtin.command: > - curl -T /root/logstash_writer_user --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/user/{{ logstash_user }} - delegate_to: "{{ elasticstack_ca }}" - changed_when: false - no_log: "{{ elasticstack_no_log }}" - run_once: true - when: - - logstash_writer_user_present.rc > 0 - - elasticstack_release | int < 8 - -- name: Put logstash_writer user into Elasticsearch > 7 - ansible.builtin.command: > - curl -T /root/logstash_writer_user --header 'Content-Type: application/json' - --cacert {{ elasticstack_ca_dir }}/ca.crt - -u elastic:{{ logstash_elasticstack_password.stdout }} - https://{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/{{ logstash_user }} +- name: Set logstash_writer_user_present + ansible.builtin.set_fact: + logstash_writer_user_present: true + when: check_logstash_writer_user_response.json.logstash_writer.username is defined and check_logstash_writer_user_response.json.logstash_writer.username == "logstash_writer" + +- name: Put logstash_writer user into Elasticsearch if not present + ansible.builtin.uri: + url: "{{ security_api_base_url }}user/{{ logstash_user }}" + ca_path: "{{ elasticstack_ca_dir }}/ca.crt" + user: elastic + password: "{{ logstash_elasticstack_password.stdout }}" + method: PUT + headers: + Content-Type: application/json + body: "{{ lookup('template', 'logstash_writer_user.j2') }}" + body_format: json + register: put_logstash_writer_user_response + when: logstash_writer_user_present is not defined delegate_to: "{{ elasticstack_ca }}" run_once: true - no_log: "{{ elasticstack_no_log }}" - changed_when: false - when: - - logstash_writer_user_present.rc > 0 - - elasticstack_release | int > 7 + failed_when: not put_logstash_writer_user_response.json.created diff --git a/roles/logstash/templates/logstash_writer_user.j2 b/roles/logstash/templates/logstash_writer_user.j2 index 7335bf68..5c21a745 100644 --- a/roles/logstash/templates/logstash_writer_user.j2 +++ b/roles/logstash/templates/logstash_writer_user.j2 @@ -1,7 +1,9 @@ { {% if logstash_password_hash | bool %} -{# using a fixed salt is neccessary for idempotency, will be created on elasticstack CA host #} - "password_hash" : "{{ logstash_password | password_hash( hashtype=logstash_password_hash_algorithm, salt=logstash_password_hash_salt, ident='2a' ) }}", +{# using a fixed salt is neccessary for idempotency, will be generated as a set fact. +rounds specifies the bcrypt version. The default version in Ansible module is 12. The acceptable one is 10 on elasticsearch 7. +On elasticsearch 8, the 12 and 10 versions will work, so we should use 10 until the support of 7 stops #} + "password_hash" : "{{ logstash_password | password_hash( hashtype=logstash_password_hash_algorithm, salt=logstash_password_hash_salt, ident='2a', rounds=10 ) }}", {% else %} "password" : "{{ logstash_password }}", {% endif %} From 058e36d2ef5700cde5f9f2019dff64796586f206 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 8 Sep 2023 11:11:03 +0200 Subject: [PATCH 06/18] Remove cach again after all roles --- .github/workflows/test_roles_pr.yml | 5 +++-- molecule/elasticstack_default/converge.yml | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index ad50534f..05027e3b 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -51,8 +51,9 @@ jobs: max-parallel: 2 matrix: distro: - - rockylinux8 - - ubuntu2204 + - centos8 + #- rockylinux8 + #- ubuntu2204 scenario: - elasticstack_default release: diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 718dc256..e993175a 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -49,6 +49,11 @@ - name: Install rsyslog ansible.builtin.package: name: rsyslog + - name: Remove cache # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + rm -rf /var/cache/* + changed_when: false - name: Configure rsyslog ansible.builtin.lineinfile: line: "*.* @@localhost:514" From a2cc6a14bd896f6602b44a70c1ceed528dc0bc51 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 8 Sep 2023 11:13:04 +0200 Subject: [PATCH 07/18] Undo changes --- .github/workflows/test_roles_pr.yml | 5 ++--- molecule/elasticstack_default/converge.yml | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 05027e3b..ad50534f 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -51,9 +51,8 @@ jobs: max-parallel: 2 matrix: distro: - - centos8 - #- rockylinux8 - #- ubuntu2204 + - rockylinux8 + - ubuntu2204 scenario: - elasticstack_default release: diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index e993175a..718dc256 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -49,11 +49,6 @@ - name: Install rsyslog ansible.builtin.package: name: rsyslog - - name: Remove cache # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - rm -rf /var/cache/* - changed_when: false - name: Configure rsyslog ansible.builtin.lineinfile: line: "*.* @@localhost:514" From 59b9aef5870121b5302bdfe57e489fa30d0e8c44 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 8 Sep 2023 12:31:48 +0200 Subject: [PATCH 08/18] Add "extra config" to `kibana.yml` (#270) fixes #208 --- docs/role-kibana.md | 1 + molecule/elasticstack_default/converge.yml | 2 ++ roles/kibana/templates/kibana.yml.j2 | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/docs/role-kibana.md b/docs/role-kibana.md index 07d17653..0581b9ff 100644 --- a/docs/role-kibana.md +++ b/docs/role-kibana.md @@ -13,6 +13,7 @@ Role Variables * *kibana_tls*: Whether to offer `https` for clients or not (default: `false`) * *kibana_tls_cert*: Path to the certificate Kibana should show to its clients (default: `/etc/kibana/certs/cert.pem`) * *kibana_tls_key*: Path to the key Kibana should use when communicating with clients (default: `/etc/kibana/certs/key.pem`) +* *kibana_extra_config*: You can add arbitraty configuration options with this option. Just start it with `|-` and indent the following lines. So you can add as many lines and options to `kibana.yml` as you like. (default: none) * *kibana_security*: Activate TLS and authentication when connecting to Elasticsearch. **Note**: Only works when `elasticstack_full_stack` is enabled. (default: `true`) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 718dc256..35dd7f8e 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -23,6 +23,8 @@ - system beats_fields: - "testbed: molecule" + kibana_extra_config: |- + ops.interval: 5000 tasks: - name: Enable Elastic installation on RHEL 9 ansible.builtin.set_fact: diff --git a/roles/kibana/templates/kibana.yml.j2 b/roles/kibana/templates/kibana.yml.j2 index eedc5018..b401caa0 100644 --- a/roles/kibana/templates/kibana.yml.j2 +++ b/roles/kibana/templates/kibana.yml.j2 @@ -30,3 +30,7 @@ server.ssl.enabled: true server.ssl.certificate: "{{ kibana_tls_cert }}" server.ssl.key: "{{ kibana_tls_key }}" {% endif %} + +{% if kibana_extra_config is defined %} +{{ kibana_extra_config }} +{% endif %} From 96c04c06263ada37b4b89d655f0ac25cb8cbc54d Mon Sep 17 00:00:00 2001 From: Afeef Ghannam <39904920+afeefghannam89@users.noreply.github.com> Date: Sun, 10 Sep 2023 13:47:41 +0200 Subject: [PATCH 09/18] * Remove cache after all roles run (#271) We should free up space after the roles run to have green cluster state in impotence step and enable Elasticsearch to allocate all indices. Close #272 --- molecule/elasticstack_default/converge.yml | 5 +++++ roles/elasticsearch/tasks/main.yml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 35dd7f8e..61ec7344 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -51,6 +51,11 @@ - name: Install rsyslog ansible.builtin.package: name: rsyslog + - name: Remove cache # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + rm -rf /var/cache/* + changed_when: false - name: Configure rsyslog ansible.builtin.lineinfile: line: "*.* @@localhost:514" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 85332673..4b813117 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -182,7 +182,6 @@ ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; rm -rf /var/cache/* - failed_when: false changed_when: false when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" From 62bf591a20234394a76ad43232bd846cb3206841 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 09:20:51 +0200 Subject: [PATCH 10/18] Bump actions/checkout from 3 to 4 (#280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4.
Release notes

Sourced from actions/checkout's releases.

v4.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v4.0.0

v3.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3.5.3...v3.6.0

v3.5.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v3.5.3

v3.5.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v3.5.1...v3.5.2

v3.5.1

What's Changed

New Contributors

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

v4.1.0

v4.0.0

v3.6.0

v3.5.3

v3.5.2

v3.5.1

v3.5.0

v3.4.0

v3.3.0

v3.2.0

v3.1.0

v3.0.2

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test_full_stack.yml | 2 +- .github/workflows/test_linting.yml | 2 +- .github/workflows/test_plugins.yml | 10 +++++----- .github/workflows/test_role_beats.yml | 2 +- .github/workflows/test_role_elasticsearch.yml | 2 +- .github/workflows/test_role_kibana.yml | 2 +- .github/workflows/test_role_logstash.yml | 2 +- .github/workflows/test_role_repos.yml | 2 +- .github/workflows/test_roles_pr.yml | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 6e78a086..3dba4e20 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_linting.yml b/.github/workflows/test_linting.yml index cbb0a4ca..fa0da927 100644 --- a/.github/workflows/test_linting.yml +++ b/.github/workflows/test_linting.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3. uses: actions/setup-python@v4 diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index 75de6cc5..3bd80ab0 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3. uses: actions/setup-python@v4 @@ -70,7 +70,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 @@ -117,7 +117,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v4 @@ -157,7 +157,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 @@ -197,7 +197,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml index 82741d96..45544ba6 100644 --- a/.github/workflows/test_role_beats.yml +++ b/.github/workflows/test_role_beats.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml index 606b92e2..b0da7d9b 100644 --- a/.github/workflows/test_role_elasticsearch.yml +++ b/.github/workflows/test_role_elasticsearch.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml index 2d75cb63..914464eb 100644 --- a/.github/workflows/test_role_kibana.yml +++ b/.github/workflows/test_role_kibana.yml @@ -56,7 +56,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index e13e8384..14198928 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index ab3441c0..7963ce6b 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index ad50534f..525f1691 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 From d5f7f542318a8a64c0c65b6bb3b384b6a686d1b9 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Mon, 16 Oct 2023 18:37:26 +0200 Subject: [PATCH 11/18] Remove redundant restart (#279) Restarting Elasticsearch takes quite a while and may lead to connection issues as well as sync issues. So keeping restarts to a minimum is important. These changes will make sure that, even when the `Restart Elasticsearch` handler is notified, it will only restart if Elasticsearch was running before. If there's a fresh start (after reconfiguration) we don't need to restart again. Same goes for Logstash and Kibana. Some restarts of these tools happen fairly fast. But others (like after fresh installs or updates) will trigger internal jobs that should not be intercepted by another restart. Beats restart very fast and as far as I know there's not a big downside to restarting them right after the first start so I didn't include them in the change. Additionally, this PR will make sure some tasks in `verify.yml` of the full stack are only run when the service to be checked is actually running on this node. This helps with spreading services over nodes to save ressources. Since GitHub hosted runners are quite low on ressources we can't run every service on every node in a cluster setup anymore. So this PR will make sure that only Elasticsearch runs everywhere and the others are spread out. Caches get cleared after every role in during a Molecule test. This helps with saving ressources, too. Elasticsearch still won't sync all shards due to full volumes, the watermarks for Elasticseach are set to extremely high volumes so that the cluster can at least get into sync. fixes #278 fixes #141 fixes #194 --- molecule/elasticsearch_default/converge.yml | 1 + molecule/elasticstack_default/converge.yml | 3 +- molecule/elasticstack_default/molecule.yml | 2 - molecule/elasticstack_default/verify.yml | 78 ++++++------- roles/beats/tasks/main.yml | 7 ++ roles/elasticsearch/defaults/main.yml | 6 + roles/elasticsearch/handlers/main.yml | 5 +- .../tasks/elasticsearch-security.yml | 104 +++++++++++++++--- roles/elasticsearch/tasks/main.yml | 8 +- roles/kibana/defaults/main.yml | 3 + roles/kibana/handlers/main.yml | 2 + roles/kibana/tasks/main.yml | 8 ++ roles/logstash/defaults/main.yml | 5 + roles/logstash/handlers/main.yml | 4 +- roles/logstash/tasks/main.yml | 8 ++ 15 files changed, 183 insertions(+), 61 deletions(-) diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml index 8cba6694..1e836b59 100644 --- a/molecule/elasticsearch_default/converge.yml +++ b/molecule/elasticsearch_default/converge.yml @@ -12,6 +12,7 @@ elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticsearch_heap: "1" + elasticstack_no_log: false tasks: - name: Include Elastics repos role ansible.builtin.include_role: diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 61ec7344..009b0fdd 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -12,8 +12,9 @@ vars: elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true + elasticsearch_monitoring_enabled: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticsearch_heap: "1" + elasticsearch_heap: "2" elasticstack_full_stack: true elasticstack_no_log: false logstash_pipeline_unsafe_shutdown: true diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index d658c84d..513db812 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -10,7 +10,6 @@ platforms: groups: - beats - logstash - - kibana - elasticsearch image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} @@ -22,7 +21,6 @@ platforms: - name: "elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}" groups: - beats - - logstash - kibana - elasticsearch image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index abceef3f..3a2e8430 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -59,42 +59,46 @@ msg: "Elasticsearch received {{ logstash_count.stdout }} events so far" when: "'elasticsearch' in group_names" - - name: fetch kibana.yml - ansible.builtin.command: cat /etc/kibana/kibana.yml - register: kibanayml - - - name: Show kibana.yml - ansible.builtin.debug: - var: kibanayml.stdout_lines - - - name: Check for Kibana port - ansible.builtin.wait_for: - port: 5601 - timeout: 120 - - - name: Connect to Kibana - ansible.builtin.command: - curl - -s - -u elastic:{{ elastic_pass.stdout }} - http://{{ ansible_hostname }}:5601/api/status - register: curl_out - failed_when: - - "'green' not in curl_out.stdout" - - "'Elasticsearch is available' not in curl_out.stdout" - - # The following might be nicer but doesn't work - #- name: Connect to Kibana - # ansible.builtin.uri: - # url: http://ansible-role-kibana_full_stack:5601/api/status - # user: elastic - # password: "{{ elastic_password.stdout }}" - # return_content: yes - # register: kibana_status - # #failed_when: "'"title": "Green"' not in kibana_status.content" - # failed_when: "'Green' not in kibana_status.content" - - - name: Health check + - name: Run Kibana checks + when: "'kibana' in group_names" + block: + + - name: Fetch kibana.yml + ansible.builtin.command: cat /etc/kibana/kibana.yml + register: kibanayml + + - name: Show kibana.yml + ansible.builtin.debug: + var: kibanayml.stdout_lines + + - name: Check for Kibana port + ansible.builtin.wait_for: + port: 5601 + timeout: 120 + + - name: Connect to Kibana + ansible.builtin.command: + curl + -s + -u elastic:{{ elastic_pass.stdout }} + http://{{ ansible_hostname }}:5601/api/status + register: curl_out + failed_when: + - "'green' not in curl_out.stdout" + - "'Elasticsearch is available' not in curl_out.stdout" + + # The following might be nicer but doesn't work + #- name: Connect to Kibana + # ansible.builtin.uri: + # url: http://ansible-role-kibana_full_stack:5601/api/status + # user: elastic + # password: "{{ elastic_password.stdout }}" + # return_content: yes + # register: kibana_status + # #failed_when: "'"title": "Green"' not in kibana_status.content" + # failed_when: "'Green' not in kibana_status.content" + + - name: Elasticsearch health check ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health method: GET @@ -110,7 +114,7 @@ delay: 10 when: groups['elasticsearch'] | length > 1 - - name: Node check + - name: Elasticsearch Node check ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes method: GET diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index f242c0a8..9d521bb1 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -82,3 +82,10 @@ - name: Import Metricbeat tasks ansible.builtin.import_tasks: metricbeat.yml when: beats_metricbeat | bool + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index bcadfb3d..3f531606 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -48,6 +48,12 @@ elasticsearch_cert_expiration_buffer: 30 elasticstack_ca_will_expire_soon: false elasticsearch_cert_will_expire_soon: false +# only used internally +elasticsearch_freshstart: + changed: false +elasticsearch_freshstart_security: + changed: false + # "global" variables for all roles elasticstack_release: 8 diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index b39f884a..ff3b5ab5 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -5,7 +5,10 @@ name: elasticsearch state: restarted daemon_reload: yes - when: elasticsearch_enable | bool + when: + - elasticsearch_enable | bool + - not elasticsearch_freshstart.changed | bool + - not elasticsearch_freshstart_security.changed | bool - name: Restart kibana if available for elasticsearch certificates ansible.builtin.include_tasks: handlers/restart_kibana.yml diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 0b1a95ac..48bcb2aa 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -352,25 +352,31 @@ name: elasticsearch state: started enabled: yes + register: elasticsearch_freshstart_security - name: Wait for all instances to start ansible.builtin.include_tasks: wait_for_instance.yml loop: "{{ groups['elasticsearch'] }}" -- name: Force all notified handlers to run at this point, not waiting for normal sync points - ansible.builtin.meta: flush_handlers - tags: - - certificates - - renew_ca - - renew_es_cert - -- name: Wait for all instances to start - ansible.builtin.include_tasks: wait_for_instance.yml - loop: "{{ groups['elasticsearch'] }}" - tags: - - certificates - - renew_ca - - renew_es_cert +- name: Restart if Elasticsearch was already running + when: + - not elasticsearch_freshstart.changed | bool + - not elasticsearch_freshstart_security.changed | bool + block: + - name: Force all notified handlers to run at this point, not waiting for normal sync points + ansible.builtin.meta: flush_handlers + tags: + - certificates + - renew_ca + - renew_es_cert + + - name: Wait for all instances to start + ansible.builtin.include_tasks: wait_for_instance.yml + loop: "{{ groups['elasticsearch'] }}" + tags: + - certificates + - renew_ca + - renew_es_cert - name: Check for passwords being set ansible.builtin.stat: @@ -383,6 +389,25 @@ elasticsearch_http_protocol: "https" when: elasticsearch_http_security +- name: Check for API with bootstrap password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}" + user: elastic + password: "{{ elasticsearch_bootstrap_pw }}" + validate_certs: false + register: elasticsearch_api_status_bootstrap + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - not elasticsearch_passwords_file.stat.exists | bool + - groups['elasticsearch'] | length > 1 + until: elasticsearch_api_status_bootstrap.json.cluster_name is defined + retries: 5 + delay: 10 + +# We need this check twice. One to wait for the API to be actually available. And a second time to +# check the actual return code. Should not cause a huge delay. + - name: Check for cluster status with bootstrap password ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" @@ -410,6 +435,57 @@ delegate_to: "{{ elasticstack_ca }}" when: elasticsearch_passwords_file.stat.exists | bool +- name: Check for API availability with elastic password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}" + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: false + register: elasticsearch_api_status + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - elasticsearch_passwords_file.stat.exists | bool + - groups['elasticsearch'] | length > 1 + until: elasticsearch_api_status.json.cluster_name is defined + retries: 20 + delay: 10 + +- name: Work around low ressources on CI/CD nodes + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" + block: + # Free up some space to let elsticsearch allocate replica in GitHub Action + - name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + + - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + curl + -k + -X PUT + "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings" + -H 'Content-Type: application/json' -d + ' + { + "persistent": { + "cluster.routing.allocation.disk.watermark.low": "97%", + "cluster.routing.allocation.disk.watermark.high": "98%", + "cluster.routing.allocation.disk.watermark.flood_stage": "99%", + "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%" + } + } + ' + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - elasticstack_password.stdout is defined + +# We need this check twice. One to wait for the API to be actually available. And a second time to +# check the actual return code. Should not cause a huge delay. + - name: Check for cluster status with elastic password ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 4b813117..0632f870 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -178,9 +178,8 @@ when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" # Free up some space to let elsticsearch allocate replica in GitHub Action -- name: Remove cache # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; +- name: Remove cache + ansible.builtin.command: > rm -rf /var/cache/* changed_when: false when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" @@ -200,6 +199,7 @@ name: elasticsearch state: started enabled: yes + register: elasticsearch_freshstart - name: Handle cluster setup without security when: not elasticsearch_security | bool @@ -237,8 +237,6 @@ group: root mode: 0644 backup: "{{ elasticsearch_config_backup }}" - notify: - - Restart Elasticsearch when: elasticsearch_manage_yaml | bool - name: Show Info about heap diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml index cc21f125..7aa06f7b 100644 --- a/roles/kibana/defaults/main.yml +++ b/roles/kibana/defaults/main.yml @@ -18,6 +18,9 @@ kibana_cert_will_expire_soon: false kibana_sniff_on_start: false kibana_sniff_on_connection_fault: false +kibana_freshstart: + changed: false + # "global" variables for all roles elasticstack_release: 8 elasticstack_full_stack: true diff --git a/roles/kibana/handlers/main.yml b/roles/kibana/handlers/main.yml index 81ffa146..532d014a 100644 --- a/roles/kibana/handlers/main.yml +++ b/roles/kibana/handlers/main.yml @@ -4,3 +4,5 @@ ansible.builtin.service: name: kibana state: restarted + when: + - not kibana_freshstart.changed | bool diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index b5b3ebb9..74e91344 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -82,6 +82,7 @@ state: started enabled: yes when: kibana_enable | bool + register: kibana_freshstart # the following is useful when running tests or extra tasks that need to # have Kibana running. Escape it on Rocky8, because it gets time out with Elastic 8 @@ -90,3 +91,10 @@ ansible.builtin.wait_for: host: localhost port: 5601 + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 8ff97804..1941e792 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -73,6 +73,11 @@ logstash_pipeline_identifier: true logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false +# Only for internal use + +logstash_freshstart: + changed: false + elasticstack_ca_dir: /opt/es-ca elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_ca_pass: PleaseChangeMe diff --git a/roles/logstash/handlers/main.yml b/roles/logstash/handlers/main.yml index eb55a868..08b3b71b 100644 --- a/roles/logstash/handlers/main.yml +++ b/roles/logstash/handlers/main.yml @@ -4,7 +4,9 @@ ansible.builtin.service: name: logstash state: restarted - when: logstash_enable | bool + when: + - logstash_enable | bool + - not logstash_freshstart.changed | bool - name: Restart Logstash noauto ansible.builtin.service: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 1dcee30b..a1d1b3de 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -230,3 +230,11 @@ state: started enabled: yes when: logstash_enable | bool + register: logstash_freshstart + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" From 976d73925fbf92c9518e69ccaa865149efec8169 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 17 Oct 2023 10:00:04 +0200 Subject: [PATCH 12/18] Set Debian 11 as new default distro for molecule (#277) We had CentOS 7 long enough so I thought it might be nice to go with Debian instead. This PR will not change much in the behaviour because usually we overwrite the default. So this is just in case and to stay up to date. fixes #245 --- molecule/beats_default/molecule.yml | 4 ++-- molecule/beats_peculiar/molecule.yml | 4 ++-- molecule/elasticsearch_cluster-oss/molecule.yml | 4 ++-- molecule/elasticsearch_default/molecule.yml | 4 ++-- molecule/elasticsearch_no-security/molecule.yml | 4 ++-- molecule/elasticsearch_roles_calculation/molecule.yml | 6 +++--- molecule/elasticstack_default/molecule.yml | 4 ++-- molecule/kibana_default/molecule.yml | 2 +- molecule/logstash_full_stack-oss/molecule.yml | 2 +- molecule/logstash_pipelines/molecule.yml | 2 +- molecule/logstash_specific_version/molecule.yml | 2 +- molecule/repos_default/molecule.yml | 2 +- molecule/repos_oss/molecule.yml | 2 +- 13 files changed, 21 insertions(+), 21 deletions(-) diff --git a/molecule/beats_default/molecule.yml b/molecule/beats_default/molecule.yml index 9e57f3c0..51fbb100 100644 --- a/molecule/beats_default/molecule.yml +++ b/molecule/beats_default/molecule.yml @@ -4,8 +4,8 @@ dependency: driver: name: docker platforms: - - name: beats_default_${MOLECULE_DISTRO:-centos7} - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + - name: beats_default_${MOLECULE_DISTRO:-debian11} + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/beats_peculiar/molecule.yml b/molecule/beats_peculiar/molecule.yml index 4eb05f8f..bbbd5849 100644 --- a/molecule/beats_peculiar/molecule.yml +++ b/molecule/beats_peculiar/molecule.yml @@ -4,8 +4,8 @@ dependency: driver: name: docker platforms: - - name: beats_peculiar_${MOLECULE_DISTRO:-centos7} - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + - name: beats_peculiar_${MOLECULE_DISTRO:-debian11} + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/elasticsearch_cluster-oss/molecule.yml b/molecule/elasticsearch_cluster-oss/molecule.yml index 50f31e05..7b78fb80 100644 --- a/molecule/elasticsearch_cluster-oss/molecule.yml +++ b/molecule/elasticsearch_cluster-oss/molecule.yml @@ -9,7 +9,7 @@ platforms: - name: elasticsearch-cluster1 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -19,7 +19,7 @@ platforms: - name: elasticsearch-cluster2 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/elasticsearch_default/molecule.yml b/molecule/elasticsearch_default/molecule.yml index 25c5022b..7c2c71b7 100644 --- a/molecule/elasticsearch_default/molecule.yml +++ b/molecule/elasticsearch_default/molecule.yml @@ -9,7 +9,7 @@ platforms: - name: elasticsearch_default1 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -19,7 +19,7 @@ platforms: - name: elasticsearch_default2 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/elasticsearch_no-security/molecule.yml b/molecule/elasticsearch_no-security/molecule.yml index 9855d8e2..8b08e708 100644 --- a/molecule/elasticsearch_no-security/molecule.yml +++ b/molecule/elasticsearch_no-security/molecule.yml @@ -9,7 +9,7 @@ platforms: - name: elasticsearch-nosecurity1 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -19,7 +19,7 @@ platforms: - name: elasticsearch-nosecurity2 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml index 6e7d35a3..6d9a59c7 100644 --- a/molecule/elasticsearch_roles_calculation/molecule.yml +++ b/molecule/elasticsearch_roles_calculation/molecule.yml @@ -9,7 +9,7 @@ platforms: - name: elasticsearch-cluster1 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -19,7 +19,7 @@ platforms: - name: elasticsearch-cluster2 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -29,7 +29,7 @@ platforms: - name: elasticsearch-cluster3 groups: - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index 513db812..122ee248 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -11,7 +11,7 @@ platforms: - beats - logstash - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw @@ -23,7 +23,7 @@ platforms: - beats - kibana - elasticsearch - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/kibana_default/molecule.yml b/molecule/kibana_default/molecule.yml index d279313b..0ae83520 100644 --- a/molecule/kibana_default/molecule.yml +++ b/molecule/kibana_default/molecule.yml @@ -5,7 +5,7 @@ driver: name: docker platforms: - name: kibana_default - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/logstash_full_stack-oss/molecule.yml b/molecule/logstash_full_stack-oss/molecule.yml index ebd40da8..199fff10 100644 --- a/molecule/logstash_full_stack-oss/molecule.yml +++ b/molecule/logstash_full_stack-oss/molecule.yml @@ -11,7 +11,7 @@ platforms: - elasticsearch - logstash - filebeat - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/logstash_pipelines/molecule.yml b/molecule/logstash_pipelines/molecule.yml index e2627f02..51d872e7 100644 --- a/molecule/logstash_pipelines/molecule.yml +++ b/molecule/logstash_pipelines/molecule.yml @@ -7,7 +7,7 @@ driver: name: docker platforms: - name: ansible-role-logstash_pipelines - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/logstash_specific_version/molecule.yml b/molecule/logstash_specific_version/molecule.yml index 5b4cf3c0..de33f462 100644 --- a/molecule/logstash_specific_version/molecule.yml +++ b/molecule/logstash_specific_version/molecule.yml @@ -7,7 +7,7 @@ driver: name: docker platforms: - name: elasticstack_version - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/repos_default/molecule.yml b/molecule/repos_default/molecule.yml index 3c857a61..a111002c 100644 --- a/molecule/repos_default/molecule.yml +++ b/molecule/repos_default/molecule.yml @@ -7,7 +7,7 @@ driver: name: docker platforms: - name: elastic-repos-default - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/molecule/repos_oss/molecule.yml b/molecule/repos_oss/molecule.yml index 9527212b..e1e7a330 100644 --- a/molecule/repos_oss/molecule.yml +++ b/molecule/repos_oss/molecule.yml @@ -7,7 +7,7 @@ driver: name: docker platforms: - name: elastic-repos-default-oss - image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw From d429b152431d521e15cd6f7f54b6df924b5c2f9b Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 17 Oct 2023 10:49:21 +0200 Subject: [PATCH 13/18] Refresh apt cache before installing packages (#179) Run an apt refresh at the start of each role. We can use grace time so we don't neet to refresh it every time and safe some time. fixes #167 --- roles/beats/tasks/main.yml | 7 +++++++ roles/elasticsearch/tasks/main.yml | 8 ++++++++ roles/kibana/tasks/main.yml | 7 +++++++ roles/logstash/tasks/main.yml | 7 +++++++ 4 files changed, 29 insertions(+) diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 9d521bb1..cce08b9e 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -6,6 +6,13 @@ - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - '{{ ansible_os_family }}.yml' +- name: Update apt cache. + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false + when: ansible_os_family == 'Debian' + - name: Prepare for whole stack roles if used when: - elasticstack_full_stack | bool diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 0632f870..fdc11ea6 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,4 +1,12 @@ --- + +- name: Update apt cache. + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false + when: ansible_os_family == 'Debian' + - name: Check-set-parameters ansible.builtin.include_tasks: elasticsearch-parameters.yml diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 74e91344..87638d4d 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -1,5 +1,12 @@ --- +- name: Update apt cache. + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false + when: ansible_os_family == 'Debian' + - name: Include OS specific vars ansible.builtin.include_vars: '{{ item }}' with_first_found: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index a1d1b3de..488926b6 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -1,5 +1,12 @@ --- +- name: Update apt cache. + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false + when: ansible_os_family == 'Debian' + - name: Include OS specific vars ansible.builtin.include_vars: '{{ item }}' with_first_found: From 60115d101e3522ab13dcbb4c5704f1e1aa19952e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 17 Oct 2023 11:51:34 +0200 Subject: [PATCH 14/18] Remove all max-parallel from GitHub workflows (#284) This will need #279 to be merged before tests can pass. fixes #283 --- .github/workflows/test_full_stack.yml | 1 - .github/workflows/test_plugins.yml | 4 ---- .github/workflows/test_role_beats.yml | 1 - .github/workflows/test_role_elasticsearch.yml | 1 - .github/workflows/test_role_kibana.yml | 1 - .github/workflows/test_role_logstash.yml | 1 - .github/workflows/test_role_repos.yml | 1 - .github/workflows/test_roles_pr.yml | 1 - 8 files changed, 11 deletions(-) diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 3dba4e20..4568fdd9 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -31,7 +31,6 @@ jobs: strategy: fail-fast: false - max-parallel: 2 matrix: distro: - rockylinux8 diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index 3bd80ab0..8632a0d7 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -66,7 +66,6 @@ jobs: strategy: fail-fast: false - max-parallel: 1 steps: - name: Check out code @@ -111,7 +110,6 @@ jobs: strategy: fail-fast: false - max-parallel: 1 matrix: python_version: [ 3.5.10, 3.6.15, 3.7.13, 3.8.16, 3.10.10 ] @@ -151,7 +149,6 @@ jobs: strategy: fail-fast: false - max-parallel: 1 matrix: ansible_core_version: [ 2.11.12, 2.12.10, 2.13.8, 2.14.4 ] @@ -191,7 +188,6 @@ jobs: strategy: fail-fast: false - max-parallel: 1 matrix: python_cryptography_version: [ 2.5, 3.0, 3.1, 3.2, 3.3, 3.4, 35.0.0, 36.0.0, 38.0.0, 40.0.1] diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml index 45544ba6..3f38fed9 100644 --- a/.github/workflows/test_role_beats.yml +++ b/.github/workflows/test_role_beats.yml @@ -47,7 +47,6 @@ jobs: strategy: fail-fast: false - max-parallel: 4 matrix: distro: [ubuntu2204] scenario: [beats_default, beats_peculiar] diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml index b0da7d9b..47c8008b 100644 --- a/.github/workflows/test_role_elasticsearch.yml +++ b/.github/workflows/test_role_elasticsearch.yml @@ -47,7 +47,6 @@ jobs: strategy: fail-fast: false - max-parallel: 4 matrix: distro: [ubuntu2204] scenario: diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml index 914464eb..f11bc1ff 100644 --- a/.github/workflows/test_role_kibana.yml +++ b/.github/workflows/test_role_kibana.yml @@ -48,7 +48,6 @@ jobs: strategy: fail-fast: false - max-parallel: 4 matrix: distro: [ubuntu2204] scenario: [kibana_default] diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 14198928..cb959a4c 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -48,7 +48,6 @@ jobs: strategy: fail-fast: false - max-parallel: 4 matrix: distro: [ubuntu2204] scenario: diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index 7963ce6b..c9d83220 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -46,7 +46,6 @@ jobs: strategy: fail-fast: false - max-parallel: 4 matrix: distro: [centos7, debian10, debian11, rockylinux8, rockylinux9, ubuntu2004, ubuntu2204] diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 525f1691..2f18d0a2 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -48,7 +48,6 @@ jobs: strategy: fail-fast: false - max-parallel: 2 matrix: distro: - rockylinux8 From 4077a85b60bce2c22388ab5d2386141ad1082f7b Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 29 Nov 2023 18:01:46 +0100 Subject: [PATCH 15/18] Typo in path of slow log (#293) Just had the wrong path to the logfiles. --- roles/beats/templates/filebeat.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/beats/templates/filebeat.yml.j2 b/roles/beats/templates/filebeat.yml.j2 index 5e788d45..43456cde 100644 --- a/roles/beats/templates/filebeat.yml.j2 +++ b/roles/beats/templates/filebeat.yml.j2 @@ -43,7 +43,7 @@ filebeat.inputs: - type: log enabled: true paths: - - /var/lib/mysql/*-slow.log + - /var/log/mysql/*-slow.log multiline.pattern: '^\#[[:space:]]Time' multiline.negate: true multiline.match: after From e0996b6aad637c6cb04239057ddaf4495f9728f9 Mon Sep 17 00:00:00 2001 From: Markus Opolka Date: Wed, 13 Dec 2023 08:07:58 +0100 Subject: [PATCH 16/18] Add Issue Templates (#294) --- .github/ISSUE_TEMPLATE/bug_report.yaml | 27 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/documentation.yaml | 10 ++++++++ .github/ISSUE_TEMPLATE/feature_request.yaml | 15 ++++++++++++ .github/ISSUE_TEMPLATE/question.yaml | 10 ++++++++ 4 files changed, 62 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml create mode 100644 .github/ISSUE_TEMPLATE/documentation.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml create mode 100644 .github/ISSUE_TEMPLATE/question.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 00000000..6adb01dd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,27 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: ["bug", "needs-triage"] +body: + - type: checkboxes + id: terms + attributes: + label: Please try to fill out as much of the information below as you can. Thank you! + options: + - label: Yes, I've searched similar issues on GitHub and didn't find any. + required: true + - type: input + id: app_version + attributes: + label: Which version contains the bug? + placeholder: 1.0.0 + - type: textarea + id: description + attributes: + label: Describe the bug + description: Please provide a concise description of the bug, add any relevant output or error messages. You can use markdown. + - type: textarea + id: recreate + attributes: + label: How to recreate the bug? + description: Please provide the steps to recreate the issue. diff --git a/.github/ISSUE_TEMPLATE/documentation.yaml b/.github/ISSUE_TEMPLATE/documentation.yaml new file mode 100644 index 00000000..088b14f6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yaml @@ -0,0 +1,10 @@ +name: Documentation +description: Suggest documentation improvements +title: "[Documentation]: " +labels: ["documentation"] +body: + - type: textarea + id: description + attributes: + label: Describe the improvements you'd like. + description: Please provide as much context as possible. You can use markdown. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 00000000..12c9e2e5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,15 @@ +name: Feature Request +description: Request a feature or enhancement +title: "[Feature]: " +labels: ["feature", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Please try to fill out as much of the information below as you can. Thank you! + **Note:** If you want to sponsor new features, contact us at info@netways.de + - type: textarea + id: description + attributes: + label: Describe the feature request + description: Please provide a concise description of the feature. You can use markdown. diff --git a/.github/ISSUE_TEMPLATE/question.yaml b/.github/ISSUE_TEMPLATE/question.yaml new file mode 100644 index 00000000..65183ea8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yaml @@ -0,0 +1,10 @@ +name: Question +description: Ask a question +title: "[Question]: " +labels: ["question"] +body: + - type: textarea + id: description + attributes: + label: Ask a question + description: Please provide as much context as possible. You can use markdown. From 64d33e7c596d94994b5151bba3d0fe9817ab8f09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 12:52:58 +0100 Subject: [PATCH 17/18] Bump actions/setup-python from 4 to 5 (#295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
Release notes

Sourced from actions/setup-python's releases.

v5.0.0

What's Changed

In scope of this release, we update node version runtime from node16 to node20 (actions/setup-python#772). Besides, we update dependencies to the latest versions.

Full Changelog: https://github.com/actions/setup-python/compare/v4.8.0...v5.0.0

v4.8.0

What's Changed

In scope of this release we added support for GraalPy (actions/setup-python#694). You can use this snippet to set up GraalPy:

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
  with:
    python-version: 'graalpy-22.3'
- run: python my_script.py

Besides, the release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.8.0

v4.7.1

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.7.1

v4.7.0

In scope of this release, the support for reading python version from pyproject.toml was added (actions/setup-python#669).

      - name: Setup Python
        uses: actions/setup-python@v4
</tr></table>

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test_full_stack.yml | 2 +- .github/workflows/test_linting.yml | 2 +- .github/workflows/test_plugins.yml | 10 +++++----- .github/workflows/test_role_beats.yml | 2 +- .github/workflows/test_role_elasticsearch.yml | 2 +- .github/workflows/test_role_kibana.yml | 2 +- .github/workflows/test_role_logstash.yml | 2 +- .github/workflows/test_role_repos.yml | 2 +- .github/workflows/test_roles_pr.yml | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 4568fdd9..c81d24ea 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -51,7 +51,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_linting.yml b/.github/workflows/test_linting.yml index fa0da927..6e334a7b 100644 --- a/.github/workflows/test_linting.yml +++ b/.github/workflows/test_linting.yml @@ -44,7 +44,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3. - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index 8632a0d7..900efc92 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3. - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.9.14 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9.14 @@ -118,7 +118,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} @@ -157,7 +157,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.9.14 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9.14 @@ -196,7 +196,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.9.14 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9.14 diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml index 3f38fed9..d4e62d65 100644 --- a/.github/workflows/test_role_beats.yml +++ b/.github/workflows/test_role_beats.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml index 47c8008b..7ae87a96 100644 --- a/.github/workflows/test_role_elasticsearch.yml +++ b/.github/workflows/test_role_elasticsearch.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml index f11bc1ff..3e3e10f5 100644 --- a/.github/workflows/test_role_kibana.yml +++ b/.github/workflows/test_role_kibana.yml @@ -58,7 +58,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index cb959a4c..5d80d30a 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index c9d83220..de2d950c 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 2f18d0a2..fc3dab70 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 From ba6e5aaec58bdd1f762ab94b10066df99c1ad862 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 14:12:08 +0000 Subject: [PATCH 18/18] Always run required checks (#301) We have quite sophisticated rules when to run which Molecule checks. Unfortunately it's quite easy to have a PR or push where the full stack checks won't run. But since we require these checks, you can end up in a situation where you don't have check results but can't merge without those results. So I simplified the rules when to run these checks. fixes #300 --- .github/workflows/test_roles_pr.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index fc3dab70..026d9126 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -13,23 +13,7 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' - paths: - - 'roles/**' - - '.github/workflows/test_roles_pr.yml' - - 'molecule/elasticstack_default/**' push: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' - paths: - - 'roles/**' - - '.github/workflows/test_roles_pr.yml' - - 'molecule/elasticstack_default/**' merge_group: jobs: