From 0015f32cc6691bb3830fcefbeb526100f2cbb841 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kalitowski Date: Mon, 15 Jul 2024 13:28:32 +0200 Subject: [PATCH] misc: filetree_read performence improvement, fix memory leak issue --- .../filetree_read_speed_improvement.yml | 4 ++++ roles/filetree_read/tasks/credentials.yml | 15 +++++++------ roles/filetree_read/tasks/hosts.yml | 15 +++++++------ roles/filetree_read/tasks/inventories.yml | 19 +++++++---------- roles/filetree_read/tasks/job_templates.yml | 19 +++++++---------- roles/filetree_read/tasks/projects.yml | 19 +++++++---------- roles/filetree_read/tasks/roles.yml | 19 +++++++---------- roles/filetree_read/tasks/teams.yml | 21 ++++++++----------- .../tasks/workflow_job_templates.yml | 21 ++++++++----------- 9 files changed, 72 insertions(+), 80 deletions(-) create mode 100644 changelogs/fragments/filetree_read_speed_improvement.yml diff --git a/changelogs/fragments/filetree_read_speed_improvement.yml b/changelogs/fragments/filetree_read_speed_improvement.yml new file mode 100644 index 000000000..1740c8d9b --- /dev/null +++ b/changelogs/fragments/filetree_read_speed_improvement.yml @@ -0,0 +1,4 @@ +--- +minor_changes: + - filetree_read speed tuning + - fix memory leak when there are plenty of job templates diff --git a/roles/filetree_read/tasks/credentials.yml b/roles/filetree_read/tasks/credentials.yml index 10902d58b..23f00d6c5 100644 --- a/roles/filetree_read/tasks/credentials.yml +++ b/roles/filetree_read/tasks/credentials.yml @@ -15,14 +15,17 @@ register: __contents_filetree_controller_credentials failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_credentials" -- name: "Populate Credentials list" +- name: "Populate Credentials Data Structure" ansible.builtin.set_fact: - __populate_controller_credentials: "{{ (__populate_controller_credentials | default([])) + __populate_credentials_list_item.ansible_facts.controller_credentials }}" - loop: "{{ __contents_filetree_controller_credentials.results }}" - loop_control: - loop_var: __populate_credentials_list_item + __populate_controller_credentials: >- + {{ + __contents_filetree_controller_credentials.results | + rejectattr('ansible_facts.controller_credentials', 'undefined') | + map(attribute='ansible_facts.controller_credentials') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_credentials.results is defined and __populate_credentials_list_item.ansible_facts.controller_credentials is defined + when: __contents_filetree_controller_credentials.results - name: "Segregated Credentials list when controller_location is defined" when: controller_location is defined and __populate_controller_credentials is defined diff --git a/roles/filetree_read/tasks/hosts.yml b/roles/filetree_read/tasks/hosts.yml index cb145b34c..4d86eb9d3 100644 --- a/roles/filetree_read/tasks/hosts.yml +++ b/roles/filetree_read/tasks/hosts.yml @@ -15,14 +15,17 @@ register: __contents_filetree_controller_hosts failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_hosts" -- name: "Populate Hosts list" +- name: "Set Hosts Data Structure" ansible.builtin.set_fact: - __populate_controller_hosts: "{{ (__populate_controller_hosts | default([])) + __populate_hosts_list_item.ansible_facts.controller_hosts }}" - loop: "{{ __contents_filetree_controller_hosts.results }}" - loop_control: - loop_var: __populate_hosts_list_item + __populate_controller_hosts: >- + {{ + __contents_filetree_controller_hosts.results | + rejectattr('ansible_facts.controller_hosts', 'undefined') | + map(attribute='ansible_facts.controller_hosts') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_hosts.results is defined and __populate_hosts_list_item.ansible_facts.controller_hosts is defined + when: __contents_filetree_controller_hosts.results - name: "Segregated Hosts list when controller_location is defined" when: controller_location is defined and __populate_controller_hosts is defined diff --git a/roles/filetree_read/tasks/inventories.yml b/roles/filetree_read/tasks/inventories.yml index aa35a64d9..e0e566e30 100644 --- a/roles/filetree_read/tasks/inventories.yml +++ b/roles/filetree_read/tasks/inventories.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_inventories failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_inventories" -- name: "Populate Inventories list" - ansible.builtin.set_fact: - __populate_controller_inventories: "{{ (__populate_controller_inventories | default([])) + __populate_inventories_list_item.ansible_facts.controller_inventories }}" - loop: "{{ __contents_filetree_controller_inventories.results }}" - loop_control: - loop_var: __populate_inventories_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_inventories.results is defined and __populate_inventories_list_item.ansible_facts.controller_inventories is defined - - name: "Set Inventories Data Structure" ansible.builtin.set_fact: - controller_inventories: "{{ __populate_controller_inventories }}" + controller_inventories: >- + {{ + __contents_filetree_controller_inventories.results | + rejectattr('ansible_facts.controller_inventories', 'undefined') | + map(attribute='ansible_facts.controller_inventories') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_inventories is defined + when: __contents_filetree_controller_inventories.results ... diff --git a/roles/filetree_read/tasks/job_templates.yml b/roles/filetree_read/tasks/job_templates.yml index 386a9a5dc..b8ed7c64c 100644 --- a/roles/filetree_read/tasks/job_templates.yml +++ b/roles/filetree_read/tasks/job_templates.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_templates failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_templates" -- name: "Populate Job Templates list" - ansible.builtin.set_fact: - __populate_controller_job_templates: "{{ (__populate_controller_job_templates | default([])) + __populate_job_templates_list_item.ansible_facts.controller_templates }}" - loop: "{{ __contents_filetree_controller_templates.results }}" - loop_control: - loop_var: __populate_job_templates_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_templates.results is defined and __populate_job_templates_list_item.ansible_facts.controller_templates is defined - - name: "Set Job Templates Data Structure" ansible.builtin.set_fact: - controller_templates: "{{ __populate_controller_job_templates }}" + controller_templates: >- + {{ + __contents_filetree_controller_templates.results | + rejectattr('ansible_facts.controller_templates', 'undefined') | + map(attribute='ansible_facts.controller_templates') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_job_templates is defined + when: __contents_filetree_controller_templates.results ... diff --git a/roles/filetree_read/tasks/projects.yml b/roles/filetree_read/tasks/projects.yml index dd50fe91f..5333a94ca 100644 --- a/roles/filetree_read/tasks/projects.yml +++ b/roles/filetree_read/tasks/projects.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_projects failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_projects" -- name: "Populate Projects list" - ansible.builtin.set_fact: - __populate_controller_projects: "{{ (__populate_controller_projects | default([])) + __populate_projects_list_item.ansible_facts.controller_projects }}" - loop: "{{ __contents_filetree_controller_projects.results }}" - loop_control: - loop_var: __populate_projects_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_projects.results is defined and __populate_projects_list_item.ansible_facts.controller_projects is defined - - name: "Set Projects Data Structure" ansible.builtin.set_fact: - controller_projects: "{{ __populate_controller_projects }}" + controller_projects: >- + {{ + __contents_filetree_controller_projects.results | + rejectattr('ansible_facts.controller_projects', 'undefined') | + map(attribute='ansible_facts.controller_projects') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_projects is defined + when: __contents_filetree_controller_projects.results ... diff --git a/roles/filetree_read/tasks/roles.yml b/roles/filetree_read/tasks/roles.yml index ce7002fbe..b9b7cb503 100644 --- a/roles/filetree_read/tasks/roles.yml +++ b/roles/filetree_read/tasks/roles.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_roles failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_roles" -- name: "Populate Roles list" - ansible.builtin.set_fact: - __populate_controller_roles: "{{ (__populate_controller_roles | default([])) + __populate_roles_list_item.ansible_facts.controller_roles }}" - loop: "{{ __contents_filetree_controller_roles.results }}" - loop_control: - loop_var: __populate_roles_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_roles.results is defined and __populate_roles_list_item.ansible_facts.controller_roles is defined - - name: "Set Roles Data Structure" ansible.builtin.set_fact: - controller_roles: "{{ __populate_controller_roles }}" + controller_roles: >- + {{ + __contents_filetree_controller_roles.results | + rejectattr('ansible_facts.controller_roles', 'undefined') | + map(attribute='ansible_facts.controller_roles') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_roles is defined + when: __contents_filetree_controller_roles.results ... diff --git a/roles/filetree_read/tasks/teams.yml b/roles/filetree_read/tasks/teams.yml index 3c6a33f9d..b8f89c1e4 100644 --- a/roles/filetree_read/tasks/teams.yml +++ b/roles/filetree_read/tasks/teams.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_teams failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_teams" -- name: "Populate Teams list" +- name: "Set Roles Data Structure" ansible.builtin.set_fact: - __populate_controller_teams: "{{ (__populate_controller_teams | default([])) + __populate_teams_list_item.ansible_facts.controller_teams }}" - loop: "{{ __contents_filetree_controller_teams.results }}" - loop_control: - loop_var: __populate_teams_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_teams.results is defined and __populate_teams_list_item.ansible_facts.controller_teams is defined - -- name: "Set Teams Data Structure" - ansible.builtin.set_fact: - controller_teams: "{{ __populate_controller_teams }}" + controller_roles: >- + {{ + __contents_filetree_controller_teams.results | + rejectattr('ansible_facts.controller_teams', 'undefined') | + map(attribute='ansible_facts.controller_teams') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_teams is defined + when: __contents_filetree_controller_teams.results ... diff --git a/roles/filetree_read/tasks/workflow_job_templates.yml b/roles/filetree_read/tasks/workflow_job_templates.yml index 00f586f91..f364aa652 100644 --- a/roles/filetree_read/tasks/workflow_job_templates.yml +++ b/roles/filetree_read/tasks/workflow_job_templates.yml @@ -15,18 +15,15 @@ register: __contents_filetree_controller_workflow_job_templates failed_when: "'VARIABLE IS NOT DEFINED' in __contents_filetree_controller_workflow_job_templates" -- name: "Populate Workflow Job Templates list" +- name: "Set Workflows Job Templates Data Structure" ansible.builtin.set_fact: - __populate_controller_workflow_job_templates: "{{ (__populate_controller_workflow_job_templates | default([])) + __populate_credentials_list_item.ansible_facts.controller_workflows }}" - loop: "{{ __contents_filetree_controller_workflow_job_templates.results }}" - loop_control: - loop_var: __populate_credentials_list_item - no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __contents_filetree_controller_workflow_job_templates.results is defined and __populate_credentials_list_item.ansible_facts.controller_workflows is defined - -- name: "Set Job Templates Data Structure" - ansible.builtin.set_fact: - controller_workflows: "{{ __populate_controller_workflow_job_templates }}" + controller_workflows: >- + {{ + __contents_filetree_controller_workflow_job_templates.results | + rejectattr('ansible_facts.controller_workflows', 'undefined') | + map(attribute='ansible_facts.controller_workflows') | + ansible.builtin.flatten + }} no_log: "{{ controller_configuration_filetree_read_secure_logging }}" - when: __populate_controller_workflow_job_templates is defined + when: __contents_filetree_controller_workflow_job_templates.results ...