diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f38dd1c --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +.vagrant diff --git a/README.md b/README.md index 859f6f6..b151f5d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# cephadm-ansible +# cephadm-ansible (ceph.cephadm) cephadm-ansible is a collection of Ansible playbooks to simplify workflows that are not covered by [cephadm]. The workflows covered diff --git a/ansible.cfg b/ansible.cfg index 257a642..63e89a0 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,7 +1,7 @@ [defaults] log_path = $HOME/ansible/ansible.log -library = ./library -module_utils = ./module_utils +library = ./plugins/modules +module_utils = ./plugins/module_utils roles_path = ./ forks = 20 @@ -23,4 +23,3 @@ control_path = %(directory)s/%%h-%%r-%%p ssh_args = -o ControlMaster=auto -o ControlPersist=600s pipelining = True retries = 10 - diff --git a/galaxy.yml b/galaxy.yml new file mode 100644 index 0000000..5f8c1e9 --- /dev/null +++ b/galaxy.yml @@ -0,0 +1,70 @@ +### REQUIRED +# The namespace of the collection. This can be a company/brand/organization or product namespace under which all +# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with +# underscores or numbers and cannot contain consecutive underscores +namespace: ceph + +# The name of the collection. Has the same character restrictions as 'namespace' +name: cephadm + +# The version of the collection. Must be compatible with semantic versioning +version: 3.0.0 + +# The path to the Markdown (.md) readme file. This path is relative to the root of the collection +readme: README.md + +# A list of the collection's content authors. Can be just the name or in the format 'Full Name (url) +# @nicks:irc/im.site#channel' +authors: + - Guillaume Abrioux + - Ken Dreyer + +### OPTIONAL but strongly recommended +# A short summary description of the collection +description: ansible playbooks to be used with cephadm + +# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only +# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file' +license: + - Apache-2.0 + +# The path to the license file for the collection. This path is relative to the root of the collection. This key is +# mutually exclusive with 'license' +# license_file: '' + +# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character +# requirements as 'namespace' and 'name' +tags: ['system'] + +# Collections that this collection requires to be installed for it to be usable. The key of the dict is the +# collection label 'namespace.name'. The value is a version range +# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version +# range specifiers can be set and are separated by ',' +dependencies: + ansible.posix: ">=1.5.4" + community.general: ">=7.5.0" + +# The URL of the originating SCM repository +repository: https://github.com/ceph/cephadm-ansible + +# The URL to any online docs +documentation: https://docs.ceph.com/projects/cephadm-ansible/ + +# The URL to the homepage of the collection/project +homepage: https://github.com/ceph/cephadm-ansible + +# The URL to the collection issue tracker +issues: https://github.com/ceph/cephadm-ansible/issues + +# A list of file glob-like patterns used to filter any files or directories that should not be included in the build +# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This +# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry', +# and '.git' are always filtered. Mutually exclusive with 'manifest' +build_ignore: [] +# A dict controlling use of manifest directives used in building the collection artifact. The key 'directives' is a +# list of MANIFEST.in style +# L(directives,https://packaging.python.org/en/latest/guides/using-manifest-in/#manifest-in-commands). The key +# 'omit_default_directives' is a boolean that controls whether the default directives are used. Mutually exclusive +# with 'build_ignore' +# manifest: null + diff --git a/meta/runtime.yml b/meta/runtime.yml new file mode 100644 index 0000000..cdc7af0 --- /dev/null +++ b/meta/runtime.yml @@ -0,0 +1,52 @@ +--- +# Collections must specify a minimum required ansible version to upload +# to galaxy +requires_ansible: '>=2.9,<3,!=2.9.10' + +# Content that Ansible needs to load from another location or that has +# been deprecated/removed +# plugin_routing: +# action: +# redirected_plugin_name: +# redirect: ns.col.new_location +# deprecated_plugin_name: +# deprecation: +# removal_version: "4.0.0" +# warning_text: | +# See the porting guide on how to update your playbook to +# use ns.col.another_plugin instead. +# removed_plugin_name: +# tombstone: +# removal_version: "2.0.0" +# warning_text: | +# See the porting guide on how to update your playbook to +# use ns.col.another_plugin instead. +# become: +# cache: +# callback: +# cliconf: +# connection: +# doc_fragments: +# filter: +# httpapi: +# inventory: +# lookup: +# module_utils: +# modules: +# netconf: +# shell: +# strategy: +# terminal: +# test: +# vars: + +# Python import statements that Ansible needs to load from another location +# import_redirection: +# ansible_collections.ns.col.plugins.module_utils.old_location: +# redirect: ansible_collections.ns.col.plugins.module_utils.new_location + +# Groups of actions/modules that take a common set of options +# action_groups: +# group_name: +# - module1 +# - module2 diff --git a/cephadm-clients.yml b/playbooks/cephadm-clients.yml similarity index 100% rename from cephadm-clients.yml rename to playbooks/cephadm-clients.yml diff --git a/cephadm-distribute-ssh-key.yml b/playbooks/cephadm-distribute-ssh-key.yml similarity index 100% rename from cephadm-distribute-ssh-key.yml rename to playbooks/cephadm-distribute-ssh-key.yml diff --git a/cephadm-preflight.yml b/playbooks/cephadm-preflight.yml similarity index 100% rename from cephadm-preflight.yml rename to playbooks/cephadm-preflight.yml diff --git a/cephadm-purge-cluster.yml b/playbooks/cephadm-purge-cluster.yml similarity index 100% rename from cephadm-purge-cluster.yml rename to playbooks/cephadm-purge-cluster.yml diff --git a/cephadm-set-container-insecure-registries.yml b/playbooks/cephadm-set-container-insecure-registries.yml similarity index 97% rename from cephadm-set-container-insecure-registries.yml rename to playbooks/cephadm-set-container-insecure-registries.yml index 1d8b22b..96cbe39 100644 --- a/cephadm-set-container-insecure-registries.yml +++ b/playbooks/cephadm-set-container-insecure-registries.yml @@ -33,4 +33,4 @@ block: | [[registry]] location = '{{ insecure_registry }}' - insecure = true \ No newline at end of file + insecure = true diff --git a/rocksdb-resharding.yml b/playbooks/rocksdb-resharding.yml similarity index 100% rename from rocksdb-resharding.yml rename to playbooks/rocksdb-resharding.yml diff --git a/validate/insecure-registries.yml b/playbooks/validate/insecure-registries.yml similarity index 100% rename from validate/insecure-registries.yml rename to playbooks/validate/insecure-registries.yml diff --git a/validate/preflight.yml b/playbooks/validate/preflight.yml similarity index 100% rename from validate/preflight.yml rename to playbooks/validate/preflight.yml diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..6260634 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,31 @@ +# Collections Plugins Directory + +This directory can be used to ship various plugins inside an Ansible collection. Each plugin is placed in a folder that +is named after the type of plugin it is in. It can also include the `module_utils` and `modules` directory that +would contain module utils and modules respectively. + +Here is an example directory of the majority of plugins currently supported by Ansible: + +``` +└── plugins + ├── action + ├── become + ├── cache + ├── callback + ├── cliconf + ├── connection + ├── filter + ├── httpapi + ├── inventory + ├── lookup + ├── module_utils + ├── modules + ├── netconf + ├── shell + ├── strategy + ├── terminal + ├── test + └── vars +``` + +A full list of plugin types can be found at [Working With Plugins](https://docs.ansible.com/ansible-core/2.15/plugins/plugins.html). diff --git a/library/__init__.py b/plugins/module_utils/__init__.py similarity index 100% rename from library/__init__.py rename to plugins/module_utils/__init__.py diff --git a/module_utils/ceph_common.py b/plugins/module_utils/ceph_common.py similarity index 93% rename from module_utils/ceph_common.py rename to plugins/module_utils/ceph_common.py index 99c5da8..8fb0a85 100644 --- a/module_utils/ceph_common.py +++ b/plugins/module_utils/ceph_common.py @@ -1,9 +1,8 @@ import datetime import time -from typing import TYPE_CHECKING, List, Dict +from typing import List, Dict -if TYPE_CHECKING: - from ansible.module_utils.basic import AnsibleModule # type: ignore +from ansible.module_utils.basic import AnsibleModule # type: ignore def retry(exceptions, retries=20, delay=1): diff --git a/module_utils/__init__.py b/plugins/modules/__init__.py similarity index 100% rename from module_utils/__init__.py rename to plugins/modules/__init__.py diff --git a/library/ceph_config.py b/plugins/modules/ceph_config.py similarity index 95% rename from library/ceph_config.py rename to plugins/modules/ceph_config.py index 57df331..863a410 100644 --- a/library/ceph_config.py +++ b/plugins/modules/ceph_config.py @@ -7,10 +7,7 @@ __metaclass__ = type from ansible.module_utils.basic import AnsibleModule # type: ignore -try: - from ansible.module_utils.ceph_common import exit_module, build_base_cmd_shell, fatal # type: ignore -except ImportError: - from module_utils.ceph_common import exit_module, build_base_cmd_shell, fatal # type: ignore +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import exit_module, build_base_cmd_shell, fatal # type: ignore import datetime import json diff --git a/library/ceph_orch_apply.py b/plugins/modules/ceph_orch_apply.py similarity index 93% rename from library/ceph_orch_apply.py rename to plugins/modules/ceph_orch_apply.py index a397328..11d3abf 100644 --- a/library/ceph_orch_apply.py +++ b/plugins/modules/ceph_orch_apply.py @@ -19,10 +19,7 @@ __metaclass__ = type from ansible.module_utils.basic import AnsibleModule # type: ignore -try: - from ansible.module_utils.ceph_common import exit_module, build_base_cmd_orch # type: ignore -except ImportError: - from module_utils.ceph_common import exit_module, build_base_cmd_orch +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import exit_module, build_base_cmd_orch # type: ignore import datetime diff --git a/library/ceph_orch_daemon.py b/plugins/modules/ceph_orch_daemon.py similarity index 95% rename from library/ceph_orch_daemon.py rename to plugins/modules/ceph_orch_daemon.py index 303d193..86f0fd5 100644 --- a/library/ceph_orch_daemon.py +++ b/plugins/modules/ceph_orch_daemon.py @@ -7,10 +7,7 @@ __metaclass__ = type from ansible.module_utils.basic import AnsibleModule # type: ignore -try: - from ansible.module_utils.ceph_common import retry, exit_module, build_base_cmd_orch, fatal # type: ignore -except ImportError: - from module_utils.ceph_common import retry, exit_module, build_base_cmd_orch, fatal # type: ignore +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import retry, exit_module, build_base_cmd_orch, fatal # type: ignore import datetime import json diff --git a/library/ceph_orch_host.py b/plugins/modules/ceph_orch_host.py similarity index 97% rename from library/ceph_orch_host.py rename to plugins/modules/ceph_orch_host.py index d7b58cb..334927c 100644 --- a/library/ceph_orch_host.py +++ b/plugins/modules/ceph_orch_host.py @@ -19,10 +19,8 @@ __metaclass__ = type from ansible.module_utils.basic import AnsibleModule # type: ignore -try: - from ansible.module_utils.ceph_common import exit_module, build_base_cmd_orch # type: ignore -except ImportError: - from module_utils.ceph_common import exit_module, build_base_cmd_orch +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import exit_module, build_base_cmd_orch # type: ignore + import datetime import json diff --git a/library/cephadm_bootstrap.py b/plugins/modules/cephadm_bootstrap.py similarity index 98% rename from library/cephadm_bootstrap.py rename to plugins/modules/cephadm_bootstrap.py index 7cd50ff..e36a4e2 100644 --- a/library/cephadm_bootstrap.py +++ b/plugins/modules/cephadm_bootstrap.py @@ -17,10 +17,8 @@ __metaclass__ = type from ansible.module_utils.basic import AnsibleModule # type: ignore -try: - from ansible.module_utils.ceph_common import exit_module # type: ignore -except ImportError: - from module_utils.ceph_common import exit_module +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import exit_module # type: ignore + import datetime import os diff --git a/library/cephadm_registry_login.py b/plugins/modules/cephadm_registry_login.py similarity index 96% rename from library/cephadm_registry_login.py rename to plugins/modules/cephadm_registry_login.py index 2f8be0f..ecc94ed 100644 --- a/library/cephadm_registry_login.py +++ b/plugins/modules/cephadm_registry_login.py @@ -19,10 +19,8 @@ from ansible.module_utils.basic import AnsibleModule # type: ignore from typing import List, Tuple -try: - from ansible.module_utils.ceph_common import exit_module, build_base_cmd, fatal # type: ignore -except ImportError: - from module_utils.ceph_common import exit_module, build_base_cmd, fatal +from ansible_collections.ceph.cephadm.plugins.module_utils.ceph_common import exit_module, build_base_cmd, fatal # type: ignore + import datetime ANSIBLE_METADATA = { diff --git a/requirements.txt b/requirements.txt index 41bb242..a3f3ca6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ testinfra>=3,<4 pytest-xdist==1.28.0 pytest>=4.6,<5.0 -ansible>=2.9,<2.10,!=2.9.10 +ansible>=2.9,<3,!=2.9.10 Jinja2>=2.10 diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..a6b0fd8 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - ansible.posix + - community.general diff --git a/ceph_defaults/README.md b/roles/ceph_defaults/README.md similarity index 100% rename from ceph_defaults/README.md rename to roles/ceph_defaults/README.md diff --git a/ceph_defaults/defaults/main.yml b/roles/ceph_defaults/defaults/main.yml similarity index 100% rename from ceph_defaults/defaults/main.yml rename to roles/ceph_defaults/defaults/main.yml diff --git a/ceph_defaults/meta/main.yml b/roles/ceph_defaults/meta/main.yml similarity index 100% rename from ceph_defaults/meta/main.yml rename to roles/ceph_defaults/meta/main.yml diff --git a/tests/Vagrantfile b/tests/Vagrantfile new file mode 100644 index 0000000..aff8eda --- /dev/null +++ b/tests/Vagrantfile @@ -0,0 +1,83 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# took from ceph-ansible project + +NNODE = 6 +NCLIENTS = 2 +LABEL_PREFIX = 'ceph-' +MEMORY = 1024 +PUBLIC_SUBNET = '192.168.9' +CLUSTER_SUBNET = '192.168.10' +BOX = ENV['CEPH_ANSIBLE_VAGRANT_BOX'] + +ansible_provision = proc do |ansible| + ansible.playbook = '../playbooks/cephadm-preflight.yml' + ansible.groups = { + 'ceph_cluster' => (0..NNODE - 1).map { |j| "#{LABEL_PREFIX}node#{j}" }, + 'clients' => (0..NCLIENTS - 1).map { |j| "client#{j}" }, + } + ansible.limit = 'all' + ansible.verbose = '-vv' + ansible.extra_vars = { + ceph_origin: "community", + } +end + +Vagrant.configure('2') do |config| + config.vm.box = BOX + config.ssh.insert_key = false # workaround for https://github.com/mitchellh/vagrant/issues/5048 +# config.ssh.private_key_path = settings['ssh_private_key_path'] + config.ssh.username = 'vagrant' + config.vm.provider :libvirt do |lv| + lv.cpu_mode = 'host-passthrough' + lv.disk_driver :cache => 'unsafe' + lv.graphics_type = 'none' + lv.cpus = 4 + end + +$last_ip_pub_digit = 9 +$last_ip_cluster_digit = 9 + +config.vm.provider :libvirt do |v,override| + override.vm.synced_folder '.', '/vagrant', disabled: true +end + + + (0..NCLIENTS - 1).each do |i| + config.vm.define "client#{i}" do |client| + client.vm.hostname = "client#{i}" + client.vm.network :private_network, + ip: "#{PUBLIC_SUBNET}.#{$last_ip_pub_digit+=1}" + + # Libvirt + client.vm.provider :libvirt do |lv| + lv.memory = MEMORY + lv.random_hostname = true + end + end + end + + (0..NNODE - 1).each do |i| + config.vm.define "#{LABEL_PREFIX}node#{i}" do |node| + node.vm.hostname = "#{LABEL_PREFIX}node#{i}" + node.vm.network :private_network, + ip: "#{PUBLIC_SUBNET}.#{$last_ip_pub_digit+=1}" + node.vm.network :private_network, + ip: "#{CLUSTER_SUBNET}.#{$last_ip_cluster_digit+=1}" + + # Libvirt + driverletters = ('a'..'z').to_a + node.vm.provider :libvirt do |lv| + # always make /dev/sd{a/b/c} so that CI can ensure that + # virtualbox and libvirt will have the same devices to use for OSDs + (0..2).each do |d| + lv.storage :file, :device => "hd#{driverletters[d]}", :size => '50G', :bus => "ide" + end + lv.memory = MEMORY + lv.random_hostname = true + end + node.vm.provision 'ansible', &ansible_provision if i == (NNODE - 1) + end + end +end diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100644 index 0000000..bb447e8 --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,22 @@ +[defaults] +log_path = $HOME/ansible/ansible.log +roles_path = ../../ +library = ../../library +module_utils = ../../module_utils +host_key_checking = False +gathering = smart +fact_caching = memory +fact_caching_timeout = 7200 +nocows = 1 +callback_whitelist = profile_tasks +stdout_callback = yaml +force_valid_group_names = ignore +inject_facts_as_vars = False +retry_files_enabled = False +timeout = 60 + +[ssh_connection] +control_path = %(directory)s/%%h-%%r-%%p +ssh_args = -o ControlMaster=auto -o ControlPersist=600s +pipelining = True +retries = 10 diff --git a/tests/cephadm-distribute-ssh-key.yml b/tests/cephadm-distribute-ssh-key.yml new file mode 120000 index 0000000..22750da --- /dev/null +++ b/tests/cephadm-distribute-ssh-key.yml @@ -0,0 +1 @@ +../playbooks/cephadm-distribute-ssh-key.yml \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0bd13b4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest + +@pytest.fixture() +def node(host, request): + ansible_vars = host.ansible.get_variables() + + if request.node.get_closest_marker("no_client") and ansible_vars['group_names'] == ['clients']: + pytest.skip("Not a valid test for client nodes") + + if request.node.get_closest_marker("client") and 'clients' not in ansible_vars['group_names']: + pytest.skip("Not a valid test for non client nodes") + + if request.node.get_closest_marker("osd") and 'osds' not in ansible_vars['group_names']: + pytest.skip("Not a valid test for non osd nodes") + + if request.node.get_closest_marker("admin") and 'admin' not in ansible_vars['group_names']: + pytest.skip("Not a valid test for non admin nodes") + + return ansible_vars diff --git a/tests/deploy-cluster-vars.yml b/tests/deploy-cluster-vars.yml new file mode 100644 index 0000000..c5e3d5d --- /dev/null +++ b/tests/deploy-cluster-vars.yml @@ -0,0 +1,8 @@ +--- +cluster: ceph +delegate_facts_host: true +ceph_container_registry: quay.io +ceph_container_registry_auth: false +ceph_container_no_proxy: "localhost,127.0.0.1" +health_mon_check_retries: 300 +health_mon_check_delay: 1 diff --git a/tests/deploy-cluster.yml b/tests/deploy-cluster.yml new file mode 100644 index 0000000..bb18af4 --- /dev/null +++ b/tests/deploy-cluster.yml @@ -0,0 +1,165 @@ +--- +- name: load variables + hosts: all + become: false + tasks: + - include_vars: deploy-cluster-vars.yml + +- name: gather facts and prepare system for cephadm + hosts: + - mons + - mgrs + - osds + become: true + gather_facts: false + tasks: + - import_role: + name: ceph_defaults + + - name: gather and delegate facts + setup: + gather_subset: + - 'all' + - '!facter' + - '!ohai' + delegate_to: "{{ item }}" + delegate_facts: true + with_items: "{{ groups['all'] }}" + run_once: true + when: delegate_facts_host | bool + + - name: container registry authentication + cephadm_registry_login: + registry_url: "{{ ceph_container_registry }}" + registry_username: "{{ ceph_container_registry_username }}" + registry_password: "{{ ceph_container_registry_password }}" + environment: + HTTP_PROXY: "{{ ceph_container_http_proxy | default('') }}" + HTTPS_PROXY: "{{ ceph_container_https_proxy | default('') }}" + NO_PROXY: "{{ ceph_container_no_proxy }}" + when: ceph_container_registry_auth | default(False) | bool + +- name: bootstrap the cluster + hosts: "{{ groups.get('mons')[0] }}" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph_defaults + + - name: bootstrap initial cluster + cephadm_bootstrap: + mon_ip: "{{ monitor_address }}" + fsid: "{{ fsid if fsid is defined else omit }}" + pull: false + +- import_playbook: cephadm-distribute-ssh-key.yml + vars: + admin_node: "{{ groups.get('admin')[0] }}" + +- name: add the other nodes + hosts: + - mons + - mgrs + - osds + become: true + gather_facts: false + tasks: + - import_role: + name: ceph_defaults + + - name: run cephadm prepare-host + command: cephadm prepare-host + changed_when: false + + - name: add hosts + ceph_orch_host: + name: "{{ ansible_facts['hostname'] }}" + address: "{{ ansible_facts['default_ipv4']['address'] if inventory_hostname != 'ceph-node00' else omit }}" + set_admin_label: "{{ True if inventory_hostname in groups.get('admin', []) else omit }}" + labels: "{{ labels }}" + delegate_to: "{{ groups['mons'][0] }}" + + +- name: adjust service placement + hosts: "{{ groups.get('mons')[0] }}" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph_defaults + + - name: update the placement of monitor hosts + ceph_orch_apply: + spec: | + service_type: mon + service_id: mon + placement: + label: mons + + - name: waiting for the monitor to join the quorum... + command: cephadm shell ceph quorum_status --format json + register: ceph_health_raw + changed_when: false + until: (ceph_health_raw.stdout | from_json)["quorum_names"] | length == groups.get('mons', []) | length + retries: "{{ health_mon_check_retries }}" + delay: "{{ health_mon_check_delay }}" + + - name: update the placement of manager hosts + command: cephadm shell ceph orch apply mgr --placement=label:mgrs + + - name: update the placement of osd hosts + ceph_orch_apply: + spec: | + service_type: osd + service_id: osd + placement: + label: osds + spec: + data_devices: + all: true + + - name: update the placement of crash hosts + ceph_orch_apply: + spec: | + service_type: crash + service_id: crash + placement: + host_pattern: '*' + + - name: enable the monitoring + command: "ceph {{ item }}" + changed_when: false + loop: + - "mgr module enable prometheus" + - "orch apply alertmanager --placement=label:monitoring" + - "orch apply grafana --placement=label:monitoring" + - "orch apply prometheus --placement=label:monitoring" + - "orch apply node-exporter --placement=*" + +- name: remove ceph-node5 + hosts: "{{ groups.get('mons')[0] }}" + become: true + gather_facts: false + tasks: + - name: drain ceph-node5 + ceph_orch_host: + state: drain + name: ceph-node5 + + - name: remove ceph-node5 + ceph_orch_host: + state: absent + name: ceph-node5 + retries: 20 + delay: 1 + until: result is succeeded + register: result +# TODO(guits): address the following tasks: + # - name: show ceph orchestrator services + # command: "{{ cephadm_cmd }} shell -- ceph --cluster {{ cluster }} orch ls --refresh" + # changed_when: false + + # - name: show ceph orchestrator daemons + # command: "{{ cephadm_cmd }} shell -- ceph --cluster {{ cluster }} orch ps --refresh" + # changed_when: false \ No newline at end of file diff --git a/tests/functional/Vagrantfile b/tests/functional/Vagrantfile index 5874903..aff8eda 100644 --- a/tests/functional/Vagrantfile +++ b/tests/functional/Vagrantfile @@ -12,7 +12,7 @@ CLUSTER_SUBNET = '192.168.10' BOX = ENV['CEPH_ANSIBLE_VAGRANT_BOX'] ansible_provision = proc do |ansible| - ansible.playbook = '../cephadm-preflight.yml' + ansible.playbook = '../playbooks/cephadm-preflight.yml' ansible.groups = { 'ceph_cluster' => (0..NNODE - 1).map { |j| "#{LABEL_PREFIX}node#{j}" }, 'clients' => (0..NCLIENTS - 1).map { |j| "client#{j}" }, diff --git a/tests/functional/cephadm-distribute-ssh-key.yml b/tests/functional/cephadm-distribute-ssh-key.yml index 0fdadc8..03b3c9b 120000 --- a/tests/functional/cephadm-distribute-ssh-key.yml +++ b/tests/functional/cephadm-distribute-ssh-key.yml @@ -1 +1 @@ -../../cephadm-distribute-ssh-key.yml \ No newline at end of file +../../playbooks/cephadm-distribute-ssh-key.yml \ No newline at end of file diff --git a/tests/functional/vagrant_ssh_config b/tests/functional/vagrant_ssh_config new file mode 100644 index 0000000..e69de29 diff --git a/tests/hosts b/tests/hosts new file mode 100644 index 0000000..1afaa38 --- /dev/null +++ b/tests/hosts @@ -0,0 +1,28 @@ +[ceph_cluster] +ceph-node0 labels="['_admin', 'mons', 'mgrs', 'monitoring']" +ceph-node1 labels="['mons', 'mgrs']" +ceph-node2 labels="['mons', 'mgrs']" +ceph-node3 labels="['rgws']" +ceph-node4 labels="['osds']" +ceph-node5 labels="['iscsigws']" + +[admin] +ceph-node0 + +[clients] +client0 +client1 + +# deploy-cluster.yml groups related +[mons] +ceph-node0 +ceph-node1 +ceph-node2 + +[mgrs] +ceph-node0 + +[osds] +ceph-node3 +ceph-node4 +ceph-node5 diff --git a/tests/module_utils/test_ceph_common.py b/tests/module_utils/test_ceph_common.py index 0e2d223..670e179 100644 --- a/tests/module_utils/test_ceph_common.py +++ b/tests/module_utils/test_ceph_common.py @@ -1,4 +1,4 @@ -import ceph_common +from ansible_collections.ceph.cephadm.plugins.module_utils import ceph_common import pytest from mock.mock import MagicMock diff --git a/tests/library/common.py b/tests/modules/common.py similarity index 100% rename from tests/library/common.py rename to tests/modules/common.py diff --git a/tests/library/test_ceph_orch_host.py b/tests/modules/test_ceph_orch_host.py similarity index 93% rename from tests/library/test_ceph_orch_host.py rename to tests/modules/test_ceph_orch_host.py index 269f335..998fce9 100644 --- a/tests/library/test_ceph_orch_host.py +++ b/tests/modules/test_ceph_orch_host.py @@ -1,12 +1,12 @@ from mock.mock import patch import pytest import common -import ceph_orch_host +from ansible_collections.ceph.cephadm.plugins.modules import ceph_orch_host class TestCephOrchHost(object): - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_absent_host_exists(self, m_run_command, m_exit_json, m_get_current_state): @@ -38,7 +38,7 @@ def test_state_absent_host_exists(self, m_run_command, m_exit_json, m_get_curren assert result['stdout'] == stdout assert result['rc'] == 0 - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_absent_host_doesnt_exist(self, m_run_command, m_exit_json, m_get_current_state): @@ -71,7 +71,7 @@ def test_state_absent_host_doesnt_exist(self, m_run_command, m_exit_json, m_get_ assert result['stdout'] == 'ceph-node1 is not present, skipping.' assert result['rc'] == 0 - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_drain(self, m_run_command, m_exit_json, m_get_current_state): @@ -111,7 +111,7 @@ def test_state_drain(self, m_run_command, m_exit_json, m_get_current_state): assert result['stdout'] == stdout assert result['rc'] == 0 - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_present_no_label_diff(self, m_run_command, m_exit_json, m_get_current_state): @@ -142,7 +142,7 @@ def test_state_present_no_label_diff(self, m_run_command, m_exit_json, m_get_cur assert result['stdout'] == stdout assert result['rc'] == 0 - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_present_label_diff(self, m_run_command, m_exit_json, m_get_current_state): @@ -177,7 +177,7 @@ def test_state_present_label_diff(self, m_run_command, m_exit_json, m_get_curren assert 'label2' in result['stdout'] assert result['rc'] == 0 - @patch('ceph_orch_host.get_current_state') + @patch('ansible_collections.ceph.cephadm.plugins.modules.ceph_orch_host.get_current_state') @patch('ansible.module_utils.basic.AnsibleModule.exit_json') @patch('ansible.module_utils.basic.AnsibleModule.run_command') def test_state_present_label_diff_error(self, m_run_command, m_exit_json, m_get_current_state): diff --git a/tests/library/test_cephadm_bootstrap.py b/tests/modules/test_cephadm_bootstrap.py similarity index 99% rename from tests/library/test_cephadm_bootstrap.py rename to tests/modules/test_cephadm_bootstrap.py index 73833b8..ad0d48a 100644 --- a/tests/library/test_cephadm_bootstrap.py +++ b/tests/modules/test_cephadm_bootstrap.py @@ -1,7 +1,7 @@ from mock.mock import patch import pytest import common -import cephadm_bootstrap +from ansible_collections.ceph.cephadm.plugins.modules import cephadm_bootstrap fake_fsid = '0f1e0605-db0b-485c-b366-bd8abaa83f3b' fake_image = 'quay.ceph.io/ceph/daemon-base:latest-main-devel' diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..73b7c9a --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + admin: for admin nodes diff --git a/tests/pythonpath/ansible_collections/ceph/cephadm/plugins b/tests/pythonpath/ansible_collections/ceph/cephadm/plugins new file mode 120000 index 0000000..ebab3b6 --- /dev/null +++ b/tests/pythonpath/ansible_collections/ceph/cephadm/plugins @@ -0,0 +1 @@ +../../../../../plugins/ \ No newline at end of file diff --git a/tests/tests/test_clients.py b/tests/tests/test_clients.py new file mode 100644 index 0000000..1cffcfb --- /dev/null +++ b/tests/tests/test_clients.py @@ -0,0 +1,21 @@ +import pytest + + +class TestClients(object): + @pytest.mark.client + def test_ceph_config_file(self, node, host): + assert host.file('/etc/ceph/ceph.conf').exists + assert host.file('/etc/ceph/ceph.conf').mode == 0o600 + + @pytest.mark.client + def test_ceph_keyring(self, node, host): + assert host.file('/etc/ceph/ceph.keyring').exists + assert host.file('/etc/ceph/ceph.keyring').mode == 0o600 + + @pytest.mark.client + def test_ceph_common_package_is_installed(self, node, host): + assert host.package("ceph-common").is_installed + + @pytest.mark.client + def test_chrony_package_is_installed(self, node, host): + assert host.package("chrony").is_installed \ No newline at end of file diff --git a/tests/tests/test_cluster.py b/tests/tests/test_cluster.py new file mode 100644 index 0000000..199b792 --- /dev/null +++ b/tests/tests/test_cluster.py @@ -0,0 +1,14 @@ +import pytest +import json + + +class TestCluster(object): + @pytest.mark.admin + def test_hosts_have_expected_labels(self, node, host): + inventory_variables = host.ansible('debug', 'msg="{{ hostvars }}"')['msg'] + result = host.run('cephadm shell ceph orch host ls --format json') + self.host_ls = json.loads(result.stdout) + + for h in self.host_ls: + name = h['hostname'] + assert sorted(inventory_variables[name]['labels']) == sorted(h['labels']) diff --git a/tests/tests/test_preflight.py b/tests/tests/test_preflight.py new file mode 100644 index 0000000..6b22c02 --- /dev/null +++ b/tests/tests/test_preflight.py @@ -0,0 +1,25 @@ +import pytest + +class TestPreflight(object): + @pytest.mark.no_client + def test_cephadm_package_is_installed(self, node, host): + assert host.package("cephadm").is_installed + + @pytest.mark.no_client + def test_lvm2_package_is_installed(self, node, host): + assert host.package("lvm2").is_installed + + def test_chrony_package_is_installed(self, node, host): + assert host.package("chrony").is_installed + + @pytest.mark.no_client + def test_podman_package_is_installed(self, node, host): + assert host.package("podman").is_installed + + def test_chronyd_is_active(self, node, host): + svc = host.service("chronyd") + assert svc.is_enabled + assert svc.is_running + + def test_cephcommon_package_is_installed(self, node, host): + assert host.package("ceph-common").is_installed diff --git a/tests/tests/test_purge.py b/tests/tests/test_purge.py new file mode 100644 index 0000000..7068156 --- /dev/null +++ b/tests/tests/test_purge.py @@ -0,0 +1,7 @@ +import pytest + +class TestPurge(object): + @pytest.mark.osd + @pytest.mark.parametrize("device", ['/dev/sda', '/dev/sdb', '/dev/sdc']) + def test_devices_are_available(self, host, device): + assert host.run('python3 -c "import os; fd = os.open({}, (os.O_RDWR | os.O_EXCL), 0); os.close(fd)"'.format(device)) diff --git a/tests/wait_all_osd_are_up.yml b/tests/wait_all_osd_are_up.yml new file mode 100644 index 0000000..ad5e011 --- /dev/null +++ b/tests/wait_all_osd_are_up.yml @@ -0,0 +1,13 @@ +--- +- hosts: "admin[0]" + become: true + gather_facts: false + tasks: + - name: wait all osd are up + command: cephadm shell -- ceph osd stat -f json + register: result + retries: 100 + delay: 2 + until: + - (result.stdout | from_json)["num_osds"] | int > 0 + - (result.stdout | from_json)["num_osds"] == (result.stdout | from_json)["num_up_osds"] diff --git a/tox.ini b/tox.ini index f4be764..6ddc0c3 100644 --- a/tox.ini +++ b/tox.ini @@ -8,13 +8,13 @@ skipsdist = True basepython = python3 deps = mypy -commands = mypy {toxinidir}/library {toxinidir}/module_utils +commands = mypy {toxinidir}/plugins/modules {toxinidir}/plugins/module_utils [testenv:flake8] basepython = python3 deps = flake8 -commands = flake8 --max-line-length 160 {toxinidir}/library/ {toxinidir}/module_utils/ {toxinidir}/tests/library/ {toxinidir}/tests/module_utils +commands = flake8 --max-line-length 160 {toxinidir}/plugins/modules/ {toxinidir}/plugins/module_utils/ {toxinidir}/tests/modules/ {toxinidir}/tests/module_utils [testenv:unittests] basepython = python3 @@ -24,8 +24,8 @@ deps = mock ansible setenv= - PYTHONPATH = {env:PYTHONPATH:}:{toxinidir}/library:{toxinidir}/module_utils:{toxinidir}/tests/library -commands = py.test -vvv -n=auto {toxinidir}/tests/library/ {toxinidir}/tests/module_utils + PYTHONPATH = {env:PYTHONPATH:}:{toxinidir}/tests/pythonpath +commands = py.test -vvv -n=auto {toxinidir}/tests/modules/ {toxinidir}/tests/module_utils [testenv:{el8,el9}-functional] allowlist_externals = @@ -51,7 +51,7 @@ deps= -r{toxinidir}/tests/requirements.txt changedir= {toxinidir}/tests/functional commands= - bash {toxinidir}/tests/scripts/vagrant_up.sh --no-provision {posargs:--provider=virtualbox} + bash {toxinidir}/tests/scripts/vagrant_up.sh --no-provision bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir} # Get a system up-to-date before deploying