From 730d930c8cad73c9bc44490a8b53068574f1932e Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 13 Nov 2024 05:00:23 +0000 Subject: [PATCH 01/29] debug --- .github/workflows/integration_test.yaml | 2 ++ jenkins_rock/rockcraft.yaml | 8 ++++---- tests/integration/test_auth_proxy.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index d0ada1b5..6642676e 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,3 +24,5 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" + tmate-dubug: true + tmate-timeout: 90 diff --git a/jenkins_rock/rockcraft.yaml b/jenkins_rock/rockcraft.yaml index 33da6922..de5372d6 100644 --- a/jenkins_rock/rockcraft.yaml +++ b/jenkins_rock/rockcraft.yaml @@ -36,17 +36,17 @@ parts: - curl - libnss3 - unzip - - default-jdk-headless + - openjdk-21-jdk-headless overlay-packages: - bash - ca-certificates-java - fonts-dejavu-core - libfontconfig1 - - default-jre-headless + - openjdk-21-jre-headless - git build-environment: - - JENKINS_VERSION: 2.462.2 - - JENKINS_PLUGIN_MANAGER_VERSION: 2.12.13 + - JENKINS_VERSION: 2.479.1 + - JENKINS_PLUGIN_MANAGER_VERSION: 2.13.2 override-build: | mkdir -p ${CRAFT_PART_INSTALL}/{srv/jenkins/,etc/default/jenkins/} # Use jenkins war rather than apt install for easier Jenkins version control. diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index 3ab03ee9..a7e41288 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -150,7 +150,7 @@ async def test_auth_proxy_integration_authorized( ) -> None: """ arrange: Deploy jenkins, the authentication bundle and DEX. - act: log into via DEX + act: log in via DEX assert: the browser is redirected to the Jenkins URL with response code 200 """ status = await application.model.get_status() From 75bb8a3108fc5ed1718abc2d53ece90f234df17a Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 13 Nov 2024 05:14:17 +0000 Subject: [PATCH 02/29] debug (typo) --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 6642676e..8a52533a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,5 +24,5 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" - tmate-dubug: true + tmate-debug: true tmate-timeout: 90 From bc9856e981175575a8ba61ecf0890ff7f95508c6 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 13 Nov 2024 07:45:21 +0000 Subject: [PATCH 03/29] fix: rockcraft version --- jenkins_rock/rockcraft.yaml | 15 ++++++++++++++- src-docs/jenkins.py.md | 35 ++++++++++++++++++----------------- src/jenkins.py | 3 ++- tests/unit/conftest.py | 4 ++-- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/jenkins_rock/rockcraft.yaml b/jenkins_rock/rockcraft.yaml index de5372d6..374bdaad 100644 --- a/jenkins_rock/rockcraft.yaml +++ b/jenkins_rock/rockcraft.yaml @@ -37,13 +37,26 @@ parts: - libnss3 - unzip - openjdk-21-jdk-headless + # Referred from https://github.com/jenkinsci/docker/blob/master/debian/bookworm/hotspot/Dockerfile overlay-packages: - bash + - ca-certificates - ca-certificates-java + - curl - fonts-dejavu-core + - git + - gnupg + - gpg - libfontconfig1 + - libfreetype6 + - libharfbuzz0b - openjdk-21-jre-headless - - git + - procps + - ssh-client + - tini + - tzdata + - wget + - unzip build-environment: - JENKINS_VERSION: 2.479.1 - JENKINS_PLUGIN_MANAGER_VERSION: 2.13.2 diff --git a/src-docs/jenkins.py.md b/src-docs/jenkins.py.md index fb02817e..d9cf0d0d 100644 --- a/src-docs/jenkins.py.md +++ b/src-docs/jenkins.py.md @@ -8,6 +8,7 @@ Functions to operate Jenkins. **Global Variables** --------------- - **WEB_PORT** +- **JENKINS_PLUGIN_MANAGER_VERSION** - **LOGIN_PATH** - **JUJU_API_TOKEN** - **REQUIRED_PLUGINS** @@ -30,7 +31,7 @@ Functions to operate Jenkins. --- - + ## function `get_admin_credentials` @@ -54,7 +55,7 @@ Retrieve admin credentials. --- - + ## function `is_storage_ready` @@ -84,7 +85,7 @@ Return whether the Jenkins home directory is mounted and owned by jenkins. --- - + ## function `install_default_config` @@ -103,7 +104,7 @@ Install default jenkins-config.xml. --- - + ## function `install_auth_proxy_config` @@ -122,7 +123,7 @@ Install jenkins-config.xml for auth_proxy. --- - + ## function `install_logging_config` @@ -141,7 +142,7 @@ Install logging config. --- - + ## function `get_agent_name` @@ -202,7 +203,7 @@ Wrapper for Jenkins functionality. Attrs: environment: the Jenkins environment configuration. web_url: the Jenkins web URL. login_url: the Jenkins login URL. version: the Jenkins version. - + ### function `__init__` @@ -256,7 +257,7 @@ Returns: the web URL. --- - + ### function `add_agent_node` @@ -281,7 +282,7 @@ Add a Jenkins agent node. --- - + ### function `bootstrap` @@ -311,7 +312,7 @@ Initialize and install Jenkins. --- - + ### function `get_node_secret` @@ -341,7 +342,7 @@ Get node secret from jenkins. --- - + ### function `remove_agent_node` @@ -366,7 +367,7 @@ Remove a Jenkins agent node. --- - + ### function `remove_unlisted_plugins` @@ -396,7 +397,7 @@ Remove plugins that are not in the list of desired plugins. --- - + ### function `rotate_credentials` @@ -425,7 +426,7 @@ Invalidate all Jenkins sessions and create new password for admin account. --- - + ### function `safe_restart` @@ -449,7 +450,7 @@ Safely restart Jenkins server after all jobs are done executing. --- - + ### function `update_prefix` @@ -467,7 +468,7 @@ Update jenkins prefix. --- - + ### function `wait_ready` @@ -529,7 +530,7 @@ Represents an error probing for Jenkins storage mount. - `msg`: Explanation of the error. - + ### function `__init__` diff --git a/src/jenkins.py b/src/jenkins.py index e949eeff..95cf99a6 100644 --- a/src/jenkins.py +++ b/src/jenkins.py @@ -31,6 +31,7 @@ logger = logging.getLogger(__name__) WEB_PORT = 8080 +JENKINS_PLUGIN_MANAGER_VERSION = "2.13.2" LOGIN_PATH = "/login?from=%2F" EXECUTABLES_PATH = Path("/srv/jenkins/") JENKINS_HOME_PATH = Path("/var/lib/jenkins") @@ -841,7 +842,7 @@ def _install_plugins( "java", *proxy_args, "-jar", - "jenkins-plugin-manager-2.12.13.jar", + f"jenkins-plugin-manager-{JENKINS_PLUGIN_MANAGER_VERSION}.jar", "-w", "jenkins.war", "-d", diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index a5eb4d24..a6d9f2d4 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -215,7 +215,7 @@ def cmd_handler(argv: list[str]) -> tuple[int, str, str]: case _ if [ "java", "-jar", - "jenkins-plugin-manager-2.12.13.jar", + f"jenkins-plugin-manager-{jenkins.JENKINS_PLUGIN_MANAGER_VERSION}.jar", "-w", "jenkins.war", "-d", @@ -237,7 +237,7 @@ def cmd_handler(argv: list[str]) -> tuple[int, str, str]: f"-Dhttps.proxyPassword={proxy_config.https_proxy.password}", f'-Dhttp.nonProxyHosts="{no_proxy_hosts}"', "-jar", - "jenkins-plugin-manager-2.12.13.jar", + f"jenkins-plugin-manager-{jenkins.JENKINS_PLUGIN_MANAGER_VERSION}.jar", "-w", "jenkins.war", "-d", From e177c03eae3666c52f49901d0eb34f0a477f625b Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sun, 17 Nov 2024 10:29:30 +0000 Subject: [PATCH 04/29] update identity bundle versions --- .../files/identity-bundle-edge-patched.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/files/identity-bundle-edge-patched.yaml b/tests/integration/files/identity-bundle-edge-patched.yaml index d1810cc6..923232f0 100644 --- a/tests/integration/files/identity-bundle-edge-patched.yaml +++ b/tests/integration/files/identity-bundle-edge-patched.yaml @@ -6,15 +6,15 @@ issues: https://github.com/canonical/iam-bundle/issues applications: hydra: charm: hydra - revision: 269 - channel: latest/edge + revision: 320 + channel: 0.3/edge scale: 1 series: jammy trust: true kratos: charm: kratos - revision: 393 - channel: latest/edge + revision: 495 + channel: 0.4/edge scale: 1 series: jammy trust: true @@ -25,8 +25,8 @@ applications: series: jammy identity-platform-login-ui-operator: charm: identity-platform-login-ui-operator - revision: 79 - channel: latest/edge + revision: 135 + channel: 0.3/edge scale: 1 series: jammy trust: true @@ -56,7 +56,7 @@ applications: channel: latest/stable series: focal scale: 1 - revision: 170 + revision: 176 trust: true relations: - [hydra:pg-database, postgresql-k8s:database] From ddbc2d61b369eb82b4dff086bdf5c595d0aeeea5 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sun, 17 Nov 2024 10:48:13 +0000 Subject: [PATCH 05/29] trivy ignore unpatched vuln --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index b9c9d71e..9e019f1e 100644 --- a/.trivyignore +++ b/.trivyignore @@ -6,3 +6,4 @@ CVE-2024-34156 CVE-2023-5072 # Jenkins CVEs CVE-2016-1000027 +CVE-2024-47072 From 90c6ed4ffab44fa772c4d5e4c55b3218b2553456 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 19 Nov 2024 02:17:16 +0000 Subject: [PATCH 06/29] test: use juju 3.5 --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8a52533a..e03b75d2 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -20,7 +20,7 @@ jobs: -c "sudo microk8s config > ${GITHUB_WORKSPACE}/kube-config chmod +x tests/integration/pre_run_script.sh ./tests/integration/pre_run_script.sh" - juju-channel: 3.1/stable + juju-channel: 3/stable self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" From 23f1434ee36802036f8799d189fdea1d17719b8d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 19 Nov 2024 13:49:58 +0000 Subject: [PATCH 07/29] build command and fix comments --- tox.ini | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 7e218e39..28d3238f 100644 --- a/tox.ini +++ b/tox.ini @@ -24,10 +24,13 @@ passenv = description = Build the rock and pack the charm allowlist_externals=sh commands = + ; bulid rock and push to microk8s registry sh -c "cd jenkins_rock && rockcraft pack && \ - sudo skopeo --insecure-policy copy oci-archive:jenkins_1.0_amd64.rock \ - docker-daemon:localhost:32000/jenkins:test" - sh -c "docker push localhost:32000/jenkins:test" + sudo rockcraft.skopeo \ + --insecure-policy copy \ + --dest-tls-verify=false \ + oci-archive:jenkins_1.0_amd64.rock \ + docker://localhost:32000/jenkins:test" sh -c "charmcraft pack" [testenv:fmt] @@ -127,10 +130,10 @@ commands = playwright install --with-deps firefox pytest --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} # uncomment the following lines to use with output of tox -e build. - # --jenkins-image=localhost:32000/jenkins:test \ - # --charm-file=./jenkins-k8s_ubuntu-22.04-amd64.charm \ - # --kube-config=~/.kube/config \ - # --keep-models + ; --jenkins-image=localhost:32000/jenkins:test \ + ; --charm-file=./jenkins-k8s_ubuntu-22.04-amd64.charm \ + ; --kube-config=~/.kube/config \ + ; --keep-models [testenv:static] description = Run static analysis tests From defafbbf6dc20c7f1c658c0d46fd84d0d13e881a Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 19 Nov 2024 13:50:20 +0000 Subject: [PATCH 08/29] try manual deploy --- tests/integration/conftest.py | 112 ++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 368f8189..23281195 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -798,50 +798,100 @@ async def oathkeeper_application_related_fixture( ): """The application related to Jenkins via auth_proxy v0 relation.""" oathkeeper = await application.model.deploy("oathkeeper", channel="edge", trust=True) - # Using a patched local bundle from identity team here as a temporary workaround for - # https://github.com/canonical/traefik-k8s-operator/issues/322 and - # https://github.com/juju/python-libjuju/issues/1042 - await ops_test.run( - "juju", "deploy", "./tests/integration/files/identity-bundle-edge-patched.yaml", "--trust" - ) - await application.model.applications["kratos-external-idp-integrator"].set_config( - { + kratos_app = await application.model.deploy( + "kratos", + channel="0.4/edge", + # Needed per https://github.com/canonical/oathkeeper-operator/issues/49 + config={"dev": "True"}, + trust=True, + ) + kratos_external_idp_integrator_app = await application.model.deploy( + "kratos-external-idp-integrator", + channel="latest/edge", + config={ "client_id": "client_id", "client_secret": "client_secret", "provider": "generic", "issuer_url": ext_idp_service, "scope": "profile email", "provider_id": "Dex", - } + }, ) - - # See https://github.com/canonical/kratos-operator/issues/182 - await application.model.wait_for_idle( - status="active", - apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, - raise_on_error=False, - timeout=30 * 60, - idle_period=5, + hydra_app = await application.model.deploy("hydra", channel="edge", series="jammy", trust=True) + postgresql_app = await ops_test.model.deploy( + entity_url="postgresql-k8s", + channel="14/stable", + series="jammy", + trust=True, + ) + traefik_public_app = await ops_test.model.deploy( + "traefik-k8s", + application_name="traefik-public", + channel="latest/edge", + config={ + "external_hostname": "public-ingress", + "enable_experimental_forward_auth": "True", + }, + trust=True, + ) + traefik_admin_app = await ops_test.model.deploy( + "traefik-k8s", + application_name="traefik-admin", + channel="latest/edge", + config={"external_hostname": "admin-ingress"}, + trust=True, + ) + ca_app = await ops_test.model.deploy( + "self-signed-certificates", + channel="latest/stable", + trust=True, + ) + login_ui_app = await ops_test.model.deploy( + "identity-platform-login-ui-operator", + channel="0.3/edge", + trust=True, ) + await application.model.add_relation(f"{hydra_app.name}:pg-database", postgresql_app.name) + await application.model.add_relation(f"{kratos_app.name}:pg-database", postgresql_app.name) await application.model.add_relation( - f"{oathkeeper.name}:certificates", "self-signed-certificates" + f"{kratos_app.name}:hydra-endpoint-info", f"{hydra_app.name}:hydra-endpoint-info" ) await application.model.add_relation( - "traefik-public:receive-ca-cert", "self-signed-certificates" + f"{kratos_external_idp_integrator_app.name}:kratos-external-idp", + f"{kratos_app.name}:kratos-external-idp", ) - await application.model.applications["traefik-public"].set_config( - {"enable_experimental_forward_auth": "True"} + await application.model.add_relation( + f"{hydra_app.name}:admin-ingress", f"{traefik_admin_app.name}:ingress" ) await application.model.add_relation( - f"{oathkeeper.name}", "traefik-public:experimental-forward-auth" + f"{hydra_app.name}:public-ingress", f"{traefik_public_app.name}:ingress" + ) + await application.model.add_relation( + f"{kratos_app.name}:admin-ingress", f"{traefik_admin_app.name}:ingress" + ) + await application.model.add_relation( + f"{kratos_app.name}:public-ingress", f"{traefik_public_app.name}:ingress" + ) + await application.model.add_relation(f"{login_ui_app.name}:ingress", traefik_public_app.name) + await application.model.add_relation( + f"{login_ui_app.name}:hydra-endpoint-info", f"{hydra_app.name}:hydra-endpoint-info" + ) + await application.model.add_relation( + f"{login_ui_app.name}:ui-endpoint-info", f"{hydra_app.name}:ui-endpoint-info" + ) + await application.model.add_relation( + f"{login_ui_app.name}:ui-endpoint-info", f"{kratos_app.name}:ui-endpoint-info" + ) + await application.model.add_relation( + f"{login_ui_app.name}:kratos-info", f"{kratos_app.name}:kratos-info" + ) + await application.model.add_relation( + f"{traefik_admin_app.name}:certificates", f"{ca_app.name}:certificates" + ) + await application.model.add_relation( + f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" ) - await application.model.add_relation(f"{oathkeeper.name}:kratos-info", "kratos") - # Needed per https://github.com/canonical/oathkeeper-operator/issues/49 - await application.model.applications["kratos"].set_config({"dev": "True"}) - await application.model.add_relation(f"{application.name}:ingress", "traefik-public") - await application.model.add_relation(f"{application.name}:auth-proxy", oathkeeper.name) - await application.model.wait_for_idle( status="active", apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, @@ -850,10 +900,8 @@ async def oathkeeper_application_related_fixture( idle_period=5, ) - get_redirect_uri_action = ( - await application.model.applications["kratos-external-idp-integrator"] - .units[0] - .run_action("get-redirect-uri") + get_redirect_uri_action = await kratos_external_idp_integrator_app.units[0].run_action( + "get-redirect-uri" ) action_output = await get_redirect_uri_action.wait() update_redirect_uri(client, action_output.results["redirect-uri"]) From 8a58d0224aa18772e82c6dace37ba90187a20a23 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 03:50:09 +0000 Subject: [PATCH 09/29] test: public ingress relation jenkins --- tests/integration/conftest.py | 1 + .../files/identity-bundle-edge-patched.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 23281195..11ffb422 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -861,6 +861,7 @@ async def oathkeeper_application_related_fixture( f"{kratos_external_idp_integrator_app.name}:kratos-external-idp", f"{kratos_app.name}:kratos-external-idp", ) + await application.model.add_relation(f"{application.name}:ingress", traefik_public_app.name) await application.model.add_relation( f"{hydra_app.name}:admin-ingress", f"{traefik_admin_app.name}:ingress" ) diff --git a/tests/integration/files/identity-bundle-edge-patched.yaml b/tests/integration/files/identity-bundle-edge-patched.yaml index 923232f0..388b2c4c 100644 --- a/tests/integration/files/identity-bundle-edge-patched.yaml +++ b/tests/integration/files/identity-bundle-edge-patched.yaml @@ -6,8 +6,8 @@ issues: https://github.com/canonical/iam-bundle/issues applications: hydra: charm: hydra - revision: 320 - channel: 0.3/edge + revision: 285 + channel: 0.2/stable scale: 1 series: jammy trust: true @@ -32,9 +32,9 @@ applications: trust: true resources: oci-image: 79 - postgresql-k8s: - charm: postgresql-k8s - revision: 300 + postgresql: + charm: postgresql-k8s + revision: 444 channel: 14/stable series: jammy scale: 1 From 2e026a596dfbe5a58d7987fa72aa371b2ac1e283 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 04:04:14 +0000 Subject: [PATCH 10/29] fix: typo & lint --- tests/integration/conftest.py | 12 ++++++------ tox.ini | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 11ffb422..91dfde8b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -794,7 +794,7 @@ async def traefik_application_fixture(model: Model): @pytest_asyncio.fixture(scope="module", name="oathkeeper_related") async def oathkeeper_application_related_fixture( - ops_test: OpsTest, application: Application, client: Client, ext_idp_service: str + application: Application, client: Client, ext_idp_service: str ): """The application related to Jenkins via auth_proxy v0 relation.""" oathkeeper = await application.model.deploy("oathkeeper", channel="edge", trust=True) @@ -818,13 +818,13 @@ async def oathkeeper_application_related_fixture( }, ) hydra_app = await application.model.deploy("hydra", channel="edge", series="jammy", trust=True) - postgresql_app = await ops_test.model.deploy( + postgresql_app = await application.model.deploy( entity_url="postgresql-k8s", channel="14/stable", series="jammy", trust=True, ) - traefik_public_app = await ops_test.model.deploy( + traefik_public_app = await application.model.deploy( "traefik-k8s", application_name="traefik-public", channel="latest/edge", @@ -834,19 +834,19 @@ async def oathkeeper_application_related_fixture( }, trust=True, ) - traefik_admin_app = await ops_test.model.deploy( + traefik_admin_app = await application.model.deploy( "traefik-k8s", application_name="traefik-admin", channel="latest/edge", config={"external_hostname": "admin-ingress"}, trust=True, ) - ca_app = await ops_test.model.deploy( + ca_app = await application.model.deploy( "self-signed-certificates", channel="latest/stable", trust=True, ) - login_ui_app = await ops_test.model.deploy( + login_ui_app = await application.model.deploy( "identity-platform-login-ui-operator", channel="0.3/edge", trust=True, diff --git a/tox.ini b/tox.ini index 28d3238f..5b5075e3 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ passenv = description = Build the rock and pack the charm allowlist_externals=sh commands = - ; bulid rock and push to microk8s registry + ; build rock and push to microk8s registry sh -c "cd jenkins_rock && rockcraft pack && \ sudo rockcraft.skopeo \ --insecure-policy copy \ From a78dd2c86a8b23672ad8563ffca76c38d98acc45 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 06:43:33 +0000 Subject: [PATCH 11/29] test: add oathkeeper relations --- tests/integration/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 91dfde8b..0b1c3eee 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -817,6 +817,14 @@ async def oathkeeper_application_related_fixture( "provider_id": "Dex", }, ) + # See https://github.com/canonical/kratos-operator/issues/182 + await application.model.wait_for_idle( + status="active", + apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, + raise_on_error=False, + timeout=30 * 60, + idle_period=5, + ) hydra_app = await application.model.deploy("hydra", channel="edge", series="jammy", trust=True) postgresql_app = await application.model.deploy( entity_url="postgresql-k8s", @@ -887,12 +895,23 @@ async def oathkeeper_application_related_fixture( await application.model.add_relation( f"{login_ui_app.name}:kratos-info", f"{kratos_app.name}:kratos-info" ) + await application.model.add_relation( + f"{oathkeeper.name}:certificates", f"{traefik_public_app.name}:experimental-forward-auth" + ) + await application.model.add_relation( + f"{oathkeeper.name}:certificates", f"{ca_app.name}:certificates" + ) + await application.model.add_relation(f"{oathkeeper.name}:kratos-info", kratos_app.name) + await application.model.add_relation(f"{oathkeeper.name}:auth-proxy", application.name) await application.model.add_relation( f"{traefik_admin_app.name}:certificates", f"{ca_app.name}:certificates" ) await application.model.add_relation( f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" ) + await application.model.add_relation( + f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" + ) await application.model.wait_for_idle( status="active", apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, From 0ef28f5678d318951d8e12d9f5c630bdd4f2ec91 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 08:00:23 +0000 Subject: [PATCH 12/29] test: remove wait for idle before deploy --- tests/integration/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0b1c3eee..14493173 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -817,14 +817,6 @@ async def oathkeeper_application_related_fixture( "provider_id": "Dex", }, ) - # See https://github.com/canonical/kratos-operator/issues/182 - await application.model.wait_for_idle( - status="active", - apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, - raise_on_error=False, - timeout=30 * 60, - idle_period=5, - ) hydra_app = await application.model.deploy("hydra", channel="edge", series="jammy", trust=True) postgresql_app = await application.model.deploy( entity_url="postgresql-k8s", From ef30be1471963f0ced83dfed9b029be7d475b6a3 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 08:26:20 +0000 Subject: [PATCH 13/29] test: fix: oathkeeper relation fix --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 14493173..3d33ab5f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -888,7 +888,7 @@ async def oathkeeper_application_related_fixture( f"{login_ui_app.name}:kratos-info", f"{kratos_app.name}:kratos-info" ) await application.model.add_relation( - f"{oathkeeper.name}:certificates", f"{traefik_public_app.name}:experimental-forward-auth" + f"{oathkeeper.name}", f"{traefik_public_app.name}:experimental-forward-auth" ) await application.model.add_relation( f"{oathkeeper.name}:certificates", f"{ca_app.name}:certificates" From 933646dd8c8811f03e3525241342ef4c97d13ec2 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 20 Nov 2024 09:23:40 +0000 Subject: [PATCH 14/29] test: traefik certificates --- tests/integration/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3d33ab5f..6398d868 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -901,9 +901,6 @@ async def oathkeeper_application_related_fixture( await application.model.add_relation( f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" ) - await application.model.add_relation( - f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" - ) await application.model.wait_for_idle( status="active", apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, From b8c5f9df9c044e86ec3312355dc879d2d727e10f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 02:53:02 +0000 Subject: [PATCH 15/29] test: use model.integrate instead of deprecated model.relatet --- tests/integration/conftest.py | 116 ++++++++++++++++------------------ 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6398d868..2153f766 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,6 +3,7 @@ """Fixtures for Jenkins-k8s-operator charm integration tests.""" +import asyncio import os import random import secrets @@ -199,23 +200,21 @@ async def jenkins_k8s_agents_fixture(model: Model): @pytest_asyncio.fixture(scope="module", name="k8s_agent_related_app") async def k8s_agent_related_app_fixture( - jenkins_k8s_agents: Application, - application: Application, + jenkins_k8s_agents: Application, application: Application, model: Model ): """The Jenkins-k8s server charm related to Jenkins-k8s agent charm through agent relation.""" - await application.relate( - state.AGENT_RELATION, f"{jenkins_k8s_agents.name}:{state.AGENT_RELATION}" + await model.integrate( + f"{application.name}:{state.AGENT_RELATION}", + f"{jenkins_k8s_agents.name}:{state.AGENT_RELATION}", ) - await application.model.wait_for_idle( + await model.wait_for_idle( apps=[application.name, jenkins_k8s_agents.name], wait_for_active=True, check_freq=5 ) return application @pytest_asyncio.fixture(scope="function", name="extra_jenkins_k8s_agents") -async def extra_jenkins_k8s_agents_fixture( - model: Model, -) -> AsyncGenerator[Application, None]: +async def extra_jenkins_k8s_agents_fixture(model: Model) -> AsyncGenerator[Application, None]: """The Jenkins k8s agent.""" agent_app: Application = await model.deploy( "jenkins-agent-k8s", @@ -230,12 +229,13 @@ async def extra_jenkins_k8s_agents_fixture( @pytest_asyncio.fixture(scope="function", name="k8s_deprecated_agent_related_app") async def k8s_deprecated_agent_related_app_fixture( - jenkins_k8s_agents: Application, - application: Application, + jenkins_k8s_agents: Application, application: Application, model: Model ): """The Jenkins-k8s charm related to Jenkins-k8s agent through deprecated agent relation.""" - await application.relate(state.DEPRECATED_AGENT_RELATION, jenkins_k8s_agents.name) - await application.model.wait_for_idle( + await model.integrate( + f"{application.name}:{state.DEPRECATED_AGENT_RELATION}", jenkins_k8s_agents.name + ) + await model.wait_for_idle( apps=[application.name, jenkins_k8s_agents.name], wait_for_active=True ) yield application @@ -288,16 +288,14 @@ async def jenkins_machine_agents_fixture( @pytest_asyncio.fixture(scope="function", name="machine_agent_related_app") async def machine_agent_related_app_fixture( - jenkins_machine_agents: Application, - application: Application, + jenkins_machine_agents: Application, application: Application, model: Model ): """The Jenkins-k8s server charm related to Jenkins agent charm through agent relation.""" - model: Model = application.model machine_model: Model = jenkins_machine_agents.model await machine_model.wait_for_idle( apps=[jenkins_machine_agents.name], wait_for_active=True, check_freq=5 ) - await model.relate( + await model.integrate( f"{application.name}:{state.AGENT_RELATION}", f"localhost:admin/{machine_model.name}.{state.AGENT_RELATION}", ) @@ -310,13 +308,11 @@ async def machine_agent_related_app_fixture( @pytest_asyncio.fixture(scope="function", name="machine_deprecated_agent_related_app") async def machine_deprecated_agent_related_app_fixture( - jenkins_machine_agents: Application, - application: Application, + jenkins_machine_agents: Application, application: Application, model: Model ): """The Jenkins-k8s server charm related to Jenkins agent charm through agent relation.""" - model: Model = application.model machine_model: Model = jenkins_machine_agents.model - await model.relate( + await model.integrate( f"{application.name}:{state.DEPRECATED_AGENT_RELATION}", f"localhost:admin/{machine_model.name}.{state.DEPRECATED_AGENT_RELATION}", ) @@ -520,11 +516,10 @@ async def jenkins_with_proxy_client_fixture( @pytest_asyncio.fixture(scope="function", name="app_with_allowed_plugins") async def app_with_allowed_plugins_fixture( - application: Application, web_address: str + application: Application, web_address: str, model: Model ) -> AsyncGenerator[Application, None]: """Jenkins charm with plugins configured.""" await application.set_config({"allowed-plugins": ",".join(ALLOWED_PLUGINS)}) - model: Model = application.model await model.wait_for_idle(apps=[application.name], wait_for_active=True) await model.block_until( lambda: requests.get(web_address, timeout=10).status_code == 403, @@ -794,18 +789,18 @@ async def traefik_application_fixture(model: Model): @pytest_asyncio.fixture(scope="module", name="oathkeeper_related") async def oathkeeper_application_related_fixture( - application: Application, client: Client, ext_idp_service: str + application: Application, client: Client, ext_idp_service: str, model: Model ): """The application related to Jenkins via auth_proxy v0 relation.""" - oathkeeper = await application.model.deploy("oathkeeper", channel="edge", trust=True) - kratos_app = await application.model.deploy( + oathkeeper = await model.deploy("oathkeeper", channel="edge", trust=True) + kratos_app = await model.deploy( "kratos", channel="0.4/edge", # Needed per https://github.com/canonical/oathkeeper-operator/issues/49 config={"dev": "True"}, trust=True, ) - kratos_external_idp_integrator_app = await application.model.deploy( + kratos_external_idp_integrator_app = await model.deploy( "kratos-external-idp-integrator", channel="latest/edge", config={ @@ -817,91 +812,88 @@ async def oathkeeper_application_related_fixture( "provider_id": "Dex", }, ) - hydra_app = await application.model.deploy("hydra", channel="edge", series="jammy", trust=True) - postgresql_app = await application.model.deploy( + hydra_app = await model.deploy("hydra", channel="edge", series="jammy", trust=True) + postgresql_app = await model.deploy( entity_url="postgresql-k8s", channel="14/stable", series="jammy", trust=True, ) - traefik_public_app = await application.model.deploy( + traefik_public_app = await model.deploy( "traefik-k8s", application_name="traefik-public", channel="latest/edge", config={ - "external_hostname": "public-ingress", "enable_experimental_forward_auth": "True", }, trust=True, ) - traefik_admin_app = await application.model.deploy( + traefik_admin_app = await model.deploy( "traefik-k8s", application_name="traefik-admin", channel="latest/edge", - config={"external_hostname": "admin-ingress"}, trust=True, ) - ca_app = await application.model.deploy( + ca_app = await model.deploy( "self-signed-certificates", channel="latest/stable", trust=True, ) - login_ui_app = await application.model.deploy( + login_ui_app = await model.deploy( "identity-platform-login-ui-operator", channel="0.3/edge", trust=True, ) - await application.model.add_relation(f"{hydra_app.name}:pg-database", postgresql_app.name) - await application.model.add_relation(f"{kratos_app.name}:pg-database", postgresql_app.name) - await application.model.add_relation( + await model.add_relation(f"{hydra_app.name}:pg-database", postgresql_app.name) + await model.add_relation(f"{kratos_app.name}:pg-database", postgresql_app.name) + await model.add_relation( f"{kratos_app.name}:hydra-endpoint-info", f"{hydra_app.name}:hydra-endpoint-info" ) - await application.model.add_relation( - f"{kratos_external_idp_integrator_app.name}:kratos-external-idp", - f"{kratos_app.name}:kratos-external-idp", - ) - await application.model.add_relation(f"{application.name}:ingress", traefik_public_app.name) - await application.model.add_relation( + await model.add_relation(f"{application.name}:ingress", traefik_public_app.name) + await model.add_relation( f"{hydra_app.name}:admin-ingress", f"{traefik_admin_app.name}:ingress" ) - await application.model.add_relation( + await model.add_relation( f"{hydra_app.name}:public-ingress", f"{traefik_public_app.name}:ingress" ) - await application.model.add_relation( + await model.add_relation( f"{kratos_app.name}:admin-ingress", f"{traefik_admin_app.name}:ingress" ) - await application.model.add_relation( + await model.add_relation( f"{kratos_app.name}:public-ingress", f"{traefik_public_app.name}:ingress" ) - await application.model.add_relation(f"{login_ui_app.name}:ingress", traefik_public_app.name) - await application.model.add_relation( + await model.add_relation(f"{login_ui_app.name}:ingress", traefik_public_app.name) + await model.add_relation( f"{login_ui_app.name}:hydra-endpoint-info", f"{hydra_app.name}:hydra-endpoint-info" ) - await application.model.add_relation( + await model.add_relation( f"{login_ui_app.name}:ui-endpoint-info", f"{hydra_app.name}:ui-endpoint-info" ) - await application.model.add_relation( + await model.add_relation( f"{login_ui_app.name}:ui-endpoint-info", f"{kratos_app.name}:ui-endpoint-info" ) - await application.model.add_relation( - f"{login_ui_app.name}:kratos-info", f"{kratos_app.name}:kratos-info" - ) - await application.model.add_relation( + await model.add_relation(f"{login_ui_app.name}:kratos-info", f"{kratos_app.name}:kratos-info") + await model.add_relation( f"{oathkeeper.name}", f"{traefik_public_app.name}:experimental-forward-auth" ) - await application.model.add_relation( - f"{oathkeeper.name}:certificates", f"{ca_app.name}:certificates" - ) - await application.model.add_relation(f"{oathkeeper.name}:kratos-info", kratos_app.name) - await application.model.add_relation(f"{oathkeeper.name}:auth-proxy", application.name) - await application.model.add_relation( + await model.add_relation(f"{oathkeeper.name}:certificates", f"{ca_app.name}:certificates") + await model.add_relation(f"{oathkeeper.name}:kratos-info", kratos_app.name) + await model.add_relation(f"{oathkeeper.name}:auth-proxy", application.name) + await model.add_relation( f"{traefik_admin_app.name}:certificates", f"{ca_app.name}:certificates" ) - await application.model.add_relation( + await model.add_relation( f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" ) - await application.model.wait_for_idle( + # try sleeping since the kratos relation does not work well. + await asyncio.sleep(10) + await model.add_relation( + f"{kratos_external_idp_integrator_app.name}:kratos-external-idp", + f"{kratos_app.name}:kratos-external-idp", + ) + + await model.wait_for_idle( status="active", apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, raise_on_error=False, From 6a7278dae343aafcf8f1df838ebaa5b841b9cd7f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 03:35:44 +0000 Subject: [PATCH 16/29] test: model wait for idle to get around buggy charms --- tests/integration/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2153f766..2fe4149c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -886,19 +886,28 @@ async def oathkeeper_application_related_fixture( await model.add_relation( f"{traefik_public_app.name}:certificates", f"{ca_app.name}:certificates" ) + # Wait for idle before running actions to unblock buggy postgresql + # "awaiting for primary endpoint to be ready" + # Wait for idle before running actions to unblock buggy kratos-external-idp-integrator + # "Waiting for Kratos to register provider" + await model.wait_for_idle( + raise_on_error=False, + timeout=30 * 60, + idle_period=30, + ) + # try sleeping since the kratos relation does not work well. await asyncio.sleep(10) await model.add_relation( f"{kratos_external_idp_integrator_app.name}:kratos-external-idp", f"{kratos_app.name}:kratos-external-idp", ) - await model.wait_for_idle( status="active", apps=[application.name, oathkeeper.name] + IDENTITY_PLATFORM_APPS, raise_on_error=False, timeout=30 * 60, - idle_period=5, + idle_period=30, ) get_redirect_uri_action = await kratos_external_idp_integrator_app.units[0].run_action( From 478f67dd014baf763c727c82298150cb772da060 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 05:16:09 +0000 Subject: [PATCH 17/29] await status code --- tests/integration/test_auth_proxy.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index a7e41288..0b964562 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -128,14 +128,18 @@ async def test_auth_proxy_integration_returns_not_authorized( """ status = await model.get_status() address = status["applications"]["traefik-public"]["public-address"] - # The certificate is self signed, so verification is disabled. - response = requests.get( # nosec - f"https://{address}/{application.model.name}-{application.name}/", - verify=False, - timeout=5, - ) - assert response.status_code == 401 + def is_auth_401(): + """Get the status code of application request via ingress.""" + response = requests.get( # nosec + f"https://{address}/{application.model.name}-{application.name}/", + # The certificate is self signed, so verification is disabled. + verify=False, + timeout=5, + ) + return response.status_code == 401 + + await wait_for(is_auth_401) @pytest.mark.abort_on_fail From a69db8746f843f2ca980a3c266f3b804e5687145 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 06:18:09 +0000 Subject: [PATCH 18/29] fix tests for auth proxy --- tests/integration/test_auth_proxy.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index 0b964562..c3523230 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -11,6 +11,7 @@ import pytest_asyncio import requests from juju.application import Application +from juju.client._definitions import DetailedStatus, UnitStatus from juju.model import Model from playwright.async_api import async_playwright, expect from playwright.async_api._generated import Browser, BrowserContext, BrowserType, Page @@ -127,10 +128,17 @@ async def test_auth_proxy_integration_returns_not_authorized( assert: a 401 is returned. """ status = await model.get_status() - address = status["applications"]["traefik-public"]["public-address"] + unit_status: UnitStatus = status["applications"]["traefik-public"]["units"]["traefik-public/0"] + workload_message = str(unit_status.workload_status.info) + # The message is: Serving at + address = workload_message.removeprefix("Serving at ") def is_auth_401(): - """Get the status code of application request via ingress.""" + """Get the status code of application request via ingress. + + Returns: + Whether the status code of the request is 401. + """ response = requests.get( # nosec f"https://{address}/{application.model.name}-{application.name}/", # The certificate is self signed, so verification is disabled. From 34dc0717973da3218341c73c321fa871c26b7432 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 06:22:05 +0000 Subject: [PATCH 19/29] fix lint --- tests/integration/test_auth_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index c3523230..4584e8df 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -11,7 +11,7 @@ import pytest_asyncio import requests from juju.application import Application -from juju.client._definitions import DetailedStatus, UnitStatus +from juju.client._definitions import UnitStatus from juju.model import Model from playwright.async_api import async_playwright, expect from playwright.async_api._generated import Browser, BrowserContext, BrowserType, Page From 8af5df9f164417c610ee8d828466590e72347cb6 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 06:32:43 +0000 Subject: [PATCH 20/29] fix lint --- tests/integration/test_auth_proxy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index 4584e8df..ff7911fc 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -5,13 +5,13 @@ import logging import re -from typing import Any, AsyncGenerator, Callable, Coroutine, Match +from typing import Any, AsyncGenerator, Callable, Coroutine, Match, cast import pytest import pytest_asyncio import requests from juju.application import Application -from juju.client._definitions import UnitStatus +from juju.client._definitions import DetailedStatus, UnitStatus from juju.model import Model from playwright.async_api import async_playwright, expect from playwright.async_api._generated import Browser, BrowserContext, BrowserType, Page @@ -129,7 +129,7 @@ async def test_auth_proxy_integration_returns_not_authorized( """ status = await model.get_status() unit_status: UnitStatus = status["applications"]["traefik-public"]["units"]["traefik-public/0"] - workload_message = str(unit_status.workload_status.info) + workload_message = str(cast(DetailedStatus, unit_status.workload_status).info) # The message is: Serving at address = workload_message.removeprefix("Serving at ") From 452410e2b04417212a3a0ac7fcff96748a135d00 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 07:52:27 +0000 Subject: [PATCH 21/29] proper auth address --- tests/integration/test_auth_proxy.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index ff7911fc..4a3fd8e1 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -115,6 +115,21 @@ async def page_fixture(context: BrowserContext) -> AsyncGenerator[Page, None]: await new_page.close() +async def get_application_unit_status(model: Model, application: str) -> UnitStatus: + """Get the application unit status object. + + Args: + model: The Juju model connection object. + application: The application unit to get the unit status. + + Returns: + The application first unit's status. + """ + status = await model.get_status() + unit_status: UnitStatus = status["applications"][application]["units"][f"{application}/0"] + return unit_status + + @pytest.mark.abort_on_fail @pytest.mark.asyncio @pytest.mark.usefixtures("oathkeeper_related") @@ -127,8 +142,7 @@ async def test_auth_proxy_integration_returns_not_authorized( act: send a request Jenkins. assert: a 401 is returned. """ - status = await model.get_status() - unit_status: UnitStatus = status["applications"]["traefik-public"]["units"]["traefik-public/0"] + unit_status = await get_application_unit_status(model=model, application=application.name) workload_message = str(cast(DetailedStatus, unit_status.workload_status).info) # The message is: Serving at address = workload_message.removeprefix("Serving at ") @@ -165,8 +179,12 @@ async def test_auth_proxy_integration_authorized( act: log in via DEX assert: the browser is redirected to the Jenkins URL with response code 200 """ - status = await application.model.get_status() - address = status["applications"]["traefik-public"]["public-address"] + unit_status = await get_application_unit_status( + model=application.model, application=application.name + ) + workload_message = str(cast(DetailedStatus, unit_status.workload_status).info) + # The message is: Serving at + address = workload_message.removeprefix("Serving at ") jenkins_url = f"https://{address}/{application.model.name}-{application.name}/" expected_url = ( f"https://{address}/{application.model.name}" From fc9e56ac5b62c49528c1285160b16d60c3d2c35f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 21 Nov 2024 14:33:43 +0000 Subject: [PATCH 22/29] fix: auth proxy test --- tests/integration/test_auth_proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_auth_proxy.py b/tests/integration/test_auth_proxy.py index 4a3fd8e1..8815c323 100644 --- a/tests/integration/test_auth_proxy.py +++ b/tests/integration/test_auth_proxy.py @@ -142,7 +142,7 @@ async def test_auth_proxy_integration_returns_not_authorized( act: send a request Jenkins. assert: a 401 is returned. """ - unit_status = await get_application_unit_status(model=model, application=application.name) + unit_status = await get_application_unit_status(model=model, application="traefik-public") workload_message = str(cast(DetailedStatus, unit_status.workload_status).info) # The message is: Serving at address = workload_message.removeprefix("Serving at ") @@ -180,7 +180,7 @@ async def test_auth_proxy_integration_authorized( assert: the browser is redirected to the Jenkins URL with response code 200 """ unit_status = await get_application_unit_status( - model=application.model, application=application.name + model=application.model, application="traefik-public" ) workload_message = str(cast(DetailedStatus, unit_status.workload_status).info) # The message is: Serving at From 1d84578d6fbe49e87f31344366c69918c42fa980 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 28 Nov 2024 06:14:38 +0000 Subject: [PATCH 23/29] test: fix thinbackup plugin request --- tests/integration/constants.py | 9 +++++++++ tests/integration/test_plugins.py | 13 +++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/integration/constants.py b/tests/integration/constants.py index 9abe6160..72ae55d0 100644 --- a/tests/integration/constants.py +++ b/tests/integration/constants.py @@ -23,3 +23,12 @@ "ssh-agent", "thinBackup", ] +DEFAULT_SYSTEM_CONFIGURE_PAYLOAD = { + "jenkins-model-MasterBuildConfiguration": {"numExecutors": "0"}, + "jenkins-model-GlobalComputerRetentionCheckIntervalConfiguration": { + "computerRetentionCheckInterval": "60" + }, + "jenkins-model-GlobalQuietPeriodConfiguration": {"quietPeriod": "5"}, + "jenkins-model-GlobalSCMRetryCountConfiguration": {"scmCheckoutRetryCount": "0"}, + "com-sonyericsson-rebuild-RebuildDescriptor": {"rememberPasswordEnabled": True}, +} diff --git a/tests/integration/test_plugins.py b/tests/integration/test_plugins.py index 3dc1d2ec..c97b07f1 100644 --- a/tests/integration/test_plugins.py +++ b/tests/integration/test_plugins.py @@ -16,7 +16,12 @@ from juju.application import Application from pytest_operator.plugin import OpsTest -from .constants import ALLOWED_PLUGINS, INSTALLED_PLUGINS, REMOVED_PLUGINS +from .constants import ( + ALLOWED_PLUGINS, + DEFAULT_SYSTEM_CONFIGURE_PAYLOAD, + INSTALLED_PLUGINS, + REMOVED_PLUGINS, +) from .helpers import ( create_kubernetes_cloud, create_secret_file_credentials, @@ -346,11 +351,7 @@ async def test_thinbackup_plugin(ops_test: OpsTest, unit_web_client: UnitWebClie await install_plugins(unit_web_client, ("thinBackup",)) backup_path = "/srv/jenkins/backup/" payload = { - "jenkins-model-MasterBuildConfiguration": { - "numExecutors": "0", - }, - "jenkins-model-GlobalQuietPeriodConfiguration": {"quietPeriod": "5"}, - "jenkins-model-GlobalSCMRetryCountConfiguration": {"scmCheckoutRetryCount": "0"}, + **DEFAULT_SYSTEM_CONFIGURE_PAYLOAD, "org-jvnet-hudson-plugins-thinbackup-ThinBackupPluginImpl": { "backupPath": backup_path, }, From 49e2a2eb404bcea8d5bfd585a598bdae9eeca3d6 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 29 Nov 2024 02:54:03 +0000 Subject: [PATCH 24/29] remove debug --- .github/workflows/integration_test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index e03b75d2..0fd97739 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,5 +24,3 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" - tmate-debug: true - tmate-timeout: 90 From 2ce7c1acc95c6c269ac4920af84752ccf4afb116 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 29 Nov 2024 03:16:29 +0000 Subject: [PATCH 25/29] fix type hint --- tests/integration/test_upgrade.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_upgrade.py b/tests/integration/test_upgrade.py index 7759202f..7cbae498 100644 --- a/tests/integration/test_upgrade.py +++ b/tests/integration/test_upgrade.py @@ -5,8 +5,8 @@ import logging +import pathlib -import ops import pytest import pytest_asyncio import requests @@ -46,7 +46,7 @@ async def jenkins_upgrade_depl(ops_test: OpsTest, model: Model): @pytest.mark.usefixtures("jenkins_upgrade_depl") async def test_jenkins_upgrade_check_job( - ops_test: OpsTest, jenkins_image: str, model: Model, charm: ops.CharmBase + ops_test: OpsTest, jenkins_image: str, model: Model, charm: str | pathlib.Path ): """ arrange: given charm has been built, deployed and a job has been defined. From 684bafe785b2d2ee81b82cb28d2868eabbd67b5a Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 29 Nov 2024 07:07:31 +0000 Subject: [PATCH 26/29] split plugins test --- .github/workflows/integration_test.yaml | 2 +- ...test_plugins.py => test_plugins_part_1.py} | 248 +--------------- tests/integration/test_plugins_part_2.py | 266 ++++++++++++++++++ 3 files changed, 268 insertions(+), 248 deletions(-) rename tests/integration/{test_plugins.py => test_plugins_part_1.py} (59%) create mode 100644 tests/integration/test_plugins_part_2.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 0fd97739..fd1c4551 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -15,7 +15,7 @@ jobs: channel: 1.28-strict/stable extra-arguments: | --kube-config=${GITHUB_WORKSPACE}/kube-config - modules: '["test_auth_proxy.py", "test_cos.py", "test_ingress.py", "test_jenkins.py", "test_k8s_agent.py", "test_machine_agent.py", "test_plugins.py", "test_proxy.py", "test_upgrade.py", "test_external_agent.py"]' + modules: '["test_auth_proxy.py", "test_cos.py", "test_ingress.py", "test_jenkins.py", "test_k8s_agent.py", "test_machine_agent.py", "test_plugins_part_1.py", "test_plugins_part_2.py", "test_proxy.py", "test_upgrade.py", "test_external_agent.py"]' pre-run-script: | -c "sudo microk8s config > ${GITHUB_WORKSPACE}/kube-config chmod +x tests/integration/pre_run_script.sh diff --git a/tests/integration/test_plugins.py b/tests/integration/test_plugins_part_1.py similarity index 59% rename from tests/integration/test_plugins.py rename to tests/integration/test_plugins_part_1.py index c97b07f1..073d390a 100644 --- a/tests/integration/test_plugins.py +++ b/tests/integration/test_plugins_part_1.py @@ -3,7 +3,6 @@ """Integration tests for jenkins-k8s-operator charm.""" -import functools import json import logging import typing @@ -23,18 +22,13 @@ REMOVED_PLUGINS, ) from .helpers import ( - create_kubernetes_cloud, - create_secret_file_credentials, - declarative_pipeline_script, gen_git_test_job_xml, gen_test_job_xml, - gen_test_pipeline_with_custom_script_xml, get_job_invoked_unit, install_plugins, - kubernetes_test_pipeline_script, wait_for, ) -from .types_ import KeycloakOIDCMetadata, LDAPSettings, UnitWebClient +from .types_ import LDAPSettings, UnitWebClient logger = logging.getLogger(__name__) @@ -405,243 +399,3 @@ async def test_bzr_plugin(unit_web_client: UnitWebClient): config_page = str(res.content, "utf-8") assert "Bazaar" in config_page, f"Bzr configuration option not found. {config_page}" - - -async def test_docker_build_publish_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with docker-build-publish plugin installed. - act: when a job configuration page is accessed. - assert: docker-build-publish plugin option exists. - """ - await install_plugins(unit_web_client, ("docker-build-publish",)) - unit_web_client.client.create_job("docker_plugin_test", gen_test_job_xml("k8s")) - res = unit_web_client.client.requester.get_url( - f"{unit_web_client.web}/job/docker_plugin_test/configure" - ) - config_page = str(res.content, "utf-8") - assert ( - "Docker Build and Publish" in config_page - ), f"docker-build-publish configuration option not found. {config_page}" - - -async def test_reverse_proxy_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with reverse-proxy-auth-plugin plugin installed. - act: when the security configuration is accessed. - assert: reverse-proxy-auth-plugin plugin option exists. - """ - await install_plugins(unit_web_client, ("reverse-proxy-auth-plugin",)) - - res = unit_web_client.client.requester.get_url( - f"{unit_web_client.web}/manage/configureSecurity" - ) - config_page = str(res.content, "utf-8") - - assert ( - "HTTP Header by reverse proxy" in config_page - ), f"reverse-proxy-auth-plugin configuration option not found. {config_page}" - - -async def test_dependency_check_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with dependency-check-jenkins-plugin plugin installed. - act: when a job configuration page is accessed. - assert: dependency-check-jenkins-plugin plugin option exists. - """ - await install_plugins(unit_web_client, ("dependency-check-jenkins-plugin",)) - unit_web_client.client.create_job("deps_plugin_test", gen_test_job_xml("k8s")) - res = unit_web_client.client.requester.get_url( - f"{unit_web_client.web}/job/deps_plugin_test/configure" - ) - job_page = str(res.content, "utf-8") - assert ( - "Invoke Dependency-Check" in job_page - ), f"Dependency check job configuration option not found. {job_page}" - res = unit_web_client.client.requester.get_url(f"{unit_web_client.web}/manage/configureTools/") - tools_page = str(res.content, "utf-8") - assert ( - "Dependency-Check installations" in tools_page - ), f"Dependency check tool configuration option not found. {tools_page}" - - -async def test_groovy_libs_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with pipeline-groovy-lib plugin installed. - act: when a job configuration page is accessed. - assert: pipeline-groovy-lib plugin option exists. - """ - await install_plugins(unit_web_client, ("pipeline-groovy-lib",)) - res = unit_web_client.client.requester.get_url(f"{unit_web_client.web}/manage/configure") - - config_page = str(res.content, "utf-8") - # The string is now "Global Trusted Pipeline Libraries" and - # "Global Untrusted Pipeline Libraries" for v727.ve832a_9244dfa_ - assert ( - "Pipeline Libraries" in config_page - ), f"Groovy libs configuration option not found. {config_page}" - - -@pytest.mark.usefixtures("k8s_agent_related_app") -async def test_rebuilder_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with rebuilder plugin installed. - act: when a job is built and a rebuild is triggered. - assert: last job is rebuilt. - """ - await install_plugins(unit_web_client, ("rebuild",)) - - job_name = "rebuild_test" - job = unit_web_client.client.create_job(job_name, gen_test_job_xml("k8s")) - job.invoke().block_until_complete() - - unit_web_client.client.requester.post_url( - f"{unit_web_client.web}/job/{job_name}/lastCompletedBuild/rebuild/" - ) - job.get_last_build().block_until_complete() - - assert job.get_last_buildnumber() == 2, "Rebuild not triggered." - - -async def test_openid_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with openid plugin installed. - act: when an openid endpoint is validated using the plugin. - assert: the response returns a 200 status code. - """ - await install_plugins(unit_web_client, ("openid",)) - - res = unit_web_client.client.requester.post_url( - f"{unit_web_client.web}/manage/descriptorByName/hudson.plugins.openid." - "OpenIdSsoSecurityRealm/validate", - data={"endpoint": "https://login.ubuntu.com/+openid"}, - ) - - assert res.status_code == 200, "Failed to validate openid endpoint using the plugin." - - -async def test_openid_connect_plugin( - unit_web_client: UnitWebClient, - keycloak_oidc_meta: KeycloakOIDCMetadata, - keycloak_ip: str, -): - """ - arrange: given a Jenkins charm with oic-auth plugin installed and a Keycloak oidc server. - act: - 1. when jenkins security realm is configured with oidc server and login page is requested. - 2. when jenkins security realm is reset and login page is requested. - assert: - 1. a redirection to Keycloak SSO is made. - 2. native Jenkins login ui is loaded. - """ - await install_plugins(unit_web_client, ("oic-auth",)) - - # 1. when jenkins security realm is configured with oidc server and login page is requested. - payload: dict = { - "securityRealm": { - "clientId": keycloak_oidc_meta.client_id, - "clientSecret": keycloak_oidc_meta.client_secret, - "automanualconfigure": "auto", - "serverConfiguration": { - "wellKnownOpenIDConfigurationUrl": keycloak_oidc_meta.well_known_endpoint, - "scopesOverride": "", - "stapler-class": "org.jenkinsci.plugins.oic.OicServerWellKnownConfiguration", - "$class": "org.jenkinsci.plugins.oic.OicServerWellKnownConfiguration", - }, - "userNameField": "sub", - "stapler-class": "org.jenkinsci.plugins.oic.OicSecurityRealm", - "$class": "org.jenkinsci.plugins.oic.OicSecurityRealm", - }, - "slaveAgentPort": {"type": "fixed", "value": "50000"}, - } - res = unit_web_client.client.requester.post_url( - f"{unit_web_client.web}/manage/configureSecurity/configure", - data=[ - ( - "json", - json.dumps(payload), - ), - ], - ) - res = requests.get(f"{unit_web_client.web}/securityRealm/commenceLogin?from=%2F", timeout=30) - assert res.history[0].status_code == 302, "Jenkins login not redirected." - assert keycloak_ip in res.history[0].headers["location"], "Login not redirected to keycloak." - - # 2. when jenkins security realm is reset and login page is requested. - payload = { - "securityRealm": { - "allowsSignup": False, - "stapler-class": "hudson.security.HudsonPrivateSecurityRealm", - "$class": "hudson.security.HudsonPrivateSecurityRealm", - }, - "authorizationStrategy": { - "allowAnonymousRead": False, - "stapler-class": "hudson.security.FullControlOnceLoggedInAuthorizationStrategy", - "$class": "hudson.security.FullControlOnceLoggedInAuthorizationStrategy", - }, - "slaveAgentPort": {"type": "fixed", "value": "50000"}, - } - res = unit_web_client.client.requester.post_url( - f"{unit_web_client.web}/manage/configureSecurity/configure", - data=[ - ( - "json", - json.dumps(payload), - ) - ], - ) - res = requests.get(f"{unit_web_client.web}/securityRealm/commenceLogin?from=%2F", timeout=30) - assert res.status_code == 404, "Security realm login not reset." - res = requests.get(f"{unit_web_client.web}/login?from=%2F", timeout=30) - assert res.status_code == 200, "Failed to load Jenkins native login UI." - - -async def test_kubernetes_plugin(unit_web_client: UnitWebClient, kube_config: str): - """ - arrange: given a Jenkins charm with kubernetes plugin installed and credentials from microk8s. - act: Run a job using an agent provided by the kubernetes plugin. - assert: Job succeeds. - """ - # Use plain credentials to be able to create secret-file/secret-text credentials - await install_plugins(unit_web_client, ("kubernetes", "plain-credentials")) - credentials_id = await wait_for( - functools.partial(create_secret_file_credentials, unit_web_client, kube_config) - ) - assert credentials_id, "Failed to create credentials id" - kubernetes_cloud_name = await wait_for( - functools.partial(create_kubernetes_cloud, unit_web_client, credentials_id) - ) - assert kubernetes_cloud_name, "Failed to create kubernetes cloud" - job = unit_web_client.client.create_job( - "kubernetes_plugin_test", - gen_test_pipeline_with_custom_script_xml(kubernetes_test_pipeline_script()), - ) - - queue_item = job.invoke() - queue_item.block_until_complete() - - build: jenkinsapi.build.Build = queue_item.get_build() - log_stream = build.stream_logs() - logs = "".join(log_stream) - logger.debug("build logs: %s", logs) - assert build.get_status() == "SUCCESS" - - -@pytest.mark.usefixtures("k8s_agent_related_app") -async def test_pipeline_model_definition_plugin(unit_web_client: UnitWebClient): - """ - arrange: given a Jenkins charm with declarative pipeline plugin installed. - act: Run a job using a declarative pipeline script. - assert: Job succeeds. - """ - await install_plugins(unit_web_client, ("pipeline-model-definition",)) - - job = unit_web_client.client.create_job( - "pipeline_model_definition_plugin_test", - gen_test_pipeline_with_custom_script_xml(declarative_pipeline_script()), - ) - - queue_item = job.invoke() - queue_item.block_until_complete() - - build: jenkinsapi.build.Build = queue_item.get_build() - assert build.get_status() == "SUCCESS" diff --git a/tests/integration/test_plugins_part_2.py b/tests/integration/test_plugins_part_2.py new file mode 100644 index 00000000..91efaa0f --- /dev/null +++ b/tests/integration/test_plugins_part_2.py @@ -0,0 +1,266 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for jenkins-k8s-operator charm.""" + +import functools +import json +import logging + +import jenkinsapi.plugin +import pytest +import requests + +from .helpers import ( + create_kubernetes_cloud, + create_secret_file_credentials, + declarative_pipeline_script, + gen_test_job_xml, + gen_test_pipeline_with_custom_script_xml, + install_plugins, + kubernetes_test_pipeline_script, + wait_for, +) +from .types_ import KeycloakOIDCMetadata, UnitWebClient + +logger = logging.getLogger(__name__) + + +async def test_docker_build_publish_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with docker-build-publish plugin installed. + act: when a job configuration page is accessed. + assert: docker-build-publish plugin option exists. + """ + await install_plugins(unit_web_client, ("docker-build-publish",)) + unit_web_client.client.create_job("docker_plugin_test", gen_test_job_xml("k8s")) + res = unit_web_client.client.requester.get_url( + f"{unit_web_client.web}/job/docker_plugin_test/configure" + ) + config_page = str(res.content, "utf-8") + assert ( + "Docker Build and Publish" in config_page + ), f"docker-build-publish configuration option not found. {config_page}" + + +async def test_reverse_proxy_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with reverse-proxy-auth-plugin plugin installed. + act: when the security configuration is accessed. + assert: reverse-proxy-auth-plugin plugin option exists. + """ + await install_plugins(unit_web_client, ("reverse-proxy-auth-plugin",)) + + res = unit_web_client.client.requester.get_url( + f"{unit_web_client.web}/manage/configureSecurity" + ) + config_page = str(res.content, "utf-8") + + assert ( + "HTTP Header by reverse proxy" in config_page + ), f"reverse-proxy-auth-plugin configuration option not found. {config_page}" + + +async def test_dependency_check_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with dependency-check-jenkins-plugin plugin installed. + act: when a job configuration page is accessed. + assert: dependency-check-jenkins-plugin plugin option exists. + """ + await install_plugins(unit_web_client, ("dependency-check-jenkins-plugin",)) + unit_web_client.client.create_job("deps_plugin_test", gen_test_job_xml("k8s")) + res = unit_web_client.client.requester.get_url( + f"{unit_web_client.web}/job/deps_plugin_test/configure" + ) + job_page = str(res.content, "utf-8") + assert ( + "Invoke Dependency-Check" in job_page + ), f"Dependency check job configuration option not found. {job_page}" + res = unit_web_client.client.requester.get_url(f"{unit_web_client.web}/manage/configureTools/") + tools_page = str(res.content, "utf-8") + assert ( + "Dependency-Check installations" in tools_page + ), f"Dependency check tool configuration option not found. {tools_page}" + + +async def test_groovy_libs_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with pipeline-groovy-lib plugin installed. + act: when a job configuration page is accessed. + assert: pipeline-groovy-lib plugin option exists. + """ + await install_plugins(unit_web_client, ("pipeline-groovy-lib",)) + res = unit_web_client.client.requester.get_url(f"{unit_web_client.web}/manage/configure") + + config_page = str(res.content, "utf-8") + # The string is now "Global Trusted Pipeline Libraries" and + # "Global Untrusted Pipeline Libraries" for v727.ve832a_9244dfa_ + assert ( + "Pipeline Libraries" in config_page + ), f"Groovy libs configuration option not found. {config_page}" + + +@pytest.mark.usefixtures("k8s_agent_related_app") +async def test_rebuilder_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with rebuilder plugin installed. + act: when a job is built and a rebuild is triggered. + assert: last job is rebuilt. + """ + await install_plugins(unit_web_client, ("rebuild",)) + + job_name = "rebuild_test" + job = unit_web_client.client.create_job(job_name, gen_test_job_xml("k8s")) + job.invoke().block_until_complete() + + unit_web_client.client.requester.post_url( + f"{unit_web_client.web}/job/{job_name}/lastCompletedBuild/rebuild/" + ) + job.get_last_build().block_until_complete() + + assert job.get_last_buildnumber() == 2, "Rebuild not triggered." + + +async def test_openid_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with openid plugin installed. + act: when an openid endpoint is validated using the plugin. + assert: the response returns a 200 status code. + """ + await install_plugins(unit_web_client, ("openid",)) + + res = unit_web_client.client.requester.post_url( + f"{unit_web_client.web}/manage/descriptorByName/hudson.plugins.openid." + "OpenIdSsoSecurityRealm/validate", + data={"endpoint": "https://login.ubuntu.com/+openid"}, + ) + + assert res.status_code == 200, "Failed to validate openid endpoint using the plugin." + + +async def test_openid_connect_plugin( + unit_web_client: UnitWebClient, + keycloak_oidc_meta: KeycloakOIDCMetadata, + keycloak_ip: str, +): + """ + arrange: given a Jenkins charm with oic-auth plugin installed and a Keycloak oidc server. + act: + 1. when jenkins security realm is configured with oidc server and login page is requested. + 2. when jenkins security realm is reset and login page is requested. + assert: + 1. a redirection to Keycloak SSO is made. + 2. native Jenkins login ui is loaded. + """ + await install_plugins(unit_web_client, ("oic-auth",)) + + # 1. when jenkins security realm is configured with oidc server and login page is requested. + payload: dict = { + "securityRealm": { + "clientId": keycloak_oidc_meta.client_id, + "clientSecret": keycloak_oidc_meta.client_secret, + "automanualconfigure": "auto", + "serverConfiguration": { + "wellKnownOpenIDConfigurationUrl": keycloak_oidc_meta.well_known_endpoint, + "scopesOverride": "", + "stapler-class": "org.jenkinsci.plugins.oic.OicServerWellKnownConfiguration", + "$class": "org.jenkinsci.plugins.oic.OicServerWellKnownConfiguration", + }, + "userNameField": "sub", + "stapler-class": "org.jenkinsci.plugins.oic.OicSecurityRealm", + "$class": "org.jenkinsci.plugins.oic.OicSecurityRealm", + }, + "slaveAgentPort": {"type": "fixed", "value": "50000"}, + } + res = unit_web_client.client.requester.post_url( + f"{unit_web_client.web}/manage/configureSecurity/configure", + data=[ + ( + "json", + json.dumps(payload), + ), + ], + ) + res = requests.get(f"{unit_web_client.web}/securityRealm/commenceLogin?from=%2F", timeout=30) + assert res.history[0].status_code == 302, "Jenkins login not redirected." + assert keycloak_ip in res.history[0].headers["location"], "Login not redirected to keycloak." + + # 2. when jenkins security realm is reset and login page is requested. + payload = { + "securityRealm": { + "allowsSignup": False, + "stapler-class": "hudson.security.HudsonPrivateSecurityRealm", + "$class": "hudson.security.HudsonPrivateSecurityRealm", + }, + "authorizationStrategy": { + "allowAnonymousRead": False, + "stapler-class": "hudson.security.FullControlOnceLoggedInAuthorizationStrategy", + "$class": "hudson.security.FullControlOnceLoggedInAuthorizationStrategy", + }, + "slaveAgentPort": {"type": "fixed", "value": "50000"}, + } + res = unit_web_client.client.requester.post_url( + f"{unit_web_client.web}/manage/configureSecurity/configure", + data=[ + ( + "json", + json.dumps(payload), + ) + ], + ) + res = requests.get(f"{unit_web_client.web}/securityRealm/commenceLogin?from=%2F", timeout=30) + assert res.status_code == 404, "Security realm login not reset." + res = requests.get(f"{unit_web_client.web}/login?from=%2F", timeout=30) + assert res.status_code == 200, "Failed to load Jenkins native login UI." + + +async def test_kubernetes_plugin(unit_web_client: UnitWebClient, kube_config: str): + """ + arrange: given a Jenkins charm with kubernetes plugin installed and credentials from microk8s. + act: Run a job using an agent provided by the kubernetes plugin. + assert: Job succeeds. + """ + # Use plain credentials to be able to create secret-file/secret-text credentials + await install_plugins(unit_web_client, ("kubernetes", "plain-credentials")) + credentials_id = await wait_for( + functools.partial(create_secret_file_credentials, unit_web_client, kube_config) + ) + assert credentials_id, "Failed to create credentials id" + kubernetes_cloud_name = await wait_for( + functools.partial(create_kubernetes_cloud, unit_web_client, credentials_id) + ) + assert kubernetes_cloud_name, "Failed to create kubernetes cloud" + job = unit_web_client.client.create_job( + "kubernetes_plugin_test", + gen_test_pipeline_with_custom_script_xml(kubernetes_test_pipeline_script()), + ) + + queue_item = job.invoke() + queue_item.block_until_complete() + + build: jenkinsapi.build.Build = queue_item.get_build() + log_stream = build.stream_logs() + logs = "".join(log_stream) + logger.debug("build logs: %s", logs) + assert build.get_status() == "SUCCESS" + + +@pytest.mark.usefixtures("k8s_agent_related_app") +async def test_pipeline_model_definition_plugin(unit_web_client: UnitWebClient): + """ + arrange: given a Jenkins charm with declarative pipeline plugin installed. + act: Run a job using a declarative pipeline script. + assert: Job succeeds. + """ + await install_plugins(unit_web_client, ("pipeline-model-definition",)) + + job = unit_web_client.client.create_job( + "pipeline_model_definition_plugin_test", + gen_test_pipeline_with_custom_script_xml(declarative_pipeline_script()), + ) + + queue_item = job.invoke() + queue_item.block_until_complete() + + build: jenkinsapi.build.Build = queue_item.get_build() + assert build.get_status() == "SUCCESS" From 41c39e612965a96007a1f67a3b364919b6e06ba2 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sun, 1 Dec 2024 05:17:44 +0000 Subject: [PATCH 27/29] try 1.29 microk8s --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index fd1c4551..8eb9f944 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,7 +12,7 @@ jobs: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: - channel: 1.28-strict/stable + channel: 1.29-strict/stable extra-arguments: | --kube-config=${GITHUB_WORKSPACE}/kube-config modules: '["test_auth_proxy.py", "test_cos.py", "test_ingress.py", "test_jenkins.py", "test_k8s_agent.py", "test_machine_agent.py", "test_plugins_part_1.py", "test_plugins_part_2.py", "test_proxy.py", "test_upgrade.py", "test_external_agent.py"]' @@ -23,4 +23,4 @@ jobs: juju-channel: 3/stable self-hosted-runner: true self-hosted-runner-label: "xlarge" - microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" + microk8s-addons: "dns ingress rbac hostpath-storage metallb:10.15.119.2-10.15.119.4 registry" From f4759cbe00a5d580b111de3d3fcdc2a192b6a25d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sun, 1 Dec 2024 07:31:45 +0000 Subject: [PATCH 28/29] debug --- .github/workflows/integration_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8eb9f944..9595677c 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,3 +24,5 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac hostpath-storage metallb:10.15.119.2-10.15.119.4 registry" + tmate-debug: true + tmate-timeout: 90 From 7d6f083b38cf1855a05e17ba1e1e6cf19fb40b25 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sun, 1 Dec 2024 08:00:57 +0000 Subject: [PATCH 29/29] kubeconfig dir & debug log --- tests/integration/conftest.py | 2 +- tests/integration/dex.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2fe4149c..b06a43f4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -41,7 +41,7 @@ from .helpers import generate_jenkins_client_from_application, get_pod_ip from .types_ import KeycloakOIDCMetadata, LDAPSettings, ModelAppUnit, UnitWebClient -KUBECONFIG = os.environ.get("TESTING_KUBECONFIG", "~/.kube/config") +KUBECONFIG = os.environ.get("TESTING_KUBECONFIG", "./kube-config") IDENTITY_PLATFORM_APPS = [ "traefik-admin", "traefik-public", diff --git a/tests/integration/dex.py b/tests/integration/dex.py index eaadd414..c7256b76 100644 --- a/tests/integration/dex.py +++ b/tests/integration/dex.py @@ -133,6 +133,7 @@ def _apply_dex_manifests( ) for obj in objs: + logger.info("Applying dex manifest: %s", obj) client.apply(obj, force=True)