Merge pull request #200 from chaen/cs_conversion_improv

Improve CS sync

diracx-cli/src/diracx/cli/internal/legacy.py

@@ -3,12 +3,16 @@
 import json
 import os
 from pathlib import Path
-from typing import cast
+from typing import TYPE_CHECKING, cast
 from urllib.parse import urljoin, urlparse
 
 import diraccfg
 import typer
 import yaml
+
+if TYPE_CHECKING:
+    from diraccfg.cfg import CFGAsDict
+
 from pydantic import BaseModel
 from typer import Option
 
@@ -48,7 +52,7 @@ class ConversionConfig(BaseModel):
 
 
 @app.command()
-def cs_sync(old_file: Path, conversion_config: Path, new_file: Path):
+def cs_sync(old_file: Path, new_file: Path):
     """Load the old CS and convert it to the new YAML format"""
     if not os.environ.get("DIRAC_COMPAT_ENABLE_CS_CONVERSION"):
         raise RuntimeError(
@@ -59,18 +63,30 @@ def cs_sync(old_file: Path, conversion_config: Path, new_file: Path):
     cfg = diraccfg.CFG().loadFromBuffer(old_data)
     raw = cfg.getAsDict()
 
-    _apply_fixes(raw, conversion_config)
+    diracx_section = cast("CFGAsDict", raw["DiracX"])
+    # DisabledVOs cannot be set if any Legacy clients are enabled
+    disabled_vos = diracx_section.get("DisabledVOs")
+    enabled_clients = []
+    for _, client_status in cast(
+        "CFGAsDict", diracx_section.get("LegacyClientEnabled", {})
+    ).items():
+        for _, str_status in cast("CFGAsDict", client_status).items():
+            enabled_clients.append(str_status == "True")
+    if disabled_vos and any(enabled_clients):
+        raise RuntimeError(
+            "DisabledVOs cannot be set if any Legacy clients are enabled"
+        )
+
+    _apply_fixes(raw)
 
     config = Config.parse_obj(raw)
     new_file.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
 
 
-def _apply_fixes(raw, conversion_config: Path):
+def _apply_fixes(raw):
     """Modify raw in place to make any layout changes between the old and new structure"""
 
-    conv_config = ConversionConfig.parse_obj(
-        yaml.safe_load(conversion_config.read_text())
-    )
+    conv_config = ConversionConfig.parse_obj(raw["DiracX"]["CsSync"])
 
     raw.pop("DiracX", None)
     # Remove dips specific parts from the CS

diracx-cli/tests/legacy/cs_sync/integration_test.cfg

@@ -30,6 +30,49 @@ DiracX
 {
   URL = https://dirac-integration-tests.invalid:1234
   LegacyExchangeApiKey = diracx:legacy:ChangeME
+
+  CsSync
+  {
+    VOs
+    {
+      Jenkins
+      {
+        DefaultGroup = jenkins_user
+        IdP
+        {
+          ClientID = 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd
+          URL = https://jenkins.invalid/
+        }
+        UserSubjects
+        {
+          adminusername = e2cb28ec-1a1e-40ee-a56d-d899b79879ce
+          ciuser = 26dbe36e-cf5c-4c52-a834-29a1c904ef74
+          trialUser = a95ab678-3fa4-41b9-b863-fe62ce8064ce
+        }
+        Support
+        {
+          Message = Contact the help desk
+          Email = [email protected]
+          Webpage = https://helpdesk.vo.invalid
+        }
+      }
+      vo
+      {
+        DefaultGroup = dirac_user
+        IdP
+        {
+          ClientID = 072afab5-ed92-46e0-a61d-4ecbc96e0770
+          URL = https://vo.invalid/
+        }
+        UserSubjects
+        {
+          adminusername = 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e
+          ciuser = d3adc733-6588-4d6f-8581-5986b02d0c87
+          trialUser = ff2152ff-34f4-4739-b106-3def37e291e3
+        }
+      }
+    }
+  }
 }
 Systems
 {

diracx-cli/tests/legacy/cs_sync/integration_test_buggy.cfg

@@ -0,0 +1,17 @@
+
+DiracX
+{
+  URL = https://dirac-integration-tests.invalid:1234
+  LegacyExchangeApiKey = diracx:legacy:ChangeME
+
+  # # DisabledVOs cannot be set if any Legacy clients are enabled
+  DisabledVOs = AnyVO
+  LegacyClientEnabled
+  {
+    WorkloadManagement
+    {
+      JobStateUpdate = True
+    }
+  }
+
+}

diracx-cli/tests/legacy/cs_sync/test_cssync.py

@@ -23,7 +23,6 @@ def test_cs_sync(tmp_path, monkeypatch):
             "legacy",
             "cs-sync",
             f"{file_path / 'integration_test.cfg'}",
-            f"{file_path / 'convert_integration_test.yaml'}",
             str(output_file),
         ],
     )
@@ -33,3 +32,25 @@ def test_cs_sync(tmp_path, monkeypatch):
     expected_output = yaml.safe_load((file_path / "integration_test.yaml").read_text())
     assert actual_output == expected_output
     Config.parse_obj(actual_output)
+
+
+def test_disabled_vos_empty(tmp_path, monkeypatch):
+    # # DisabledVOs cannot be set if any Legacy clients are enabled
+    monkeypatch.setenv("DIRAC_COMPAT_ENABLE_CS_CONVERSION", "Yes")
+
+    output_file = tmp_path / "default.yaml"
+
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "legacy",
+            "cs-sync",
+            f"{file_path / 'integration_test_buggy.cfg'}",
+            str(output_file),
+        ],
+    )
+    assert result.exit_code == 1
+    assert not output_file.is_file()
+    assert isinstance(result.exception, RuntimeError)
+    assert "DisabledVOs cannot be set" in str(result.exception)

diracx-testing/src/diracx/testing/__init__.py

@@ -9,7 +9,7 @@
 from html.parser import HTMLParser
 from pathlib import Path
 from typing import TYPE_CHECKING
-from urllib.parse import urljoin
+from urllib.parse import parse_qs, urljoin, urlparse
 from uuid import uuid4
 
 import pytest
@@ -551,5 +551,13 @@ def handle_starttag(self, tag, attrs):
         verify=ca_path,
     )
     r.raise_for_status()
+    approval_url = r.url  # This is not the same as URL as we redirect to dex
+    # Do the actual approval
+    r = requests.post(
+        approval_url,
+        {"approval": "approve", "req": parse_qs(urlparse(r.url).query)["req"][0]},
+        verify=ca_path,
+    )
+
     # This should have redirected to the DiracX page that shows the login is complete
     assert "Please close the window" in r.text

docs/MIGRATION.md

@@ -0,0 +1,20 @@
+
+Start from DIRAC v8
+
+Edit the CS:
+* DiracX/DisabledVO is all of them
+* DiracX/LegacyClientEnabled is empty
+
+Add in the CS:
+* conversion info
+
+
+Update to v9
+
+Create a DiracX cluster
+
+Run the CS Sync
+
+slowly work through the Disabled VO
+
+Once it's empty, start enabling LegacyClient

run_local.sh

@@ -19,12 +19,8 @@ dirac internal add-vo "${tmp_dir}/cs_store/initialRepo" \
     --idp-client-id="idp-client-id" \
     --default-group=admin
 
-dirac internal add-group "${tmp_dir}/cs_store/initialRepo" \
-    --vo=diracAdmin \
-    --group=admin
-
 dirac internal add-user "${tmp_dir}/cs_store/initialRepo" \
-  --vo=diracAdmin --user-group=admin \
+  --vo=diracAdmin --group=admin \
   --sub=75212b23-14c2-47be-9374-eb0113b0575e \
   --preferred-username=localuser