Merge pull request #123 from DSD-DBS/fix-t4c-backup-copying

Unmess images and fragments during backups

Makefile

@@ -11,7 +11,7 @@ T4C_LICENCE_SECRET ?= XXX
 T4C_REPOSITORIES ?= testrepo
 
 # T4C server host
-T4C_SERVER_HOST ?= localhost
+T4C_SERVER_HOST ?= host.docker.internal
 
 # T4C server port
 T4C_SERVER_PORT ?= 2036
@@ -31,6 +31,9 @@ HTTP_LOGIN ?= admin
 # T4C http password
 HTTP_PASSWORD ?= password
 
+# Connection type to T4C server
+CONNECTION_TYPE ?= telnet
+
 # Remote container rdp password
 RMT_PASSWORD ?= tmp_passwd2
 
@@ -313,8 +316,7 @@ run-t4c/client/remote/pure-variants: t4c/client/remote/pure-variants
 		$(DOCKER_PREFIX)t4c/client/remote/pure-variants:$(DOCKER_TAG)
 
 run-t4c/client/backup: t4c/client/backup
-	docker run $(DOCKER_RUN_FLAGS) \
-		--network="host" \
+	docker run $(DOCKER_RUN_FLAGS) --rm -it \
 		-e GIT_REPO_URL="$(GIT_REPO_URL)" \
 		-e GIT_REPO_BRANCH="$(GIT_REPO_BRANCH)" \
 		-e T4C_REPO_HOST="$(T4C_SERVER_HOST)" \
@@ -325,7 +327,11 @@ run-t4c/client/backup: t4c/client/backup
 		-e T4C_PASSWORD="$(T4C_PASSWORD)" \
 		-e GIT_USERNAME="$(GIT_USERNAME)" \
 		-e GIT_PASSWORD="$(GIT_PASSWORD)" \
+		-e HTTP_LOGIN="$(HTTP_LOGIN)" \
+		-e HTTP_PASSWORD="$(HTTP_PASSWORD)" \
+		-e HTTP_PORT="$(HTTP_PORT)" \
 		-e LOG_LEVEL="$(LOG_LEVEL)" \
+		-e CONNECTION_TYPE="$(CONNECTION_TYPE)" \
 		$(DOCKER_PREFIX)t4c/client/backup:$(DOCKER_TAG)
 
 run-t4c/client/exporter: t4c/client/exporter

backups/Dockerfile

@@ -23,14 +23,16 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 COPY git_askpass.py /etc/git_askpass.py
-COPY backup.py /opt/scripts/backup.py
+RUN chmod 555 /etc/git_askpass.py
 
 ENV DISPLAY :99
 
 WORKDIR /opt/capella
 
 RUN chown -R techuser /opt/capella
 
+COPY backup.py /opt/scripts/backup.py
+
 USER techuser
 
 ENTRYPOINT [ "/tini", "--", "xvfb-run", "python", "/opt/scripts/backup.py" ]

backups/backup.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import mimetypes
 import os
 import pathlib
 import re
@@ -73,27 +74,51 @@ def run_importer_script() -> None:
             http_port,
         ]
 
-    with subprocess.Popen(
-        command, cwd="/opt/capella", stdout=subprocess.PIPE, text=True
-    ) as popen:
-        if popen.stdout:
-            for line in popen.stdout:
-                print(line, end="", flush=True)
-                if (
-                    "Team for Capella server unreachable" in line
-                    or "Name or service not known" in line
-                ):
-                    raise RuntimeError(
-                        f"{ERROR_PREFIX} - Team for Capella server unreachable"
-                    )
-                elif "1 copies failed" in line:
-                    raise RuntimeError(
-                        f"{ERROR_PREFIX} - Failed to copy to output folder ({OUTPUT_FOLDER})"
-                    )
+    stdout = ""
+    try:
+        popen = subprocess.Popen(
+            command, cwd="/opt/capella", stdout=subprocess.PIPE, text=True
+        )
+        assert popen.stdout
+        for line in popen.stdout:
+            stdout += line
+            print(line, end="")
+
+            # Failed imports still have exit code 0.
+            # In addition, the process hangs up when these log lines appear sometimes.
+            # This covers some of the error cases we experienced.
+
+            if (
+                "Team for Capella server unreachable" in line
+                or "Name or service not known" in line
+            ):
+                raise RuntimeError(
+                    f"{ERROR_PREFIX} - Team for Capella server unreachable"
+                )
+            if re.search(r"[1-9][0-9]* projects imports failed", line):
+                raise RuntimeError(
+                    f"{ERROR_PREFIX} - Backup failed. Please check the logs above."
+                )
+            if re.search(r"[1-9][0-9]* copies failed", line):
+                raise RuntimeError(
+                    f"{ERROR_PREFIX} - Failed to copy to output folder ({OUTPUT_FOLDER})"
+                )
+    finally:
+        if popen:
+            popen.terminate()
 
     if (return_code := popen.returncode) != 0:
         raise subprocess.CalledProcessError(return_code, command)
 
+    if not re.search(r"[1-9][0-9]* projects imports succeeded", stdout):
+        raise RuntimeError(
+            f"{ERROR_PREFIX} - '[1-9][0-9]* projects imports succeeded' not found in logs"
+        )
+    if not re.search(r"[1-9][0-9]* copies succeeded", stdout):
+        raise RuntimeError(
+            f"{ERROR_PREFIX} - '[1-9][0-9]* copies succeeded' not found in logs"
+        )
+
     log.info("Import of model from TeamForCapella server finished")
 
 
@@ -132,7 +157,34 @@ def checkout_git_repository() -> pathlib.Path:
     return git_dir
 
 
-def copy_exported_files_into_git_repo(model_dir: pathlib.Path) -> None:
+def unmess_file_structure(model_dir: pathlib.Path) -> None:
+    """Sort images and airdfragments into folders
+
+    The importer application of TeamForCapella places all files into the project directory,
+    without sorting into the images and fragments directories.
+    This function sorts '*.airdfragment' files into the fragments directory.
+    Images with mimetype 'image/*' are copied into the images directory.
+    """
+
+    log.info("Start unmessing files...")
+
+    (model_dir / "images").mkdir(exist_ok=True)
+    (model_dir / "fragments").mkdir(exist_ok=True)
+
+    for file in model_dir.iterdir():
+        if file.is_file():
+            mimetype = mimetypes.guess_type(file)[0]
+            if file.suffix in (".airdfragment", ".capellafragment"):
+                file.rename(model_dir / "fragments" / file.name)
+            elif mimetype and mimetype.startswith("image/"):
+                file.rename(model_dir / "images" / file.name)
+
+    log.info("Finished unmessing files...")
+
+
+def copy_exported_files_into_git_repo(project_dir: pathlib.Path) -> None:
+    log.info("Start copying files...")
+
     if entrypoint := os.getenv("GIT_REPO_ENTRYPOINT", None):
         target_directory = pathlib.Path(
             "/tmp/git", str(pathlib.Path(entrypoint).parent).lstrip("/")
@@ -143,19 +195,29 @@ def copy_exported_files_into_git_repo(model_dir: pathlib.Path) -> None:
             "/tmp/git",
         )
 
-    for file in model_dir.glob("*/*"):
-        shutil.copy2(file, target_directory)
+    model_dir = project_dir / urllib.parse.quote(
+        os.environ["T4C_PROJECT_NAME"]
+    )
+    unmess_file_structure(model_dir)
+
+    shutil.copytree(
+        model_dir,
+        target_directory,
+        dirs_exist_ok=True,
+    )
 
     if os.getenv("INCLUDE_COMMIT_HISTORY", "true") == "true":
         shutil.copy2(
-            next(model_dir.glob("CommitHistory__*.activitymetadata")),
+            next(project_dir.glob("CommitHistory__*.activitymetadata")),
             target_directory / "CommitHistory.activitymetadata",
         )
         shutil.copy2(
-            next(model_dir.glob("CommitHistory__*.json*")),
+            next(project_dir.glob("CommitHistory__*.json*")),
             target_directory / "CommitHistory.json",
         )
 
+    log.info("Finished copying files...")
+
 
 def git_commit_and_push(git_dir: pathlib.Path) -> None:
     subprocess.run(
@@ -241,8 +303,8 @@ def get_http_envs() -> tuple[str, str, str]:
 
 
 if __name__ == "__main__":
-    _model_dir = pathlib.Path(OUTPUT_FOLDER)
-    _model_dir.mkdir(exist_ok=True)
+    _project_dir = pathlib.Path(OUTPUT_FOLDER)
+    _project_dir.mkdir(exist_ok=True)
 
     run_importer_script()
 
@@ -251,7 +313,7 @@ def get_http_envs() -> tuple[str, str, str]:
         pass
     else:  # USE GIT
         _git_dir = checkout_git_repository()
-        copy_exported_files_into_git_repo(_model_dir)
+        copy_exported_files_into_git_repo(_project_dir)
         git_commit_and_push(_git_dir)
 
     log.info("Import of model from TeamForCapella server finished")