From 0fefe76a21422799124424f1d12f956d13346dd2 Mon Sep 17 00:00:00 2001 From: MoritzWeber Date: Mon, 27 Feb 2023 14:48:26 +0100 Subject: [PATCH 1/3] feat: Improve error handling for backups image --- Makefile | 12 +++++++--- backups/Dockerfile | 4 +++- backups/backup.py | 58 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index e8589b60..ae2d35b2 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ T4C_LICENCE_SECRET ?= XXX T4C_REPOSITORIES ?= testrepo # T4C server host -T4C_SERVER_HOST ?= localhost +T4C_SERVER_HOST ?= host.docker.internal # T4C server port T4C_SERVER_PORT ?= 2036 @@ -31,6 +31,9 @@ HTTP_LOGIN ?= admin # T4C http password HTTP_PASSWORD ?= password +# Connection type to T4C server +CONNECTION_TYPE ?= telnet + # Remote container rdp password RMT_PASSWORD ?= tmp_passwd2 @@ -307,8 +310,7 @@ run-t4c/client/remote/pure-variants: t4c/client/remote/pure-variants $(DOCKER_PREFIX)t4c/client/remote/pure-variants:$(DOCKER_TAG) run-t4c/client/backup: t4c/client/backup - docker run $(DOCKER_RUN_FLAGS) \ - --network="host" \ + docker run $(DOCKER_RUN_FLAGS) --rm -it \ -e GIT_REPO_URL="$(GIT_REPO_URL)" \ -e GIT_REPO_BRANCH="$(GIT_REPO_BRANCH)" \ -e T4C_REPO_HOST="$(T4C_SERVER_HOST)" \ @@ -319,7 +321,11 @@ run-t4c/client/backup: t4c/client/backup -e T4C_PASSWORD="$(T4C_PASSWORD)" \ -e GIT_USERNAME="$(GIT_USERNAME)" \ -e GIT_PASSWORD="$(GIT_PASSWORD)" \ + -e HTTP_LOGIN="$(HTTP_LOGIN)" \ + -e HTTP_PASSWORD="$(HTTP_PASSWORD)" \ + -e HTTP_PORT="$(HTTP_PORT)" \ -e LOG_LEVEL="$(LOG_LEVEL)" \ + -e CONNECTION_TYPE="$(CONNECTION_TYPE)" \ $(DOCKER_PREFIX)t4c/client/backup:$(DOCKER_TAG) run-t4c/client/exporter: t4c/client/exporter diff --git a/backups/Dockerfile b/backups/Dockerfile index bc38393d..e0bbe32b 100644 --- a/backups/Dockerfile +++ b/backups/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* COPY git_askpass.py /etc/git_askpass.py -COPY backup.py /opt/scripts/backup.py +RUN chmod 555 /etc/git_askpass.py ENV DISPLAY :99 @@ -31,6 +31,8 @@ WORKDIR /opt/capella RUN chown -R techuser /opt/capella +COPY backup.py /opt/scripts/backup.py + USER techuser ENTRYPOINT [ "/tini", "--", "xvfb-run", "python", "/opt/scripts/backup.py" ] diff --git a/backups/backup.py b/backups/backup.py index 416ad1df..971f803b 100644 --- a/backups/backup.py +++ b/backups/backup.py @@ -73,27 +73,51 @@ def run_importer_script() -> None: http_port, ] - with subprocess.Popen( - command, cwd="/opt/capella", stdout=subprocess.PIPE, text=True - ) as popen: - if popen.stdout: - for line in popen.stdout: - print(line, end="", flush=True) - if ( - "Team for Capella server unreachable" in line - or "Name or service not known" in line - ): - raise RuntimeError( - f"{ERROR_PREFIX} - Team for Capella server unreachable" - ) - elif "1 copies failed" in line: - raise RuntimeError( - f"{ERROR_PREFIX} - Failed to copy to output folder ({OUTPUT_FOLDER})" - ) + stdout = "" + try: + popen = subprocess.Popen( + command, cwd="/opt/capella", stdout=subprocess.PIPE, text=True + ) + assert popen.stdout + for line in popen.stdout: + stdout += line + print(line, end="") + + # Failed imports still have exit code 0. + # In addition, the process hangs up when these log lines appear sometimes. + # This covers some of the error cases we experienced. + + if ( + "Team for Capella server unreachable" in line + or "Name or service not known" in line + ): + raise RuntimeError( + f"{ERROR_PREFIX} - Team for Capella server unreachable" + ) + if re.search(r"[1-9][0-9]* projects imports failed", line): + raise RuntimeError( + f"{ERROR_PREFIX} - Backup failed. Please check the logs above." + ) + if re.search(r"[1-9][0-9]* copies failed", line): + raise RuntimeError( + f"{ERROR_PREFIX} - Failed to copy to output folder ({OUTPUT_FOLDER})" + ) + finally: + if popen: + popen.terminate() if (return_code := popen.returncode) != 0: raise subprocess.CalledProcessError(return_code, command) + if not re.search(r"[1-9][0-9]* projects imports succeeded", stdout): + raise RuntimeError( + f"{ERROR_PREFIX} - '[1-9][0-9]* projects imports succeeded' not found in logs" + ) + if not re.search(r"[1-9][0-9]* copies succeeded", stdout): + raise RuntimeError( + f"{ERROR_PREFIX} - '[1-9][0-9]* copies succeeded' not found in logs" + ) + log.info("Import of model from TeamForCapella server finished") From f765de3e455c298477758e3d974408dfc1002f83 Mon Sep 17 00:00:00 2001 From: MoritzWeber Date: Mon, 27 Feb 2023 14:50:45 +0100 Subject: [PATCH 2/3] refactor: Use `shutil.copytree` instead of custom copy --- backups/backup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backups/backup.py b/backups/backup.py index 971f803b..df1b8c36 100644 --- a/backups/backup.py +++ b/backups/backup.py @@ -167,8 +167,11 @@ def copy_exported_files_into_git_repo(model_dir: pathlib.Path) -> None: "/tmp/git", ) - for file in model_dir.glob("*/*"): - shutil.copy2(file, target_directory) + shutil.copytree( + model_dir / urllib.parse.quote(os.environ["T4C_PROJECT_NAME"]), + target_directory, + dirs_exist_ok=True, + ) if os.getenv("INCLUDE_COMMIT_HISTORY", "true") == "true": shutil.copy2( From ea1ba025b26bdc18de499bd2bcf4a3e091b60dcc Mon Sep 17 00:00:00 2001 From: MoritzWeber Date: Tue, 28 Feb 2023 09:23:15 +0100 Subject: [PATCH 3/3] feat: Unmess file structure for backup --- backups/backup.py | 49 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/backups/backup.py b/backups/backup.py index df1b8c36..9b4626fe 100644 --- a/backups/backup.py +++ b/backups/backup.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +import mimetypes import os import pathlib import re @@ -156,7 +157,34 @@ def checkout_git_repository() -> pathlib.Path: return git_dir -def copy_exported_files_into_git_repo(model_dir: pathlib.Path) -> None: +def unmess_file_structure(model_dir: pathlib.Path) -> None: + """Sort images and airdfragments into folders + + The importer application of TeamForCapella places all files into the project directory, + without sorting into the images and fragments directories. + This function sorts '*.airdfragment' files into the fragments directory. + Images with mimetype 'image/*' are copied into the images directory. + """ + + log.info("Start unmessing files...") + + (model_dir / "images").mkdir(exist_ok=True) + (model_dir / "fragments").mkdir(exist_ok=True) + + for file in model_dir.iterdir(): + if file.is_file(): + mimetype = mimetypes.guess_type(file)[0] + if file.suffix in (".airdfragment", ".capellafragment"): + file.rename(model_dir / "fragments" / file.name) + elif mimetype and mimetype.startswith("image/"): + file.rename(model_dir / "images" / file.name) + + log.info("Finished unmessing files...") + + +def copy_exported_files_into_git_repo(project_dir: pathlib.Path) -> None: + log.info("Start copying files...") + if entrypoint := os.getenv("GIT_REPO_ENTRYPOINT", None): target_directory = pathlib.Path( "/tmp/git", str(pathlib.Path(entrypoint).parent).lstrip("/") @@ -167,22 +195,29 @@ def copy_exported_files_into_git_repo(model_dir: pathlib.Path) -> None: "/tmp/git", ) + model_dir = project_dir / urllib.parse.quote( + os.environ["T4C_PROJECT_NAME"] + ) + unmess_file_structure(model_dir) + shutil.copytree( - model_dir / urllib.parse.quote(os.environ["T4C_PROJECT_NAME"]), + model_dir, target_directory, dirs_exist_ok=True, ) if os.getenv("INCLUDE_COMMIT_HISTORY", "true") == "true": shutil.copy2( - next(model_dir.glob("CommitHistory__*.activitymetadata")), + next(project_dir.glob("CommitHistory__*.activitymetadata")), target_directory / "CommitHistory.activitymetadata", ) shutil.copy2( - next(model_dir.glob("CommitHistory__*.json*")), + next(project_dir.glob("CommitHistory__*.json*")), target_directory / "CommitHistory.json", ) + log.info("Finished copying files...") + def git_commit_and_push(git_dir: pathlib.Path) -> None: subprocess.run( @@ -268,8 +303,8 @@ def get_http_envs() -> tuple[str, str, str]: if __name__ == "__main__": - _model_dir = pathlib.Path(OUTPUT_FOLDER) - _model_dir.mkdir(exist_ok=True) + _project_dir = pathlib.Path(OUTPUT_FOLDER) + _project_dir.mkdir(exist_ok=True) run_importer_script() @@ -278,7 +313,7 @@ def get_http_envs() -> tuple[str, str, str]: pass else: # USE GIT _git_dir = checkout_git_repository() - copy_exported_files_into_git_repo(_model_dir) + copy_exported_files_into_git_repo(_project_dir) git_commit_and_push(_git_dir) log.info("Import of model from TeamForCapella server finished")