diff --git a/.gitignore b/.gitignore index 4397c05d..86eab8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ prometheus* volumes *.egg-info/ .env + +**/set_memory_flags.py +!eclipse/set_memory_flags.py diff --git a/Makefile b/Makefile index e6e2a081..f9f4ab30 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,10 @@ DOCKER_REGISTRY ?= localhost:12345 # Log level when running Docker containers LOG_LEVEL ?= DEBUG +# Set memory options for the TeamForCapella server +MEMORY_MAX ?= 90% +MEMORY_MIN ?= 70% + # If this option is set to 1, all tests that require a running t4c server # will be executed. To run these tests, you need a Makefile in # t4c/server with a target t4c/server/server that builds the t4c server @@ -181,6 +185,7 @@ jupyter-notebook: base capella/base: SHELL=./capella_loop.sh capella/base: base envsubst < capella/.dockerignore.template > capella/.dockerignore + cp eclipse/set_memory_flags.py capella/setup/set_memory_flags.py docker build $(DOCKER_BUILD_FLAGS) \ -t $(DOCKER_PREFIX)$@:$$DOCKER_TAG \ --build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \ @@ -192,16 +197,19 @@ capella/base: base --build-arg INSTALL_OLD_GTK_VERSION=$(INSTALL_OLD_GTK_VERSION) \ capella rm capella/.dockerignore + rm capella/setup/set_memory_flags.py $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) IMAGENAME=$@ .push papyrus/base: DOCKER_TAG=$(PAPYRUS_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION) papyrus/base: DOCKER_BUILD_FLAGS=--platform linux/amd64 papyrus/base: base + cp eclipse/set_memory_flags.py papyrus/set_memory_flags.py docker build $(DOCKER_BUILD_FLAGS) \ -t $(DOCKER_PREFIX)$@:$(DOCKER_TAG) \ --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$(CAPELLA_DOCKERIMAGES_REVISION) \ --build-arg PAPYRUS_VERSION=$(PAPYRUS_VERSION) \ papyrus + rm papyrus/set_memory_flags.py $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) DOCKER_TAG=$(DOCKER_TAG) IMAGENAME=$@ .push eclipse/base: DOCKER_TAG=$(ECLIPSE_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION) @@ -299,6 +307,8 @@ run-capella/remote: capella/remote -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e AUTOSTART_CAPELLA=$(AUTOSTART_CAPELLA) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ @@ -312,6 +322,8 @@ run-papyrus/remote: papyrus/remote -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ -e WORKSPACE_DIR=/workspace/papyrus \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ @@ -324,6 +336,8 @@ run-eclipse/remote: eclipse/remote -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ -e WORKSPACE_DIR=/workspace/eclipse \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ @@ -338,6 +352,8 @@ run-eclipse/remote/pure-variants: eclipse/remote/pure-variants -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ -e WORKSPACE_DIR=/workspace/eclipse_pv \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -v $$(pwd)/volumes/pure-variants:/inputs/pure-variants \ -v $$(pwd)/volumes/workspace:/workspace \ -v $$(pwd)/pure-variants/versions:/opt/versions \ @@ -358,6 +374,8 @@ run-t4c/client/remote-legacy: t4c/client/remote -e T4C_USERNAME=$(T4C_USERNAME) \ -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ @@ -372,6 +390,8 @@ run-t4c/client/remote: t4c/client/remote -e T4C_USERNAME=$(T4C_USERNAME) \ -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ + -e MEMORY_MIN=$(MEMORY_MIN) \ + -e MEMORY_MAX=$(MEMORY_MAX) \ -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ @@ -379,14 +399,14 @@ run-t4c/client/remote: t4c/client/remote run-t4c/client/remote/pure-variants: t4c/client/remote/pure-variants docker run $(DOCKER_RUN_FLAGS) \ + -v $$(pwd)/volumes/pure-variants:/inputs/pure-variants \ + -v $$(pwd)/volumes/workspace:/workspace \ -e T4C_LICENCE_SECRET=$(T4C_LICENCE_SECRET) \ -e T4C_JSON=$(T4C_JSON) \ -e RMT_PASSWORD=$(RMT_PASSWORD) \ -e T4C_USERNAME=$(T4C_USERNAME) \ -e PURE_VARIANTS_LICENSE_SERVER=$(PURE_VARIANTS_LICENSE_SERVER) \ -e PURE_VARIANTS_KNOWN_SERVERS=$(PURE_VARIANTS_KNOWN_SERVERS) \ - -v $$(pwd)/volumes/pure-variants:/inputs/pure-variants \ - -v $$(pwd)/volumes/workspace:/workspace \ -e AUTOSTART_CAPELLA=$(AUTOSTART_CAPELLA) \ -e CONNECTION_METHOD=$(CONNECTION_METHOD) \ -e XPRA_SUBPATH=$(XPRA_SUBPATH) \ diff --git a/capella/Dockerfile b/capella/Dockerfile index 554fa313..b3945406 100644 --- a/capella/Dockerfile +++ b/capella/Dockerfile @@ -142,7 +142,7 @@ RUN mkdir -p /opt/capella/configuration/.settings; \ chown techuser /opt /opt/capella/capella.ini RUN echo '-Dosgi.configuration.area=file:/opt/capella/configuration' >> /opt/capella/capella.ini -COPY startup/* /opt/setup/ +COPY setup/* /opt/setup/ ENV AUTOSTART_CAPELLA=1 ENV RESTART_CAPELLA=1 @@ -151,6 +151,12 @@ COPY ./autostart /home/techuser/.config/openbox/autostart ENV ECLIPSE_INSTALLATION_PATH=/opt/capella ENV ECLIPSE_EXECUTABLE=/opt/capella/capella +# Set memory options for the JVM (used by set_memory_flags.py) +ARG MEMORY_MAX=90% +ENV MEMORY_MAX=${MEMORY_MAX} +ARG MEMORY_MIN=70% +ENV MEMORY_MIN=${MEMORY_MIN} + COPY ./autostart /home/techuser/.config/openbox/autostart COPY git_askpass.py /etc/git_askpass.py diff --git a/capella/setup/README.md b/capella/setup/README.md new file mode 100644 index 00000000..1d7f876b --- /dev/null +++ b/capella/setup/README.md @@ -0,0 +1,9 @@ + + +# Directory for startup scripts + +This directory contains scripts that are executed during container startup. As +of now, we support scripts with file-ending `.sh` and `.py`. diff --git a/capella/startup/provisioning.py b/capella/setup/provisioning.py similarity index 100% rename from capella/startup/provisioning.py rename to capella/setup/provisioning.py diff --git a/capella/startup/replace_env_variables.sh b/capella/setup/replace_env_variables.sh similarity index 100% rename from capella/startup/replace_env_variables.sh rename to capella/setup/replace_env_variables.sh diff --git a/capella/startup/setup_workspace.py b/capella/setup/setup_workspace.py similarity index 100% rename from capella/startup/setup_workspace.py rename to capella/setup/setup_workspace.py diff --git a/capella/startup.sh b/capella/startup.sh index 5da4543e..93c0626d 100755 --- a/capella/startup.sh +++ b/capella/startup.sh @@ -3,5 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 export DISPLAY=":99" +python3 /opt/setup/set_memory_flags.py (Xvfb ${DISPLAY} -screen 0 1920x1080x8 -nolisten tcp 0>/dev/null 2>&1 &) /opt/capella/capella "$@" diff --git a/ci-templates/gitlab/image-builder.yml b/ci-templates/gitlab/image-builder.yml index e1874cdd..7710e365 100644 --- a/ci-templates/gitlab/image-builder.yml +++ b/ci-templates/gitlab/image-builder.yml @@ -221,6 +221,7 @@ capella/base: - *docker # prettier-ignore - mv ../capella.tar.gz ./capella/versions/$CAPELLA_VERSION/$BUILD_ARCHITECTURE/capella.tar.gz + - cp ./eclipse/set_memory_flags.py ./capella/setup/set_memory_flags.py - > if [[ -n "$(find ../dropins -maxdepth 1 -type d)" ]]; then mv ../dropins/* ./capella/versions/$CAPELLA_VERSION/dropins/ @@ -435,6 +436,7 @@ papyrus/base: - *prepare-papyrus - *docker - mv ../papyrus.tar.gz ./papyrus/versions/$PAPYRUS_VERSION/papyrus.tar.gz + - cp ./eclipse/set_memory_flags.py ./papyrus/set_memory_flags.py - | docker build $DOCKER_BUILD_ARGS \ -t ${IMAGE}:${DOCKER_TAG} \ diff --git a/docs/docs/capella/base.md b/docs/docs/capella/base.md index 6c0c4376..a703bf12 100644 --- a/docs/docs/capella/base.md +++ b/docs/docs/capella/base.md @@ -244,6 +244,9 @@ Please replace the followings variables: (default) and when `AUTOSTART_CAPELLA=1`, Capella will be re-started as soon as it has been exited (after clean quits as well as crashs). +If you want to configure the JVM memory options, have a look at +[Eclipse memory options](../eclipse/memory-options.md). + ### Example to export representations (diagrams) as SVG images Replace `/path/to/model` and `` to pass any local Capella model. diff --git a/docs/docs/eclipse/base.md b/docs/docs/eclipse/base.md index d4459c46..47b5c4f7 100644 --- a/docs/docs/eclipse/base.md +++ b/docs/docs/eclipse/base.md @@ -115,3 +115,6 @@ Please replace the followings variables: - `RESTART_ECLIPSE` defines the restart behaviour of Eclipse. When set to 1 (default) and when `RESTART_ECLIPSE=1`, Eclipse will be re-started as soon as it has been exited (after clean quits as well as crashs). + +If you want to configure the JVM memory options, have a look at +[Eclipse memory options](./memory-options.md). diff --git a/docs/docs/eclipse/memory-options.md b/docs/docs/eclipse/memory-options.md new file mode 100644 index 00000000..5423db26 --- /dev/null +++ b/docs/docs/eclipse/memory-options.md @@ -0,0 +1,29 @@ + + +# Memory options for Eclipse + +To specify fixed memory options for the JVM, you can pass the following +environment variables to the `docker run` commands: + +- `MEMORY_MIN` (default `70%`), translated to `-Xms` for absolute values and + `-XX:InitialRAMPercentage` and `-XX:MinRAMPercentage` for percentage values. + Percentage values are calculated according to the total memory or the + requested memory by the container. +- `MEMORY_MAX` (default `90%`), translated to `-Xmx` for absolute values and + `-XX:MaxRAMPercentage` for percentage values. Percentage values are + calculated according to the total memory of the system or the total memory + available to the container. + +If the value ends with a %, we assume that it's a percentage value. + +- If used in a Kubernetes cluster, it determines the values from the Pod + requests/limits. +- If used on a host system, it determines the value from the host memory. + +See also: + +- https://stackoverflow.com/a/65327769 +- https://www.merikan.com/2019/04/jvm-in-a-container/#java-10 diff --git a/docs/docs/papyrus/base.md b/docs/docs/papyrus/base.md index 8b6e6345..51da7abc 100644 --- a/docs/docs/papyrus/base.md +++ b/docs/docs/papyrus/base.md @@ -61,6 +61,9 @@ To customise the Papyrus client you can docker build -t papyrus/base papyrus --build-arg PAPYRUS_VERSION=$PAPYRUS_VERSION ``` +If you want to configure the JVM memory options, have a look at +[Eclipse memory options](../eclipse/memory-options.md). + ## Run the container ### Locally on X11 systems diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 2057c314..8bbd34e6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -45,6 +45,7 @@ nav: - Base: papyrus/base.md - Eclipse: - Base: eclipse/base.md + - Memory Options: eclipse/memory-options.md - 'pure::variants': pure-variants.md - Remote: remote.md - Development: diff --git a/eclipse/Dockerfile b/eclipse/Dockerfile index 5999e154..5c1b7f56 100644 --- a/eclipse/Dockerfile +++ b/eclipse/Dockerfile @@ -23,14 +23,13 @@ WORKDIR /opt/ RUN tar -xf eclipse.tar.gz && \ rm -rf eclipse.tar.gz -ARG MEMORY_LIMIT=5500m - RUN mkdir -p /workspace; \ # Set workspace permissions chown -R techuser /workspace && \ chmod +x eclipse/eclipse && \ - chown -R techuser /opt/eclipse && \ - sed -i "s/-Xmx[^ ]*/-Xmx$MEMORY_LIMIT/g" /opt/eclipse/eclipse.ini + chown -R techuser /opt/eclipse + +COPY set_memory_flags.py /opt/setup/set_memory_flags.py USER techuser @@ -52,6 +51,12 @@ ENV RESTART_ECLIPSE=1 ENV ECLIPSE_INSTALLATION_PATH=/opt/eclipse ENV ECLIPSE_EXECUTABLE=/opt/eclipse/eclipse +# Set memory options for the JVM (used by set_memory_flags.py) +ARG MEMORY_MAX=90% +ENV MEMORY_MAX=${MEMORY_MAX} +ARG MEMORY_MIN=70% +ENV MEMORY_MIN=${MEMORY_MIN} + ENV BASE_TYPE=eclipse USER techuser diff --git a/eclipse/set_memory_flags.py b/eclipse/set_memory_flags.py new file mode 100644 index 00000000..cfdd5406 --- /dev/null +++ b/eclipse/set_memory_flags.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 +"""Set the memory flags in the *.ini file for Eclipse applications. + +The script reads the following environment variable: + +ECLIPSE_EXECUTABLE: The path to the eclipse executable. +MEMORY_MIN: The minimum memory that the JVM should use. + Check the set_memory_flags function for more information. +MEMORY_MAX: The maximum memory that the JVM should use. + Check the set_memory_flags function for more information. + +The script is called during container startup. +""" + +import os +import pathlib + + +def remove_all_memory_flags(file_content: list[str]) -> list[str]: + """Remove all memory flags from *.ini files. + + To make sure that we don't have any conflicting memory flags in the + .ini file, we remove all flags that start with -Xm. + + This will explicitely remove the -Xms and -Xmx flags. + """ + + return [ + line for line in file_content if not line.lstrip().startswith("-Xm") + ] + + +def set_memory_flags( + file_content: list[str], memory_min: str, memory_max: str +) -> None: + """Set the memory flags in *.ini. + + If the value of memory_max ends with a %, we assume that it's a percentage value. + - If used in a Kubernetes cluster, it determines the values from the Pod requests/limits. + - If used on a host system, it determines the value from the host memory. + + See Also + -------- + - https://stackoverflow.com/a/65327769 + - https://www.merikan.com/2019/04/jvm-in-a-container/#java-10 + """ + + if memory_min.strip().endswith("%"): + append_flag_to_file( + file_content, "XX:InitialRAMPercentage=", memory_min.strip("%") + ) + append_flag_to_file( + file_content, "XX:MinRAMPercentage=", memory_min.strip("%") + ) + else: + append_flag_to_file(file_content, "Xms", memory_min) + + if memory_max.strip().endswith("%"): + append_flag_to_file( + file_content, "XX:MaxRAMPercentage=", memory_max.strip("%") + ) + else: + append_flag_to_file(file_content, "Xmx", memory_max) + + +def print_vm_memory_usage_during_start(file_content: list[str]) -> None: + """Print the JVM memory options during start.""" + + file_content.append("-XshowSettings:vm") + + +def append_flag_to_file( + file_content: list[str], flag: str, value: str +) -> None: + """Append a flag to the *.ini file.""" + + file_content.append(f"-{flag}{value}") + + +if __name__ == "__main__": + eclipse_executable = pathlib.Path(os.environ["ECLIPSE_EXECUTABLE"]) + _memory_min = os.environ["MEMORY_MIN"].strip() + _memory_max = os.environ["MEMORY_MAX"].strip() + + ini_path = eclipse_executable.with_suffix(".ini") + + _file_content = ini_path.read_text("utf-8").splitlines() + + _file_content = remove_all_memory_flags(_file_content) + set_memory_flags(_file_content, _memory_min, _memory_max) + print_vm_memory_usage_during_start(_file_content) + + ini_path.write_text("\n".join(_file_content), encoding="utf-8") diff --git a/papyrus/Dockerfile b/papyrus/Dockerfile index 4db7f9fc..63ad1330 100644 --- a/papyrus/Dockerfile +++ b/papyrus/Dockerfile @@ -19,13 +19,12 @@ WORKDIR /opt/ RUN tar -xf papyrus.tar.gz && \ rm -rf papyrus.tar.gz -ARG MEMORY_LIMIT=5500m - +COPY set_memory_flags.py /opt/setup/set_memory_flags.py RUN mkdir -p /workspace; \ # Set workspace permissions chown -R techuser /workspace && \ chmod +x Papyrus/papyrus && \ - sed -i "s/-Xmx[^ ]*/-Xmx$MEMORY_LIMIT/g" /opt/Papyrus/papyrus.ini + chown techuser /opt/Papyrus/papyrus.ini # Install SysML RUN /opt/Papyrus/papyrus \ @@ -43,6 +42,12 @@ ENV RESTART_PAPYRUS=1 ENV ECLIPSE_INSTALLATION_PATH=/opt/Papyrus ENV ECLIPSE_EXECUTABLE=/opt/Papyrus/papyrus +# Set memory options for the JVM (used by set_memory_flags.py) +ARG MEMORY_MAX=90% +ENV MEMORY_MAX=${MEMORY_MAX} +ARG MEMORY_MIN=70% +ENV MEMORY_MIN=${MEMORY_MIN} + ENV BASE_TYPE=papyrus USER techuser