From f8545a2e3f699e7bb7936b3fa7695ae3c1cec27f Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 16:37:33 +0100 Subject: [PATCH 01/12] REFACTOR: Dotnet coreclr loading --- src/pyedb/dotnet/clr_module.py | 123 ++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 31 deletions(-) diff --git a/src/pyedb/dotnet/clr_module.py b/src/pyedb/dotnet/clr_module.py index 24636c61f0..04750e5e03 100644 --- a/src/pyedb/dotnet/clr_module.py +++ b/src/pyedb/dotnet/clr_module.py @@ -1,68 +1,126 @@ import os +from pathlib import Path import pkgutil +import shutil import sys import warnings +import pyedb + +LINUX_WARNING = ( + "Due to compatibility issues between .NET Core and libssl on some Linux versions, " + "for example Ubuntu 22.04, we are going to stop depending on `dotnetcore2`." + "Instead of using this package which embeds .NET Core 3, users will be required to " + "install .NET themselves. For more information, see " + "https://edb.docs.pyansys.com/version/stable/build_breaking_change.html" +) + +existing_showwarning = warnings.showwarning + + +def custom_show_warning(message, category, filename, lineno, file=None, line=None): + """Custom warning used to remove :loc: pattern.""" + print(f"{category.__name__}: {message}") + + +warnings.showwarning = custom_show_warning + modules = [tup[1] for tup in pkgutil.iter_modules()] cpython = "IronPython" not in sys.version and ".NETFramework" not in sys.version is_linux = os.name == "posix" is_windows = not is_linux is_clr = False +pyedb_path = Path(pyedb.__file__).parent +sys.path.append(str(pyedb_path / "dlls" / "PDFReport")) -try: - import pyedb - pyedb_path = os.path.dirname(os.path.abspath(pyedb.__file__)) - sys.path.append(os.path.join(pyedb_path, "dlls", "PDFReport")) -except ImportError: - pyedb_path = None - warnings.warn("Cannot import pyedb.") +def find_dotnet_root() -> Path: + """Find dotnet root path.""" + dotnet_path = shutil.which("dotnet") + if not dotnet_path: + raise FileNotFoundError("The 'dotnet' executable was not found in the PATH.") -if is_linux and cpython: # pragma: no cover + dotnet_path = Path(dotnet_path).resolve() + dotnet_root = dotnet_path.parent + return dotnet_root + + +def find_runtime_config(dotnet_root: Path) -> Path: + """Find dotnet runtime configuration file path.""" + sdk_path = dotnet_root / "sdk" + if not sdk_path.is_dir(): + raise EnvironmentError(f"The 'sdk' directory could not be found in: {dotnet_root}") + sdk_versions = sorted(sdk_path.iterdir(), key=lambda x: x.name, reverse=True) + if not sdk_versions: + raise FileNotFoundError("No SDK versions were found.") + runtime_config = sdk_versions[0] / "dotnet.runtimeconfig.json" + if not runtime_config.is_file(): + raise FileNotFoundError(f"The configuration file '{runtime_config}' does not exist.") + return runtime_config + + +if is_linux: # pragma: no cover + from pythonnet import load + + # Use system DOTNET core runtime try: + from clr_loader import get_coreclr + + runtime = get_coreclr() + load(runtime) + is_clr = True + # Define DOTNET root and runtime config file to load DOTNET core runtime + except Exception: if os.environ.get("DOTNET_ROOT") is None: - runtime = None try: - import dotnet + dotnet_root = find_dotnet_root() + runtime_config = find_runtime_config(dotnet_root) + except Exception: + warnings.warn( + "Unable to set DOTNET root and locate the runtime configuration file. " + "Falling back to using dotnetcore2." + ) + warnings.warn(LINUX_WARNING) - runtime = os.path.join(os.path.dirname(dotnet.__path__)) - except: import dotnetcore2 - runtime = os.path.join(os.path.dirname(dotnetcore2.__file__), "bin") - finally: - os.environ["DOTNET_ROOT"] = runtime - - from pythonnet import load - - if pyedb_path is not None: - json_file = os.path.abspath(os.path.join(pyedb_path, "misc", "pyedb.runtimeconfig.json")) - load("coreclr", runtime_config=json_file, dotnet_root=os.environ["DOTNET_ROOT"]) + dotnet_root = Path(dotnetcore2.__file__).parent / "bin" + runtime_config = pyedb_path / "misc" / "pyedb.runtimeconfig.json" + else: + dotnet_root = Path(os.environ["DOTNET_ROOT"]) + try: + runtime_config = find_runtime_config(dotnet_root) + except Exception as e: + raise RuntimeError( + "Configuration file could not be found from DOTNET_ROOT. " + "Please ensure that .NET SDK is correctly installed or " + "that DOTNET_ROOT is correctly set." + ) + try: + load("coreclr", runtime_config=str(runtime_config), dotnet_root=str(dotnet_root)) print("DotNet Core correctly loaded.") if "mono" not in os.getenv("LD_LIBRARY_PATH", ""): warnings.warn("LD_LIBRARY_PATH needs to be setup to use pyedb.") - warnings.warn("export ANSYSEM_ROOT232=/path/to/AnsysEM/v232/Linux64") + warnings.warn("export ANSYSEM_ROOT242=/path/to/AnsysEM/v242/Linux64") msg = "export LD_LIBRARY_PATH=" - msg += "$ANSYSEM_ROOT232/common/mono/Linux64/lib64:$LD_LIBRARY_PATH" + msg += "$ANSYSEM_ROOT242/common/mono/Linux64/lib64:$LD_LIBRARY_PATH" msg += ( - "If PyEDB will run on AEDT<2023.2 then $ANSYSEM_ROOT222/Delcross should be added to LD_LIBRARY_PATH" + "If PyEDB is used with AEDT<2023.2 then /path/to/AnsysEM/v2XY/Linux64/Delcross " + "should be added to LD_LIBRARY_PATH." ) warnings.warn(msg) is_clr = True - else: - print("DotNet Core not correctly loaded.") - except ImportError: - msg = "pythonnet or dotnetcore not installed. Pyedb will work only in client mode." - warnings.warn(msg) + except ImportError: + msg = "pythonnet or dotnetcore not installed. Pyedb will work only in client mode." + warnings.warn(msg) else: try: from pythonnet import load load("coreclr") is_clr = True - except: - pass + warnings.warn("Unable to load DOTNET core runtime") try: # work around a number formatting bug in the EDB API for non-English locales @@ -93,6 +151,7 @@ Dictionary = None Array = None edb_initialized = False + if "win32com" in modules: try: import win32com.client as win32_client @@ -101,3 +160,5 @@ import win32com.client as win32_client except ImportError: win32_client = None + +warnings.showwarning = existing_showwarning From 861534e33a9f2d50ed621b04a52e71edb109597c Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 16:43:54 +0100 Subject: [PATCH 02/12] REFACTOR: Remove warning from pyedb import --- src/pyedb/__init__.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/pyedb/__init__.py b/src/pyedb/__init__.py index 64f28cdbf6..50870ad661 100644 --- a/src/pyedb/__init__.py +++ b/src/pyedb/__init__.py @@ -11,13 +11,6 @@ os.environ["PYEDB_USE_DOTNET"] = "0" LATEST_DEPRECATED_PYTHON_VERSION = (3, 7) -LINUX_WARNING = ( - "Due to compatibility issues between .NET Core and libssl on some Linux versions, " - "for example Ubuntu 22.04, we are going to stop depending on `dotnetcore2`." - "Instead of using this package which embeds .NET Core 3, users will be required to " - "install .NET themselves. For more information, see " - "https://edb.docs.pyansys.com/version/stable/build_breaking_change.html" -) def deprecation_warning(): @@ -46,25 +39,6 @@ def custom_show_warning(message, category, filename, lineno, file=None, line=Non warnings.showwarning = existing_showwarning -def linux_warning(): - """Warning message informing Linux users a future breaking change is coming.""" - # Store warnings showwarning - existing_showwarning = warnings.showwarning - - # Define and use custom showwarning - def custom_show_warning(message, category, filename, lineno, file=None, line=None): - """Custom warning used to remove :loc: pattern.""" - print("{}: {}".format(category.__name__, message)) - - warnings.showwarning = custom_show_warning - - if os.name == "posix": - warnings.warn(LINUX_WARNING, FutureWarning) - - # Restore warnings showwarning - warnings.showwarning = existing_showwarning - - deprecation_warning() # From 4608932d881be6a2a4add7e581c6159a087289f8 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 16:46:25 +0100 Subject: [PATCH 03/12] TESTS: Remove warning tests on pyedb import --- tests/test_warnings.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/tests/test_warnings.py b/tests/test_warnings.py index 47ed1b3904..9e1e36630d 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -20,17 +20,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import os import sys from unittest.mock import patch import warnings -from pyedb import ( - LATEST_DEPRECATED_PYTHON_VERSION, - LINUX_WARNING, - deprecation_warning, - linux_warning, -) +from pyedb import LATEST_DEPRECATED_PYTHON_VERSION, deprecation_warning @patch.object(warnings, "warn") @@ -48,14 +42,3 @@ def test_deprecation_warning(mock_warn): mock_warn.assert_called_once_with(expected, PendingDeprecationWarning) else: mock_warn.assert_not_called() - - -@patch.object(warnings, "warn") -def test_linux_warning(mock_warn): - linux_warning() - - is_linux = os.name == "posix" - if is_linux: - mock_warn.assert_called_once_with(LINUX_WARNING, PendingDeprecationWarning) - else: - mock_warn.assert_not_called() From 7689c0f88db76675ac7e21690d2905ccd08f0c74 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 16:47:32 +0100 Subject: [PATCH 04/12] DOCS: Update breaking changes --- doc/source/build_breaking_change.rst | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/doc/source/build_breaking_change.rst b/doc/source/build_breaking_change.rst index 0e5f382ea5..4ba554cdd9 100644 --- a/doc/source/build_breaking_change.rst +++ b/doc/source/build_breaking_change.rst @@ -18,14 +18,9 @@ A temporary workaround was considered, which involved manually installing an old `libssl1.1` library. While this allowed the use of `dotnetcore2`, it is **not recommended** as a long-term solution for the following reasons: -- **Security risks**: Installing an older version of `libssl` introduces vulnerabilities, as it may -lack the latest security updates provided in the newer versions. -- **System instability**: Manually forcing an older version of `libssl` can lead to dependency -conflicts with other software packages that rely on newer versions of this library, potentially -causing further compatibility issues in the system. -- **Maintenance overhead**: Relying on deprecated or unsupported libraries increases the -complexity of future upgrades and system maintenance, making the environment harder to manage in the -long term. +- **Security risks**: Installing an older version of `libssl` introduces vulnerabilities, as it may lack the latest security updates provided in the newer versions. +- **System instability**: Manually forcing an older version of `libssl` can lead to dependency conflicts with other software packages that rely on newer versions of this library, potentially causing further compatibility issues in the system. +- **Maintenance overhead**: Relying on deprecated or unsupported libraries increases the complexity of future upgrades and system maintenance, making the environment harder to manage in the long term. Impact ------ @@ -35,7 +30,7 @@ Microsoft documentation for `.NET` on Linux to ensure proper setup and compatibi `Register Microsoft package repository `_ and `Install .NET `_. -.. note:: Ubuntu 22.04 and later versions +.. note:: Starting with Ubuntu 22.04, `.NET` is available in the official Ubuntu repository. If you want to use the Microsoft package to install `.NET`, you can use the following approach to *"demote"* the Ubuntu packages so that the Microsoft packages take precedence. From e5ffdb1882b324cce10889830efc08453a691186 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 17:03:18 +0100 Subject: [PATCH 05/12] DOCS: Update documentation --- doc/source/build_breaking_change.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/build_breaking_change.rst b/doc/source/build_breaking_change.rst index 4ba554cdd9..5be766739d 100644 --- a/doc/source/build_breaking_change.rst +++ b/doc/source/build_breaking_change.rst @@ -19,7 +19,9 @@ A temporary workaround was considered, which involved manually installing an old long-term solution for the following reasons: - **Security risks**: Installing an older version of `libssl` introduces vulnerabilities, as it may lack the latest security updates provided in the newer versions. + - **System instability**: Manually forcing an older version of `libssl` can lead to dependency conflicts with other software packages that rely on newer versions of this library, potentially causing further compatibility issues in the system. + - **Maintenance overhead**: Relying on deprecated or unsupported libraries increases the complexity of future upgrades and system maintenance, making the environment harder to manage in the long term. Impact From 8bdbf8eb452e6e6a330483d656e735aabac957ac Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Wed, 27 Nov 2024 17:04:08 +0100 Subject: [PATCH 06/12] TBR: Run pyaedt tests --- .github/workflows/ci_cd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 630ffcaab4..056df0ea8a 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -206,7 +206,6 @@ jobs: test-pyaedt-main-windows: name: Testing PyAEDT main branch (Windows) - if: github.event_name == 'push' && contains(github.ref, 'refs/tags') needs: [smoke-tests] runs-on: [ self-hosted, Windows, pyaedt ] steps: @@ -282,7 +281,6 @@ jobs: test-pyaedt-main-linux: name: Testing PyAEDT main branch (Linux) - if: github.event_name == 'push' && contains(github.ref, 'refs/tags') needs: [smoke-tests] runs-on: [ self-hosted, Linux, pyaedt ] env: From 24a523233c78212e9b0ee48a097665c7277cc49d Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Thu, 28 Nov 2024 14:07:31 +0100 Subject: [PATCH 07/12] CI: Update environment variable path --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 056df0ea8a..73b6bfcdc6 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -284,7 +284,7 @@ jobs: needs: [smoke-tests] runs-on: [ self-hosted, Linux, pyaedt ] env: - ANSYSEM_ROOT242: '/ansys_inc/AnsysEM/v242/Linux64' + ANSYSEM_ROOT242: '/opt/AnsysEM/v242/Linux64' ANS_NODEPCHECK: '1' steps: - name: Install Git and checkout project From d3b3e431861a9e2878aedca57cc8531e91e44afc Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Thu, 28 Nov 2024 14:20:18 +0100 Subject: [PATCH 08/12] TBR: Target updated VM --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 73b6bfcdc6..830c908c86 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -282,7 +282,7 @@ jobs: test-pyaedt-main-linux: name: Testing PyAEDT main branch (Linux) needs: [smoke-tests] - runs-on: [ self-hosted, Linux, pyaedt ] + runs-on: [ self-hosted, Linux, pyaedt, testing_dotnet ] env: ANSYSEM_ROOT242: '/opt/AnsysEM/v242/Linux64' ANS_NODEPCHECK: '1' From 24637ec5327426f20ca08ff77cd1c4a94cf0b7c0 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Thu, 28 Nov 2024 15:02:17 +0100 Subject: [PATCH 09/12] TEST: strange behavior might be related to delcross --- .github/workflows/ci_cd.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 830c908c86..2ed18eb8d6 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -302,12 +302,10 @@ jobs: - name: Create Python venv run: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH python -m venv .venv - name: Update pip run: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH . .venv/bin/activate python -m pip install -U pip @@ -320,18 +318,19 @@ jobs: - name: Install PyAEDT main branch version with its test dependencies run: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate pip install --no-cache-dir external/pyaedt[tests] - name: Install PyEDB run: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate python -m pip install . - name: Install CI dependencies (e.g. vtk-osmesa) run: | + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate # Uninstall conflicting dependencies pip uninstall --yes vtk @@ -346,7 +345,7 @@ jobs: retry_on: error timeout_minutes: 50 command: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate pytest -n auto --dist loadfile --durations=50 -v external/pyaedt/tests/system/general/ @@ -359,7 +358,7 @@ jobs: retry_on: error timeout_minutes: 50 command: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:${{ env.ANSYSEM_ROOT242 }}/Delcross:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate pytest --durations=50 -v external/pyaedt/tests/system/solvers From 159f12f28dfe09e704e1ef3cd2c4f6241b6266e2 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Thu, 28 Nov 2024 15:24:24 +0100 Subject: [PATCH 10/12] CI: fix typo --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 2ed18eb8d6..1d5d3c7db0 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -358,7 +358,7 @@ jobs: retry_on: error timeout_minutes: 50 command: | - export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${{ env.ANSYSEM_ROOT242 }}/common/mono/Linux64/lib64:$LD_LIBRARY_PATH . .venv/bin/activate pytest --durations=50 -v external/pyaedt/tests/system/solvers From 2e9a32dbc71e5a11cddb69ea29a303df95ecd918 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Thu, 28 Nov 2024 21:23:55 +0100 Subject: [PATCH 11/12] REFACTOR: Remove print --- src/pyedb/dotnet/clr_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pyedb/dotnet/clr_module.py b/src/pyedb/dotnet/clr_module.py index 04750e5e03..710d31329a 100644 --- a/src/pyedb/dotnet/clr_module.py +++ b/src/pyedb/dotnet/clr_module.py @@ -98,7 +98,6 @@ def find_runtime_config(dotnet_root: Path) -> Path: ) try: load("coreclr", runtime_config=str(runtime_config), dotnet_root=str(dotnet_root)) - print("DotNet Core correctly loaded.") if "mono" not in os.getenv("LD_LIBRARY_PATH", ""): warnings.warn("LD_LIBRARY_PATH needs to be setup to use pyedb.") warnings.warn("export ANSYSEM_ROOT242=/path/to/AnsysEM/v242/Linux64") From 4e445d718a08376dc4d8819914fc6dc65e363b01 Mon Sep 17 00:00:00 2001 From: Sebastien Morais Date: Fri, 29 Nov 2024 08:44:56 +0100 Subject: [PATCH 12/12] CI: Revert testing changes --- .github/workflows/ci_cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 1d5d3c7db0..1328f44602 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -206,6 +206,7 @@ jobs: test-pyaedt-main-windows: name: Testing PyAEDT main branch (Windows) + if: github.event_name == 'push' && contains(github.ref, 'refs/tags') needs: [smoke-tests] runs-on: [ self-hosted, Windows, pyaedt ] steps: @@ -281,8 +282,9 @@ jobs: test-pyaedt-main-linux: name: Testing PyAEDT main branch (Linux) + if: github.event_name == 'push' && contains(github.ref, 'refs/tags') needs: [smoke-tests] - runs-on: [ self-hosted, Linux, pyaedt, testing_dotnet ] + runs-on: [ self-hosted, Linux, pyaedt ] env: ANSYSEM_ROOT242: '/opt/AnsysEM/v242/Linux64' ANS_NODEPCHECK: '1'