diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5b0ef98..540d4c8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -41,6 +41,7 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install -r dev-requirements.txt + pip install opm - name: Install expreccs run: | @@ -48,9 +49,9 @@ jobs: - name: Check code style and linting run: | - black --check src/ tests/ setup.py - pylint src/ tests/ setup.py - mypy --ignore-missing-imports src/ tests/ setup.py + black --check src/ tests/ + pylint src/ tests/ + mypy --ignore-missing-imports src/ tests/ - name: Run the tests run: | diff --git a/.gitignore b/.gitignore index 3fa4fe2..d4017d6 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,8 @@ opm-common/ opm-grid/ opm-models/ opm-simulators/ +build_opm_mpi.sh +build_opm_macos.sh # Extra folders tests/configs/back diff --git a/README.md b/README.md index 3ed0a58..b40e833 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build Status](https://github.com/cssr-tools/expreccs/actions/workflows/CI.yml/badge.svg)](https://github.com/cssr-tools/expreccs/actions/workflows/CI.yml) - + [![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![DOI](https://zenodo.org/badge/760077220.svg)](https://zenodo.org/doi/10.5281/zenodo.12100600) @@ -12,9 +12,15 @@ This repository contains scripts to set up a workflow to run site and regional r ## Installation You will first need to install -* Flow (https://opm-project.org, Release 2024.04 or current master branches) +* Flow (https://opm-project.org, Release 2024.04 or current master branches) -You can install the Python requirements in a virtual environment with the following commands: +To install the _expreccs_ executable in an existing Python environment: + +```bash +pip install git+https://github.com/cssr-tools/expreccs.git +``` + +If you are interested in modifying the source code, then you can clone the repository and install the Python requirements in a virtual environment with the following commands: ```bash # Clone the repo @@ -27,15 +33,13 @@ python3 -m venv vexpreccs source vexpreccs/bin/activate # Upgrade pip, setuptools, and wheel pip install --upgrade pip setuptools wheel -# Install the expreccs package (in editable mode for contributions/modifications; otherwise, pip install .) +# Install the expreccs package pip install -e . # For contributions/testing/linting, install the dev-requirements pip install -r dev-requirements.txt ``` -If you are a Linux user (including the Windows subsystem for Linux), then you could try to build Flow from the master branches with mpi support, by running the script `./build_opm-flow_mpi.bash` (see the [_CI.yml_](https://github.com/cssr-tools/expreccs/blob/main/.github/workflows/CI.yml)), which in turn should build the executable in ./build/opm-simulators/bin/flow_gaswater_dissolution. - -For macOS users with the latest chips (M1/M2, guessing also M3?), the opm Python package is not available via pip install, while resdata might not be available depending on the Python version (e.g., it is not found using Python 3.9, but it is installed using Python 3.10). If you face this issue, then before installation, remove resdata and opm from the `requirements.txt`, then proceed with the Python requirements installation, install the OPM Flow dependencies (using macports or brew), and once inside the vexpreccs Python environment, run the `./build_opm-flow_macOS.bash`, and deactivate and activate the virtual environment (this script builds OPM Flow as well as the opm Python package, and it exports the required PYTHONPATH). +See the [_installation_](https://cssr-tools.github.io/exprecss/installation.html) for further details on building OPM Flow from the master branches in Linux, Windows, and macOS, as well as the opm Python package. ## Running expreccs You can run _expreccs_ as a single command line: @@ -43,10 +47,10 @@ You can run _expreccs_ as a single command line: expreccs -i some_input.txt -o some_output_folder ``` Run `expreccs --help` to see all possible command line argument options. Inside the `some_input.txt` file you provide the path to the -flow executable and simulation parameters. See the .txt files in the examples and tests/configs folders. +flow executable and simulation parameters. See the .txt files in the [_examples_](https://github.com/cssr-tools/expreccs/tree/main/examples) and [_tests_](https://github.com/cssr-tools/expreccs/tree/main/tests/configs) folders. ## Getting started -See the [_documentation_](https://cssr-tools.github.io/expreccs/introduction.html). +See the [_examples_](https://cssr-tools.github.io/expreccs/examples.html) in the [_documentation_](https://cssr-tools.github.io/expreccs/introduction.html). ## About expreccs The expreccs package is funded by Wintershall Dea, Equinor, Shell, and the Research Council of Norway [project number 336294]. diff --git a/build_opm-flow_macOS.bash b/build_opm-flow_macOS.bash deleted file mode 100755 index 69a5c6b..0000000 --- a/build_opm-flow_macOS.bash +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: 2023 NORCE -# SPDX-License-Identifier: GPL-3.0 - -CURRENT_DIRECTORY="$PWD" - -# Dune modules -for module in common geometry grid istl -do git clone https://gitlab.dune-project.org/core/dune-$module.git -done -for module in common geometry grid istl -do ./dune-common/bin/dunecontrol --only=dune-$module cmake -DCMAKE_DISABLE_FIND_PACKAGE_MPI=1 - ./dune-common/bin/dunecontrol --only=dune-$module make -j5 -done - -# OPM modules -for repo in common grid models simulators -do - git clone https://github.com/OPM/opm-$repo.git -done - -source vexpreccs/bin/activate - -mkdir build - -for repo in common grid models -do - mkdir build/opm-$repo - cd build/opm-$repo - cmake -DPYTHON_EXECUTABLE=$(which python) -DUSE_MPI=0 -DNDEBUG=1 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo - make -j5 - cd ../.. -done - -mkdir build/opm-simulators -cd build/opm-simulators -cmake -DUSE_MPI=0 -DNDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators -make -j5 flow -cd ../.. - -echo "export PYTHONPATH=\$PYTHONPATH:$CURRENT_DIRECTORY/build/opm-common/build/python" >> $CURRENT_DIRECTORY/vexpreccs/bin/activate \ No newline at end of file diff --git a/build_opm-flow_mpi.bash b/build_opm-flow_mpi.bash deleted file mode 100644 index 5e6f59a..0000000 --- a/build_opm-flow_mpi.bash +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: 2023 NORCE -# SPDX-License-Identifier: GPL-3.0 - -CURRENT_DIRECTORY="$PWD" - -# OPM modules -for repo in common grid models simulators -do - git clone https://github.com/OPM/opm-$repo.git -done - -mkdir build - -for repo in common grid models -do - mkdir build/opm-$repo - cd build/opm-$repo - cmake -DUSE_MPI=1 -DNDEBUG=1 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo - make -j5 - cd ../.. -done - -mkdir build/opm-simulators -cd build/opm-simulators -cmake -DUSE_MPI=1 -DNDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators -make -j5 flow -cd ../.. \ No newline at end of file diff --git a/buildopm.sh b/buildopm.sh new file mode 100644 index 0000000..6ed298a --- /dev/null +++ b/buildopm.sh @@ -0,0 +1,29 @@ +CURRENT_DIRECTORY="$PWD" + +for module in common +do git clone https://gitlab.dune-project.org/core/dune-$module.git --branch v2.9.1 +done +for module in common +do ./dune-common/bin/dunecontrol --only=dune-$module cmake -DCMAKE_DISABLE_FIND_PACKAGE_MPI=1 + ./dune-common/bin/dunecontrol --only=dune-$module make -j5 +done + +for repo in common +do + git clone https://github.com/OPM/opm-$repo.git +done + +source vexpreccs/bin/activate + +mkdir build + +for repo in common +do + mkdir build/opm-$repo + cd build/opm-$repo + cmake -DPYTHON_EXECUTABLE=$(which python) -DWITH_NDEBUG=1 -DUSE_MPI=0 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo + make -j5 + cd ../.. +done + +echo "export PYTHONPATH=\$PYTHONPATH:$CURRENT_DIRECTORY/build/opm-common/build/python" >> $CURRENT_DIRECTORY/vexpreccs/bin/activate \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index 7ba71be..e6a74d0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,7 @@ black -pylint +numpydoc mypy +pylint pytest-cov sphinx sphinx-rtd-theme \ No newline at end of file diff --git a/docs/_images/plopm.png b/docs/_images/plopm.png new file mode 100644 index 0000000..b7b084f Binary files /dev/null and b/docs/_images/plopm.png differ diff --git a/docs/_images/pycopm.gif b/docs/_images/pycopm.gif new file mode 100644 index 0000000..40d2df9 Binary files /dev/null and b/docs/_images/pycopm.gif differ diff --git a/docs/_images/pyopmspe11.gif b/docs/_images/pyopmspe11.gif index 1d22e7d..e0a88a5 100644 Binary files a/docs/_images/pyopmspe11.gif and b/docs/_images/pyopmspe11.gif differ diff --git a/docs/_sources/examples.rst.txt b/docs/_sources/examples.rst.txt index d6dc1cd..1572e96 100644 --- a/docs/_sources/examples.rst.txt +++ b/docs/_sources/examples.rst.txt @@ -63,3 +63,15 @@ to generate the animation (using ResInsight) in the :doc:`introduction section < .. code-block:: bash expreccs -i example2.txt -m reference + +Generic (under development) +--------------------------- + +See/run the last lines (34 to 42) in the `test_main.py `_ +for an example where **expreccs** is used in two given models (regional and site, in this case they are created using +the **expreccs** package, but in general can be any given geological models), generating a new input deck where +the pressures are projected. + +.. code-block:: bash + + expreccs -e name_of_folder_for_the_regional_model,name_of_folder_for_the_site_model \ No newline at end of file diff --git a/docs/_sources/expreccs.utils.reg_sit_given_decks.rst.txt b/docs/_sources/expreccs.utils.reg_sit_given_decks.rst.txt new file mode 100644 index 0000000..49d0623 --- /dev/null +++ b/docs/_sources/expreccs.utils.reg_sit_given_decks.rst.txt @@ -0,0 +1,8 @@ +expreccs.utils.reg\_sit\_given\_decks module +============================================ + +.. automodule:: expreccs.utils.reg_sit_given_decks + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/_sources/expreccs.utils.rst.txt b/docs/_sources/expreccs.utils.rst.txt index 4734616..10fdb0e 100644 --- a/docs/_sources/expreccs.utils.rst.txt +++ b/docs/_sources/expreccs.utils.rst.txt @@ -11,6 +11,7 @@ Submodules expreccs.utils.inputvalues expreccs.utils.mapboundaries expreccs.utils.mapproperties + expreccs.utils.reg_sit_given_decks expreccs.utils.runs expreccs.utils.writefile diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt index a0a733d..1cc32c5 100644 --- a/docs/_sources/index.rst.txt +++ b/docs/_sources/index.rst.txt @@ -7,6 +7,7 @@ Welcome to expreccs's documentation! :maxdepth: 4 introduction + installation configuration_file examples api diff --git a/docs/_sources/installation.rst.txt b/docs/_sources/installation.rst.txt new file mode 100644 index 0000000..269b075 --- /dev/null +++ b/docs/_sources/installation.rst.txt @@ -0,0 +1,136 @@ +============ +Installation +============ + +Python package +-------------- + +To install the **expreccs** executable in an existing Python environment: + +.. code-block:: bash + + pip install git+https://github.com/cssr-tools/expreccs.git + +If you are interested in modifying the source code, then you can clone the repository and +install the Python requirements in a virtual environment with the following commands: + +.. code-block:: console + + # Clone the repo + git clone https://github.com/cssr-tools/expreccs.git + # Get inside the folder + cd expreccs + # Create virtual environment + python3 -m venv vexpreccs + # Activate virtual environment + source vexpreccs/bin/activate + # Upgrade pip, setuptools, and wheel + pip install --upgrade pip setuptools wheel + # Install the expreccs package + pip install -e . + # For contributions/testing/linting, install the dev-requirements + pip install -r dev-requirements.txt + +.. note:: + + For not macOS users, to install the Python opm package, execute in the terminal **pip install opm**. + For macOS, see :ref:`macOS`. + +OPM Flow +-------- +You also need to install: + +* OPM Flow (https://opm-project.org, Release 2024.04 or current master branches) + +.. tip:: + + See the `CI.yml `_ script + for installation of OPM Flow (binary packages) and the expreccs package in Linux. + +Source build in Linux/Windows ++++++++++++++++++++++++++++++ +If you are a Linux user (including the Windows subsystem for Linux), then you could try to build Flow (after installing the `prerequisites `_) from the master branches with mpi support by running +in the terminal the following lines (which in turn should build flow in the folder ./build/opm-simulators/bin/flow.): + +.. code-block:: console + + CURRENT_DIRECTORY="$PWD" + + for repo in common grid models simulators + do + git clone https://github.com/OPM/opm-$repo.git + done + + mkdir build + + for repo in common grid models + do + mkdir build/opm-$repo + cd build/opm-$repo + cmake -DUSE_MPI=1 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo + make -j5 opm$repo + cd ../.. + done + + mkdir build/opm-simulators + cd build/opm-simulators + cmake -DUSE_MPI=1 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators + make -j5 flow + cd ../.. + + +.. tip:: + + You can create a .sh file (e.g., build_opm_mpi.sh), copy the previous lines, and run in the terminal **source build_opm_mpi.sh** + +.. _macOS: + +Source build in macOS ++++++++++++++++++++++ +For macOS, there are no available binary packages, so OPM Flow needs to be built from source, in addition to the dune libraries and the opm Python +package (see the `prerequisites `_, which can be installed using macports or brew). This can be achieved by the following lines: + +.. code-block:: console + + CURRENT_DIRECTORY="$PWD" + + for module in common geometry grid istl + do git clone https://gitlab.dune-project.org/core/dune-$module.git --branch v2.9.1 + done + for module in common geometry grid istl + do ./dune-common/bin/dunecontrol --only=dune-$module cmake -DCMAKE_DISABLE_FIND_PACKAGE_MPI=1 + ./dune-common/bin/dunecontrol --only=dune-$module make -j5 + done + + for repo in common grid models simulators + do + git clone https://github.com/OPM/opm-$repo.git + done + + source vexpreccs/bin/activate + + mkdir build + + for repo in common grid models + do + mkdir build/opm-$repo + cd build/opm-$repo + cmake -DPYTHON_EXECUTABLE=$(which python) -DWITH_NDEBUG=1 -DUSE_MPI=0 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo + make -j5 + cd ../.. + done + + mkdir build/opm-simulators + cd build/opm-simulators + cmake -DUSE_MPI=0 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators + make -j5 flow + cd ../.. + + echo "export PYTHONPATH=\$PYTHONPATH:$CURRENT_DIRECTORY/build/opm-common/python" >> $CURRENT_DIRECTORY/vexpreccs/bin/activate + + +This builds OPM Flow as well as the opm Python package, and it exports the required PYTHONPATH. Then after execution, deactivate and activate the Python virtual environment. + +Regarding the resdata Python package, it might not be available depending on the Python version (e.g., it is not found using Python 3.9, but it is installed using Python 3.10). +Then, it is recommended to use a Python version equal or higher than 3.10; otherwise, remove resdata from the requirements in the `pyproject.toml `_, +and the opm Python package will be used (this is the default package for reading the simulation files, see the :ref:`overview`). \ No newline at end of file diff --git a/docs/_sources/introduction.rst.txt b/docs/_sources/introduction.rst.txt index b958589..e4dfa7f 100644 --- a/docs/_sources/introduction.rst.txt +++ b/docs/_sources/introduction.rst.txt @@ -7,7 +7,7 @@ Introduction This documentation describes the content of the **expreccs** package. The numerical simulations are performed using the -`Flow `_ simulator. +`OPM Flow `_ simulator. Concept ------- @@ -18,31 +18,28 @@ Simplified and flexible testing framework for two-stage approach to improve regi - Set the fluxes/pressures from the regional model as boundary conditions on the site model. - Simulate the site model. +.. _overview: + Overview -------- The current implementation supports the following executable with the argument options: .. code-block:: bash - expreccs -i input.txt -o output -m all -c '' -p 'yes' -r opm + expreccs -i input.txt -o output -m all -c '' -p 'no' -r opm -u gaswater -t 0 -e '' where -- \-i, \-input: The base name of the :doc:`configuration file <./configuration_file>` ('input.txt' by default). -- \-o, \-output: The base name of the :doc:`output folder <./output_folder>` ('output' by default). -- \-m, \-mode: Run the whole framework ('all'), only the reference ('reference'), only the site ('site'), or only regional and site models ('noreference') ('all' by default). -- \-c, \-compare: Generate metric plots for the current outputed folders ('compare') (' ' by default). -- \-p, \-plot: Create nice figures in the postprocessing folder ('no' by default). -- \-r, \-reading: Using the 'opm' or 'ecl' python package ('opm' by default). +- \-i: The base name of the :doc:`configuration file <./configuration_file>` ('input.txt' by default). +- \-o: The base name of the :doc:`output folder <./output_folder>` ('output' by default). +- \-m: Run the whole framework ('all'), only the reference ('reference'), only the site ('site'), or only regional and site models ('noreference') ('all' by default). +- \-c: Generate metric plots for the current outputed folders ('compare') ('' by default). +- \-p: Create nice figures in the postprocessing folder ('no' by default). +- \-r: Using the 'opm' or 'resdata' python package ('opm' by default). +- \-u: Using 'gasoil' or 'gaswater' co2store implementation ('gaswater' by default). +- \-t: Grades to rotate the site geological model ('0' by default). +- \-e:: Name of the regional and site folders to project pressures ('' by default). In the **configuration file** the geological model is defined by generation of corner-point grids (cpg), adding heterogeinities (e.g., different rock properties, faults), wells, and defining schedules for the -operations (see the :doc:`configuration file <./configuration_file>` section). - -Installation ------------- - -See the `Github page `_. - -.. tip:: - Check the `CI.yml `_ file. \ No newline at end of file +operations (see the :doc:`configuration file <./configuration_file>` section). \ No newline at end of file diff --git a/docs/_sources/related.rst.txt b/docs/_sources/related.rst.txt index 0e95a88..5729c22 100644 --- a/docs/_sources/related.rst.txt +++ b/docs/_sources/related.rst.txt @@ -23,6 +23,23 @@ pyopmnearwell `A framework to simulate near well dynamics using OPM Flow `_. +****** +pycopm +****** + +.. image:: ./figs/pycopm.gif + :scale: 60% + +`Simplified and flexible framework for coarsening geological models `_. + +***** +plopm +***** + +.. image:: ./figs/plopm.png + +`Quick generation of PNG figures from a simulation model given any 2D slide `_. + ******* ad-micp ******* diff --git a/docs/_static/basic.css b/docs/_static/basic.css index 30fee9d..f316efc 100644 --- a/docs/_static/basic.css +++ b/docs/_static/basic.css @@ -4,7 +4,7 @@ * * Sphinx stylesheet -- basic theme. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/docs/_static/doctools.js b/docs/_static/doctools.js index d06a71d..4d67807 100644 --- a/docs/_static/doctools.js +++ b/docs/_static/doctools.js @@ -4,7 +4,7 @@ * * Base JavaScript utilities for all Sphinx HTML documentation. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/docs/_static/language_data.js b/docs/_static/language_data.js index 250f566..367b8ed 100644 --- a/docs/_static/language_data.js +++ b/docs/_static/language_data.js @@ -5,7 +5,7 @@ * This script contains the language-specific data used by searchtools.js, * namely the list of stopwords, stemmer, scorer and splitter. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @@ -13,7 +13,7 @@ var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"]; -/* Non-minified version is copied as a separate JS file, is available */ +/* Non-minified version is copied as a separate JS file, if available */ /** * Porter Stemmer diff --git a/docs/_static/searchtools.js b/docs/_static/searchtools.js index 7918c3f..b08d58c 100644 --- a/docs/_static/searchtools.js +++ b/docs/_static/searchtools.js @@ -4,7 +4,7 @@ * * Sphinx JavaScript utilities for the full-text search. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @@ -99,7 +99,7 @@ const _displayItem = (item, searchTerms, highlightTerms) => { .then((data) => { if (data) listItem.appendChild( - Search.makeSearchSummary(data, searchTerms) + Search.makeSearchSummary(data, searchTerms, anchor) ); // highlight search terms in the summary if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js @@ -116,8 +116,8 @@ const _finishSearch = (resultCount) => { ); else Search.status.innerText = _( - `Search finished, found ${resultCount} page(s) matching the search query.` - ); + "Search finished, found ${resultCount} page(s) matching the search query." + ).replace('${resultCount}', resultCount); }; const _displayNextItem = ( results, @@ -137,6 +137,22 @@ const _displayNextItem = ( // search finished, update title and status message else _finishSearch(resultCount); }; +// Helper function used by query() to order search results. +// Each input is an array of [docname, title, anchor, descr, score, filename]. +// Order the results by score (in opposite order of appearance, since the +// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically. +const _orderResultsByScoreThenName = (a, b) => { + const leftScore = a[4]; + const rightScore = b[4]; + if (leftScore === rightScore) { + // same score: sort alphabetically + const leftTitle = a[1].toLowerCase(); + const rightTitle = b[1].toLowerCase(); + if (leftTitle === rightTitle) return 0; + return leftTitle > rightTitle ? -1 : 1; // inverted is intentional + } + return leftScore > rightScore ? 1 : -1; +}; /** * Default splitQuery function. Can be overridden in ``sphinx.search`` with a @@ -160,13 +176,26 @@ const Search = { _queued_query: null, _pulse_status: -1, - htmlToText: (htmlString) => { + htmlToText: (htmlString, anchor) => { const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html'); - htmlElement.querySelectorAll(".headerlink").forEach((el) => { el.remove() }); + for (const removalQuery of [".headerlink", "script", "style"]) { + htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() }); + } + if (anchor) { + const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`); + if (anchorContent) return anchorContent.textContent; + + console.warn( + `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.` + ); + } + + // if anchor not specified or not found, fall back to main content const docContent = htmlElement.querySelector('[role="main"]'); - if (docContent !== undefined) return docContent.textContent; + if (docContent) return docContent.textContent; + console.warn( - "Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template." + "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template." ); return ""; }, @@ -239,16 +268,7 @@ const Search = { else Search.deferQuery(query); }, - /** - * execute search (requires search index to be loaded) - */ - query: (query) => { - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const titles = Search._index.titles; - const allTitles = Search._index.alltitles; - const indexEntries = Search._index.indexentries; - + _parseQuery: (query) => { // stem the search terms and add them to the correct list const stemmer = new Stemmer(); const searchTerms = new Set(); @@ -284,21 +304,38 @@ const Search = { // console.info("required: ", [...searchTerms]); // console.info("excluded: ", [...excludedTerms]); - // array of [docname, title, anchor, descr, score, filename] - let results = []; + return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; + }, + + /** + * execute search (requires search index to be loaded) + */ + _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + + // Collect multiple result groups to be sorted separately and then ordered. + // Each is an array of [docname, title, anchor, descr, score, filename]. + const normalResults = []; + const nonMainIndexResults = []; + _removeChildren(document.getElementById("search-progress")); - const queryLower = query.toLowerCase(); + const queryLower = query.toLowerCase().trim(); for (const [title, foundTitles] of Object.entries(allTitles)) { - if (title.toLowerCase().includes(queryLower) && (queryLower.length >= title.length/2)) { + if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) { for (const [file, id] of foundTitles) { - let score = Math.round(100 * queryLower.length / title.length) - results.push([ + const score = Math.round(Scorer.title * queryLower.length / title.length); + const boost = titles[file] === title ? 1 : 0; // add a boost for document titles + normalResults.push([ docNames[file], titles[file] !== title ? `${titles[file]} > ${title}` : title, id !== null ? "#" + id : "", null, - score, + score + boost, filenames[file], ]); } @@ -308,46 +345,47 @@ const Search = { // search for explicit entries in index directives for (const [entry, foundEntries] of Object.entries(indexEntries)) { if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) { - for (const [file, id] of foundEntries) { - let score = Math.round(100 * queryLower.length / entry.length) - results.push([ + for (const [file, id, isMain] of foundEntries) { + const score = Math.round(100 * queryLower.length / entry.length); + const result = [ docNames[file], titles[file], id ? "#" + id : "", null, score, filenames[file], - ]); + ]; + if (isMain) { + normalResults.push(result); + } else { + nonMainIndexResults.push(result); + } } } } // lookup as object objectTerms.forEach((term) => - results.push(...Search.performObjectSearch(term, objectTerms)) + normalResults.push(...Search.performObjectSearch(term, objectTerms)) ); // lookup as search terms in fulltext - results.push(...Search.performTermsSearch(searchTerms, excludedTerms)); + normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms)); // let the scorer override scores with a custom scoring function - if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item))); - - // now sort the results by score (in opposite order of appearance, since the - // display function below uses pop() to retrieve items) and then - // alphabetically - results.sort((a, b) => { - const leftScore = a[4]; - const rightScore = b[4]; - if (leftScore === rightScore) { - // same score: sort alphabetically - const leftTitle = a[1].toLowerCase(); - const rightTitle = b[1].toLowerCase(); - if (leftTitle === rightTitle) return 0; - return leftTitle > rightTitle ? -1 : 1; // inverted is intentional - } - return leftScore > rightScore ? 1 : -1; - }); + if (Scorer.score) { + normalResults.forEach((item) => (item[4] = Scorer.score(item))); + nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item))); + } + + // Sort each group of results by score and then alphabetically by name. + normalResults.sort(_orderResultsByScoreThenName); + nonMainIndexResults.sort(_orderResultsByScoreThenName); + + // Combine the result groups in (reverse) order. + // Non-main index entries are typically arbitrary cross-references, + // so display them after other results. + let results = [...nonMainIndexResults, ...normalResults]; // remove duplicate search results // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept @@ -361,7 +399,12 @@ const Search = { return acc; }, []); - results = results.reverse(); + return results.reverse(); + }, + + query: (query) => { + const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query); + const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms); // for debugging //Search.lastresults = results.slice(); // a copy @@ -466,14 +509,18 @@ const Search = { // add support for partial matches if (word.length > 2) { const escapedWord = _escapeRegExp(word); - Object.keys(terms).forEach((term) => { - if (term.match(escapedWord) && !terms[word]) - arr.push({ files: terms[term], score: Scorer.partialTerm }); - }); - Object.keys(titleTerms).forEach((term) => { - if (term.match(escapedWord) && !titleTerms[word]) - arr.push({ files: titleTerms[word], score: Scorer.partialTitle }); - }); + if (!terms.hasOwnProperty(word)) { + Object.keys(terms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: terms[term], score: Scorer.partialTerm }); + }); + } + if (!titleTerms.hasOwnProperty(word)) { + Object.keys(titleTerms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: titleTerms[term], score: Scorer.partialTitle }); + }); + } } // no match but word was a required one @@ -496,9 +543,8 @@ const Search = { // create the mapping files.forEach((file) => { - if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1) - fileMap.get(file).push(word); - else fileMap.set(file, [word]); + if (!fileMap.has(file)) fileMap.set(file, [word]); + else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word); }); }); @@ -549,8 +595,8 @@ const Search = { * search summary for a given text. keywords is a list * of stemmed words. */ - makeSearchSummary: (htmlText, keywords) => { - const text = Search.htmlToText(htmlText); + makeSearchSummary: (htmlText, keywords, anchor) => { + const text = Search.htmlToText(htmlText, anchor); if (text === "") return null; const textLower = text.toLowerCase(); diff --git a/docs/about.html b/docs/about.html index 3c2f071..ceeb3f1 100644 --- a/docs/about.html +++ b/docs/about.html @@ -16,7 +16,7 @@ - + @@ -47,6 +47,7 @@ diff --git a/docs/examples.html b/docs/examples.html index 8e6f8da..c38aa8d 100644 --- a/docs/examples.html +++ b/docs/examples.html @@ -16,7 +16,7 @@ - + @@ -48,10 +48,12 @@