Merge pull request #771 from dbmi-bgm/dmichaels-20240709

Use snovault search modules rather than the local ones.

CHANGELOG.rst

@@ -6,6 +6,12 @@ cgap-portal
 Change Log
 ----------
 
+15.5.0
+======
+* Use snovault search modules rather than the local ones (in the src/encoded/search directory).
+* Update to dcicutils 8.13.3.
+
+
 15.4.0
 ======
 

Makefile

@@ -123,7 +123,9 @@ kibana-stop:
 kill:  # kills back-end processes associated with the application. Use with care.
 	pkill -f postgres &
 	pkill -f elasticsearch &
+	pkill -f opensearch &
 	pkill -f moto_server &
+	pkill -f nginx &
 
 clean-python:
 	@echo -n "Are you sure? This will wipe all libraries installed on this virtualenv [y/N] " && read ans && [ $${ans:-N} = y ]

base.ini

@@ -9,6 +9,14 @@ file_wfout_bucket = cgap-unit-testing-wfout
 # th
 # system_bucket = elasticbeanstalk-encoded-4dn-system
 elasticsearch.server = 127.0.0.1:9200
+# elasticsearch.server.actual_port = 9201
+# The above elasticsearch.server.actual_port property is useful (only) for running a localhost ElasticSearch
+# proxy in order to observe traffic (requests/responses) between portal and ElasticSearch with a tool like
+# mitmweb; e.g. setting elasticsearch.server.actual_port to 9201 and elasticsearch.server to localhost:9200
+# will case ElasticSearch to actually run on port 9201 but will cause portal to talk to it via port 9200,
+# and then we can run mitmweb --mode reverse:http://localhost:9201 -p 9200 --web-port 8081 which will
+# allow us to browse to http://localhost:8081 locally to observe all of the ElasticSearch traffic;
+# and note to install mitmweb do: pip install mitmproxy
 ontology_path = %(here)s/ontology.json
 aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
 #this is to reroute downloads to something other than aws s3

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "cgap-portal".
 name = "encoded"
-version = "15.4.0"
+version = "15.5.0"
 description = "Computational Genome Analysis Platform"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -40,9 +40,9 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.9,<3.13"
-awscli = ">=1.33.17"
-boto3 = "^1.34.136"
-botocore = "^1.34.136"
+awscli = ">=1.33.25"
+boto3 = "^1.34.143"
+botocore = "^1.34.143"
 certifi = ">=2021.5.30"
 chardet = "^5.2.0"
 codeguru-profiler-agent = "^1.2.4"
@@ -53,7 +53,7 @@ cryptography = "39.0.2"
 colorama = "0.3.3"
 dcicpyvcf = "^3.1.0"
 dcicsnovault = "^11.18.0"
-dcicutils = "^8.13.0"
+dcicutils = "^8.13.3"
 elasticsearch = "7.13.4"
 execnet = "1.4.1"
 future = ">=0.18.2,<1"
@@ -134,8 +134,8 @@ WSGIProxy2 = "0.4.2"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-boto3-stubs = "^1.34.136"
-botocore-stubs = "^1.34.136"
+boto3-stubs = "^1.34.143"
+botocore-stubs = "^1.34.143"
 coverage = ">=6.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=3.3.1"

src/encoded/__init__.py

@@ -208,8 +208,8 @@ def main(global_config, **local_config):
 
     if 'elasticsearch.server' in config.registry.settings:
         config.include('snovault.elasticsearch')
-        config.include('.search.search')
-        config.include('.search.compound_search')  # could make enabling configurable
+        config.include('snovault.search.search')
+        config.include('snovault.search.compound_search')  # could make enabling configurable
 
     # this contains fall back url, so make sure it comes just before static_resoruces
     config.include('.types.page')

src/encoded/batch_download.py

@@ -11,7 +11,7 @@
 from snovault.util import simple_path_ids, debug_log
 
 from .batch_download_utils import stream_tsv_output, convert_item_to_sheet_dict, human_readable_filter_block_queries
-from .search.compound_search import CompoundSearchBuilder
+from snovault.search.compound_search import CompoundSearchBuilder
 from .types.variant import get_spreadsheet_mappings
 
 

src/encoded/tests/test_lucene_builder.py

@@ -1,5 +1,5 @@
 import pytest
-from ..search.lucene_builder import LuceneBuilder
+from snovault.search.lucene_builder import LuceneBuilder
 
 
 pytestmark = [pytest.mark.setone, pytest.mark.broken]

src/encoded/tests/test_search.py

@@ -12,8 +12,8 @@
 from snovault.schema_utils import load_schema
 from snovault.util import add_default_embeds
 from webtest import AppError
-from ..search.lucene_builder import LuceneBuilder
-from ..search.search_utils import find_nested_path
+from snovault.search.lucene_builder import LuceneBuilder
+from snovault.search.search_utils import find_nested_path
 
 
 pytestmark = [pytest.mark.working, pytest.mark.schema, pytest.mark.search, pytest.mark.workbook]
@@ -1245,7 +1245,7 @@ class TestSearchHiddenAndAdditionalFacets:
     """ Encapsulates tests meant for testing behavior associated with default_hidden, hidden
         and additional_facets
     """
-    DEFAULT_FACETS = ['first_name']  # 'validation_errors.name'
+    DEFAULT_FACETS = ['first_name', 'type']  # 'validation_errors.name'
     DEFAULT_HIDDEN_FACETS = ['last_name', 'sid']
     ADDITIONAL_FACETS = ['unfaceted_string', 'unfaceted_integer']
     DISABLED_FACETS = ['disabled_string', 'disabled_integer']

src/encoded/tests/test_types_variant_sample.py

@@ -163,7 +163,7 @@ def _search(self, *args, **kwargs):
             # TODO Add "total": 0 and other properties to response.
             return {"@graph": []}
 
-@mock.patch("encoded.search.search.SearchBuilder", new=MockedSearchBuilder)
+@mock.patch("snovault.search.search.SearchBuilder", new=MockedSearchBuilder)
 def test_variant_sample_patch_notes_process_success(
     testapp,
     bgm_user,

src/encoded/types/file.py

@@ -49,7 +49,7 @@
     urlparse,
 )
 from ..authentication import session_properties
-from ..search.search import make_search_subreq
+from snovault.search.search import make_search_subreq
 from ..util import check_user_is_logged_in, make_s3_client
 from .base import (
     Item,

src/encoded/types/page.py

@@ -23,7 +23,7 @@
     no_validate_item_content_patch
 )
 from urllib.parse import urlparse, urlencode
-from ..search.search import get_iterable_search_results
+from snovault.search.search import get_iterable_search_results
 from .base import Item, collection_add, item_edit
 # from .user_content import StaticSection
 

src/encoded/types/variant.py

@@ -26,7 +26,7 @@
 from ..custom_embed import CustomEmbed
 from ..ingestion.common import CGAP_CORE_PROJECT
 from ..inheritance_mode import InheritanceMode
-from ..search.search import get_iterable_search_results
+from snovault.search.search import get_iterable_search_results
 from ..types.base import Item, get_item_or_none
 from ..util import resolve_file_path, build_s3_presigned_get_url, convert_integer_to_comma_string