From c0b821630ac4445b9c5892f5917c8ccb0c5f61bc Mon Sep 17 00:00:00 2001 From: computermacgyver <computermacgyver> Date: Fri, 6 Oct 2023 21:53:07 +0900 Subject: [PATCH 1/2] First pass at OpenSearch locally --- docker-compose.yml | 25 +++++++++++++------------ manage.py | 2 +- opensearch/Dockerfile | 4 ++++ opensearch/alegre_similarity.json | 14 ++++++++------ 4 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 opensearch/Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 221d8801..2202a744 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,23 +5,24 @@ volumes: postgres: services: elasticsearch: - build: ./elasticsearch + build: ./opensearch ports: - "9200:9200" environment: discovery.type: single-node transport.host: 127.0.0.1 - xpack.security.enabled: "false" + plugins.security.disabled: "true" + # xpack.security.enabled: "false" volumes: - - "elasticsearch:/usr/share/elasticsearch/data" - kibana: - image: docker.elastic.co/kibana/kibana:7.9.2 - ports: - - "5601:5601" - depends_on: - - elasticsearch - environment: - ELASTICSEARCH_URL: http://elasticsearch:9200 + - "opensearch:/usr/share/opensearch/data" + # kibana: + # image: docker.elastic.co/kibana/kibana:7.9.2 + # ports: + # - "5601:5601" + # depends_on: + # - elasticsearch + # environment: + # ELASTICSEARCH_URL: http://elasticsearch:9200 redis: image: redis:5 ports: @@ -141,7 +142,7 @@ services: - ".:/app" depends_on: - postgres - - kibana + # - kibana - redis # - video # - xlm_r_bert_base_nli_stsb_mean_tokens diff --git a/manage.py b/manage.py index bffdb1d9..2dfd36c7 100644 --- a/manage.py +++ b/manage.py @@ -275,7 +275,7 @@ def init(): else: raise es.indices.put_mapping( - body=json.load(open('./elasticsearch/alegre_similarity.json')), + body=json.load(open('./opensearch/alegre_similarity.json')), # include_type_name=True, index=app.config['ELASTICSEARCH_SIMILARITY'] ) diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile new file mode 100644 index 00000000..04738366 --- /dev/null +++ b/opensearch/Dockerfile @@ -0,0 +1,4 @@ +FROM opensearchproject/opensearch:2 +RUN echo y | bin/opensearch-plugin install analysis-icu +RUN echo y | bin/opensearch-plugin install repository-s3 + diff --git a/opensearch/alegre_similarity.json b/opensearch/alegre_similarity.json index 1fb35c73..24219572 100644 --- a/opensearch/alegre_similarity.json +++ b/opensearch/alegre_similarity.json @@ -1,5 +1,4 @@ { - "mappings": { "properties": { "vector_768": { "type": "knn_vector", @@ -9,10 +8,6 @@ "type": "knn_vector", "dimension": 768 }, - "vector_openai-text-embedding-ada-002": { - "type": "dense_vector", - "dims": 1536 - }, "vector_paraphrase-filipino-mpnet-base-v2": { "type": "knn_vector", "dimension": 768 @@ -21,6 +16,14 @@ "type": "knn_vector", "dimension": 768 }, + "vector_paraphrase-multilingual-mpnet-base-v2": { + "type": "knn_vector", + "dimension": 768 + }, + "vector_openai-text-embedding-ada-002": { + "type": "knn_vector", + "dimension": 1536 + }, "content": { "type": "text" }, @@ -32,7 +35,6 @@ }, "model": { "type": "keyword" - } } } } From 69d415a3104b61d22d6e562c2da53de606132933 Mon Sep 17 00:00:00 2001 From: computermacgyver <computermacgyver> Date: Fri, 6 Oct 2023 21:56:02 +0900 Subject: [PATCH 2/2] fix typo --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2202a744..6c12c3dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: plugins.security.disabled: "true" # xpack.security.enabled: "false" volumes: - - "opensearch:/usr/share/opensearch/data" + - "elasticsearch:/usr/share/opensearch/data" # kibana: # image: docker.elastic.co/kibana/kibana:7.9.2 # ports: @@ -148,4 +148,4 @@ services: # - xlm_r_bert_base_nli_stsb_mean_tokens # - indian_sbert env_file: - - .env_file \ No newline at end of file + - .env_file