Docs Ubuntu Upgrade

deploy-docs.sh

@@ -25,10 +25,10 @@ if [ "$RELEASE" != "true" ]; then
 
     # Index the build directory.
     ocdsindex sphinx build/ https://standard.open-contracting.org/"$PREFIX""$PATH_PREFIX""$REF"/ > documents.json
-    ocdsindex index https://standard.open-contracting.org:9200 documents.json
+    ocdsindex index https://standard.open-contracting.org/search documents.json
     if [ "$REF" == "$VERSION" ]; then
         ocdsindex sphinx build/ https://standard.open-contracting.org/"$PREFIX""$PATH_PREFIX"latest/ > documents.json
-        ocdsindex index https://standard.open-contracting.org:9200 documents.json
+        ocdsindex index https://standard.open-contracting.org/search documents.json
     fi
 
     if [ "$PRODUCTION" == "true" ]; then

docs/deploy/create_server.rst

@@ -288,7 +288,6 @@ For Django application servers:
 For OCDS documentation servers:
 
 #. Copy the ``/home/ocds-docs/web`` directory
-#. Update the IP addresses in the ``pillar/cove.sls`` file, and deploy the ``cove-*`` services
 
 For Redmine servers:
 

docs/develop/update/elasticsearch.rst

@@ -46,82 +46,6 @@ Set swappiness value
    vm:
      swappiness: 1
 
-Enable public access
---------------------
-
-As stated by Elasticsearch, `"Do not expose Elasticsearch directly to users." <https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-security.html>`__ For the OCDS documentation, we use the `ReadOnlyREST <https://readonlyrest.com>`__ plugin to control access.
-
-#. Add the ``elasticsearch.plugins.readonlyrest`` state file to your service's target in the ``salt/top.sls`` file.
-
-#. Allow anyone to access Elasticsearch. Add to your service's Pillar file:
-
-   .. code-block:: yaml
-
-      elasticsearch:
-        public_access: True
-
-#. Allow cross-origin HTTP requests (optional). Add to your service's Pillar file, for example:
-
-   .. code-block:: yaml
-      :emphasize-lines: 2
-
-      elasticsearch:
-        allowed_origins: https://standard.open-contracting.org
-
-#. Configure ReadOnlyREST SSL certificates in your service's Pillar file. If :doc:`Apache<apache>` and Elasticsearch serve content from the same domain, you can reuse the certificates acquired by the ``mod_md`` module. For example:
-
-   .. code-block:: yaml
-
-      elasticsearch:
-        plugins:
-          readonlyrest:
-            certificate_key_file: /etc/apache2/md/domains/standard.open-contracting.org/privkey.pem
-            certificate_file: /etc/apache2/md/domains/standard.open-contracting.org/pubcert.pem
-
-   If reusing certificates, configure the ``mod_md`` module to restart Elasticsearch after renewing certificates:
-
-   .. code-block:: yaml
-      :emphasize-lines: 2-4
-
-      apache:
-        modules:
-          mod_md:
-            MDNotifyCmd: /opt/restart-elasticsearch.sh
-
-#. Add users for public searches and for admin actions. Add to your service's *private* Pillar file, replacing ``AUTH_KEY_SHA512`` with the output of ``echo -n 'USERNAME:PASSWORD' | shasum -a 512`` (replacing ``USERNAME`` and ``PASSWORD`` with a strong password each time):
-
-   .. code-block:: yaml
-      :emphasize-lines: 4-10
-
-      elasticsearch:
-        plugins:
-          readonlyrest:
-            users:
-              - auth_key_sha512: AUTH_KEY_SHA512
-                username: public
-                groups:
-                  - public
-              - auth_key_sha512: AUTH_KEY_SHA512
-                username: manage
-                groups:
-                  - manage
-
-#. :doc:`Deploy the service<../../deploy/deploy>`
-
-#. Test the public user, replacing ``PASSWORD``. For example, for the ``standard.open-contracting.org`` domain:
-
-   .. code-block:: bash
-
-      curl -u 'public:PASSWORD' https://standard.open-contracting.org:9200/ocdsindex_en/_search \
-      -H 'Content-Type: application/json' \
-      -d '{"query": {"term": {"base_url": "https://standard.open-contracting.org/staging/1.1-dev/"}}}'
-
-#. Test the admin user, replacing ``PASSWORD``. For example, for the ``standard.open-contracting.org`` domain:
-
-   .. code-block:: bash
-
-      curl -u 'manage:PASSWORD' https://standard.open-contracting.org:9200/_cat/indices
-
 Troubleshoot
 ~~~~~~~~~~~~
 
@@ -152,7 +76,7 @@ You will see a message like (newlines are added for readability):
        MET:HEAD,
        PTH:/ocdsindex_en,
        CNT:<N/A>,
-       HDR:Accept=*/*, Authorization=<OMITTED>, Host=standard.open-contracting.org:9200, User-Agent=curl/7.64.1, content-length=0,
+       HDR:Accept=*/*, Authorization=<OMITTED>, Host=standard.open-contracting.org, User-Agent=curl/7.64.1, content-length=0,
        HIS:
          [Allow localhost->
            RULES:[hosts->false],

docs/maintain/elasticsearch.rst

@@ -54,35 +54,35 @@ List indices:
 
 .. code-block:: bash
 
-   curl -n https://standard.open-contracting.org:9200/_cat/indices
+   curl -n https://standard.open-contracting.org/search/_cat/indices
 
 List base URLs in a given index, for example:
 
 .. code-block:: bash
 
-   curl -n -X GET 'https://standard.open-contracting.org:9200/ocdsindex_en/_search?size=0&pretty' \
+   curl -n -X GET 'https://standard.open-contracting.org/search/ocdsindex_en/_search?size=0&pretty' \
    -H 'Content-Type: application/json' \
    -d '{"aggs": {"base_urls": {"terms": {"field": "base_url", "size": 10000}}}}'
 
 Delete documents matching a base URL:
 
 .. code-block:: bash
 
-   curl -n -X POST 'https://standard.open-contracting.org:9200/ocdsindex_en/_delete_by_query' \
+   curl -n -X POST 'https://standard.open-contracting.org/search/ocdsindex_en/_delete_by_query' \
    -H 'Content-Type: application/json' \
    -d '{"query": {"term": {"base_url": "https://standard.open-contracting.org/staging/1.1-dev/"}}}'
 
 Expire documents using `OCDS Index <https://github.com/open-contracting/ocds-index>`__:
 
 .. code-block:: bash
 
-   ocdsindex expire https://standard.open-contracting.org:9200 --exclude-file=ocdsindex-exclude.txt
+   ocdsindex expire https://standard.open-contracting.org/search --exclude-file=ocdsindex-exclude.txt
 
 Search documents in a given index matching a base URL, for example:
 
 .. code-block:: bash
 
-   curl -n -X GET 'https://standard.open-contracting.org:9200/ocdsindex_en/_search?size=10000' \
+   curl -n -X GET 'https://standard.open-contracting.org/search/ocdsindex_en/_search?size=10000' \
    -H 'Content-Type: application/json' \
    -d '{"query": {"term": {"base_url": "https://standard.open-contracting.org/staging/1.1-dev/"}}}'
 

pillar/cove.sls

@@ -15,9 +15,6 @@ python_apps:
         VALIDATION_ERROR_LOCATIONS_LENGTH: 100
     apache:
       configuration: django
-      context:
-        docs_ipv4: 5.28.62.151
-        docs_ipv6: 2001:41c9:1:41c::151
     uwsgi:
       configuration: django
       harakiri: 1800 # 30 min

pillar/docs.sls

@@ -1,3 +1,11 @@
+network:
+  host_id: ocp19
+  ipv4: 178.79.135.174
+  #ipv6: 2001:db8::19
+  networkd:
+    template: linode
+    gateway4: 178.79.135.1
+
 ssh:
   docs:
     # Public key for salt/private/keys/docs_ci
@@ -17,14 +25,11 @@ apache:
       servername: standard.open-contracting.org
 
 elasticsearch:
-  public_access: True
-  # Allow OCDS documentation and GitHub Actions.
-  allowed_origins: "*"
   # This is to inform the installation of ReadOnlyREST – not to control the version of Elasticsearch to install.
-  version: 7.17.9
+  version: 8.6.2
   plugins:
     readonlyrest:
-      version: 1.47.0_es7.17.9
+      version: 1.47.0_es8.6.2
       configuration: docs
       certificate_key_file: /etc/elasticsearch/ssl/standard.open-contracting.org/privkey.pem
       certificate_file: /etc/elasticsearch/ssl/standard.open-contracting.org/pubcert.pem

pillar/docs_maintenance.sls

@@ -3,3 +3,6 @@ maintenance:
   patching: manual
   rkhunter_customisation: |
     ALLOWHIDDENDIR=/etc/.java
+    ALLOW_SSH_ROOT_USER=yes
+    RTKT_FILE_WHITELIST=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9
+    USER_FILEPROP_FILES_DIRS=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9

salt-config/roster

@@ -2,7 +2,7 @@
 
 cove-oc4ids:         ocp17.open-contracting.org
 cove-ocds:           ocp18.open-contracting.org
-docs:                ocp07.open-contracting.org
+docs:                ocp19.open-contracting.org
 kingfisher-process:  ocp04.open-contracting.org
 kingfisher-replica:  ocp05.open-contracting.org
 prometheus:          ocp03.open-contracting.org
@@ -16,6 +16,7 @@ registry:
 # ocp01 was cove-oc4ids on Ubuntu 18
 # ocp02 was cove-ocds on Ubuntu 18
 # ocp06 was covid19-dev
+# ocp07 was docs on Ubuntu 18
 # ocp08 was redash.open-contracting.org on Ubuntu 18
 # ocp09 was toucan.open-contracting.org
 # ocp10 was archive.kingfisher.open-contracting.org

salt/apache/files/sites/docs.conf.include

@@ -1,7 +1,7 @@
 # vi: ft=apache
 
-{# The paths should be in decreasing order of specificity to generate the correct order of precedence. #}
-{%
+{#- The paths should be in decreasing order of specificity to generate the correct order of precedence. #}
+{%-
     set options = {
         '/profiles/eforms': {
             'versions': ['latest'],
@@ -37,10 +37,10 @@
     }
 %}
 
-{# Matches all root paths. #}
-{% set pattern = '(profiles/[^/]+/|infrastructure/)?' %}
+{#- Matches all root paths. #}
+{%- set pattern = '(profiles/[^/]+/|infrastructure/)?' %}
 
-{% set documentroot = '/home/ocds-docs/web' %}
+{%- set documentroot = '/home/ocds-docs/web' %}
 
 DocumentRoot {{ documentroot }}
 RewriteMap unescape int:unescape
@@ -76,13 +76,6 @@ SetEnv BANNER /includes/banner_live.html
     SetEnv BANNER /includes/banner_staging_profiles_ppp.html
 </Location>
 
-# Needed to proxy to SSL servers.
-SSLProxyEngine on
-# ProxyPreserveHost was on but we need it to be Off, as it makes adding SSL much easier.
-ProxyPreserveHost Off
-# With a keepalive, we had problems with headers being interpreted as the start of the response.
-SetEnv proxy-nokeepalive 1
-
 # Remember: The Directory directive applies only to static files, not to proxied or redirected paths.
 <Directory {{ documentroot }}>
     Require all granted
@@ -347,3 +340,8 @@ RedirectMatch ^/infrastructure/review/(.*)$ https://review-oc4ids.standard.open-
     # The backreferences are the root, version and language.
     RewriteRule ^{{ documentroot }}/staging/{{ pattern }}([^/]*) https://standard.open-contracting.org/staging/$1$2/${unescape:%1}/? [R]
 </LocationMatch>
+
+####################
+# ElasticSearch
+####################
+ProxyPassMatch "^/search/(.*)?" http://localhost:9200/$1

salt/apache/includes/cove.include.jinja

@@ -1,6 +1 @@
 ErrorDocument 500 "<h2>Sorry, something went wrong.</h2> <p>Sometimes this happens because the input file is too big - maybe try again with a smaller sample.</p><p>Please file a <a href=\"https://github.com/open-contracting/cove-ocds/issues/new\">GitHub issue</a> or email <a href=\"mailto:[email protected]\">[email protected]</a> if this problem persists.</p>"
-
-RemoteIPHeader X-Forwarded-For
-# Trust docs reverse proxy.
-RemoteIPTrustedProxy {{ docs_ipv4 }}
-RemoteIPTrustedProxy {{ docs_ipv6 }}

salt/docs/init.sls

@@ -6,8 +6,16 @@ include:
   - apache.modules.rewrite
 
 {% set user = 'ocds-docs' %}
+{% set userdir = '/home/' + user %}
 {{ create_user(user, authorized_keys=pillar.ssh.docs) }}
 
+allow Apache access to {{ userdir }}:
+  file.directory:
+    - name: {{ userdir }}
+    - mode: 755
+    - require:
+      - user: {{ user }}_user_exists
+
 # Needed to create a ZIP file of the schema and codelists.
 # https://ocdsdeploy.readthedocs.io/en/latest/deploy/docs.html#copy-the-schema-and-zip-file-into-place
 zip:

salt/elasticsearch/files/config/readonlyrest-docs.yml

@@ -1,14 +1,20 @@
 readonlyrest:
   # https://github.com/beshu-tech/readonlyrest-docs/blob/master/kibana.md#loading-settings-order-of-precedence
   force_load_from_file: True
-  ssl:
-    server_certificate_key_file: {{ pillar.elasticsearch.plugins.readonlyrest.certificate_key_file }}
-    server_certificate_file: {{ pillar.elasticsearch.plugins.readonlyrest.certificate_file }}
   # https://github.com/beshu-tech/readonlyrest-docs/blob/master/actionstrings/action_strings_es7.10.1.txt
   access_control_rules:
-    - name: Allow localhost
-      hosts:
-        - localhost
+    - name: Allow the ocpadmin group to administrate Elasticsearch
+      groups:
+        - ocpadmin
+      x_forwarded_for:
+        - "127.0.0.0/8"
+        - "{{ pillar.network.ipv4 }}"
+        {%- if "ipv6" in pillar.network %}
+        - "{{ pillar.network.ipv6 }}"
+        {%- endif %}
+        {%- for ip in pillar.elasticsearch.get("admin_ips", []) %}
+        - "{{ ip }}"
+        {%- endfor %}
     - name: Allow the public group to search indices created by OCDS Index
       groups:
         - public
@@ -18,6 +24,7 @@ readonlyrest:
         must_involve_indices: True
       actions:
         - indices:data/read/search
+      x_forwarded_for: ["0.0.0.0/0"]
     - name: Allow the manage group to manage indices created by OCDS Index
       groups:
         - manage
@@ -31,4 +38,5 @@ readonlyrest:
         - indices:data/write/bulk
         - indices:data/write/delete/byquery
         - indices:monitor/settings/get
+      x_forwarded_for: ["0.0.0.0/0"]
   users: {{ pillar.elasticsearch.plugins.readonlyrest.users|yaml }}