Upgrade script notes and fix for 1.16 on Debian (#3824)

Co-authored-by: ammar92 <[email protected]>
Co-authored-by: Jan Klopper <[email protected]>

boefjes/tests/integration/test_bench.py

@@ -3,10 +3,11 @@
 from pathlib import Path
 
 import pytest
-from tools.upgrade_v1_16_0 import upgrade
+from tools.upgrade_v1_17_0 import upgrade
 
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.config import BASE_DIR
+from boefjes.models import Organisation
 from boefjes.sql.organisation_storage import SQLOrganisationStorage
 from octopoes.connector.octopoes import OctopoesAPIConnector
 from octopoes.models import Reference
@@ -25,6 +26,9 @@ def test_migration(
     octopoes_api_connector.session._timeout.connect = 60
     octopoes_api_connector.session._timeout.read = 60
 
+    # Create an organisation that does not exist in Octopoes
+    organisation_storage.create(Organisation(id="test2", name="Test 2"))
+
     iterations = 30
     cache_path = Path(BASE_DIR.parent / ".ci" / f".cache_{iterations}.json")
     hostname_range = range(0, iterations)

boefjes/tools/upgrade_v1_16_0.py → boefjes/tools/upgrade_v1_17_0.py

@@ -80,6 +80,17 @@ def migrate_organisation(
     and set the source_method to the boefje id. Then update the origin, i.e. save it and delete the old one.
     """
 
+    try:
+        connector.health()
+    except HTTPStatusError as e:
+        if e.response.status_code == 404:
+            logger.warning(
+                "Organisation found that does not exist in Octopoes [organisation_id=%s]. Make sure to remove this "
+                "organisation from the Katalogus database if it is no longer in use.",
+                organisation_id,
+            )
+        raise
+
     failed = 0
     processed = 0
 

docs/source/release-notes/1.16.rst

@@ -91,25 +91,6 @@ Then follow the steps above using the HTTP endpoints to delete the other entries
 After these steps, again the normal instructions for upgrading :ref:`Debian packages<Upgrading Debian>`
 or upgrading :ref:`containers <Upgrading_Containers>` should be followed.
 
-Running the Origin Migration
-================================
-Upon upgrading, one migration needs to be triggered manually.
-This is the `boefjes/tools/upgrade_v1_16_0.py` script, which you can run in your ``boefje`` environment using:
-
-.. code-block:: sh
-
-    python -m tools.upgrade_v1_16_0
-
-Or, using Docker:
-
-.. code-block:: sh
-
-    docker compose exec boefje python -m tools.upgrade_v1_16_0
-
-Please check the logs for any errors in the migration.
-The script can be run multiple times in case unexpected errors appear, although some warnings are to be expected.
-After running the script (ideally once), the next run should log that `total_processed=0` and `total_failed=0`.
-
 Full Changelog
 ==============
 

docs/source/release-notes/1.17.rst

@@ -135,6 +135,35 @@ Upgrading
 All bits all need to be rerun because of model changes. This can be done on the
 organization settings page.
 
+Running the Origin Migration
+================================
+Upon upgrading, one migration needs to be triggered manually.
+This is the `boefjes/tools/upgrade_v1_16_0.py` script, which you can run in your ``boefje`` environment using:
+
+.. code-block:: sh
+
+    python -m tools.upgrade_v1_16_0
+
+Or more concretely, using Docker:
+
+.. code-block:: sh
+
+    docker compose run --rm boefje python -m tools.upgrade_v1_16_0
+
+And using the Debian packages, either as the `kat` user or root depending on your configuration:
+
+.. code-block:: sh
+
+    source /opt/venvs/kat-boefjes/bin/activate
+    export $(cat /usr/lib/kat/boefjes.defaults | grep -v "#") && export $(cat /etc/kat/boefjes.conf | grep -v "#")
+    cd /opt/venvs/kat-boefjes/lib/python3.11/site-packages
+    /opt/venvs/kat-boefjes/bin/python -m tools.upgrade_v1_16_0
+
+Please check the logs for any errors in the migration.
+The script can be run multiple times in case unexpected errors appear, although some warnings are to be expected.
+After running the script (ideally once), the next run should log that `total_processed=0` and `total_failed=0`.
+
+
 Full Changelog
 ==============