Merge branch 'release/2024-12-20_python3.10_infrastructure'

.gitignore

@@ -28,6 +28,7 @@ htmlcov/*
 # Don't push our dev configs or the secret app configs.
 dev.cfg
 app.cfg
+instance.cfg
 
 # directories of transient stuff that gets created locally
 media/*

deploy/dev/huey_events.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_events_queue.events_queue >> ~/huey_events_queue.log 2>&1

deploy/dev/huey_scheduled_long.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue >> ~/huey_scheduled_long_queue.log 2>&1

deploy/dev/huey_scheduled_short.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue >> ~/huey_scheduled_short_queue.log 2>&1

deploy/nginx/doaj → deploy/nginx/production/doaj

@@ -35,29 +35,24 @@ map $http_user_agent $block_ua {
     ~*wget              1;
     ~*curl              1;
 }
-
-# the public server (deprecated, use failover)
-upstream doaj_apps {
-    server 10.131.191.139:5050;   #doaj-public-app-1
+# For public site components, try all servers
+upstream doaj_apps_failover {
+    server 10.131.191.147:5050;        #doaj-public-1
+    server 10.131.191.148:5050 backup;   #doaj-background-1
+    server 10.131.191.149:5050 backup;  #doaj-editor-1
 }
 
 # Background server runs async tasks
 upstream doaj_bg_apps {
-    server 10.131.12.33:5050;     #doaj-background-app-1
+    server 10.131.191.148:5050;     #doaj-background-app-1
 }
 
 # Editor and admin site components
 upstream doaj_ed_failover {
-    server 10.131.56.133:5050;    #doaj-editor-app-1
-    server 10.131.12.33:5050 backup;     #doaj-background-app-1
+    server 10.131.191.149:5050;    #doaj-editor-app-1
+    server 10.131.191.148:5050 backup;     #doaj-background-app-1
 }
 
-# For public site components, try all servers
-upstream doaj_apps_failover {
-    server 10.131.191.139:5050;        #doaj-public-app-1
-    server 10.131.12.33:5050 backup;   #doaj-background-app-1
-    server 10.131.56.133:5050 backup;  #doaj-editor-app-1
-}
 upstream doaj_index {
     server 10.131.191.132:9200;
     server 10.131.191.133:9200;

deploy/supervisor/production-background/huey-events.conf

@@ -0,0 +1,9 @@
+[program:huey-events]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_events_queue.events_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/production-background/huey-scheduled-long.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-long]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/production-background/huey-scheduled-short.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-short]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-events.conf

@@ -0,0 +1,9 @@
+[program:huey-events]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_events_queue.events_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-scheduled-long.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-long]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-scheduled-short.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-short]
+command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

doajtest/fixtures/background.py

@@ -52,12 +52,14 @@ def cleanup(self):
 
 
 def save_mock_bgjob(action=None, status=None, created_before_sec=0, is_save=True,
-                    queue_id=None):
+                    blocking=True, queue_id=None):
     bgjob = BackgroundJob()
 
-    if action:
+    if not action:
         from portality.tasks.journal_csv import JournalCSVBackgroundTask
         bgjob.action = JournalCSVBackgroundTask.__action__
+    else:
+        bgjob.action = action
 
     if status:
         bgjob._set_with_struct("status", status)
@@ -69,6 +71,6 @@ def save_mock_bgjob(action=None, status=None, created_before_sec=0, is_save=True
         bgjob.queue_id = queue_id
 
     if is_save:
-        bgjob.save(blocking=True)
+        bgjob.save(blocking=blocking)
 
     return bgjob

doajtest/helpers.py

@@ -19,7 +19,7 @@
 from portality.lib import paths, dates
 from portality.lib.dates import FMT_DATE_STD
 from portality.lib.thread_utils import wait_until
-from portality.tasks.redis_huey import main_queue, long_running
+from portality.tasks.redis_huey import events_queue, scheduled_short_queue, scheduled_long_queue
 from portality.util import url_for
 
 
@@ -183,8 +183,9 @@ def setUpClass(cls) -> None:
         cls.app_test.logger.setLevel(logging.DEBUG)
 
         # Run huey jobs straight away
-        main_queue.immediate = True
-        long_running.immediate = True
+        events_queue.immediate = True
+        scheduled_short_queue.immediate = True
+        scheduled_long_queue.immediate = True
 
         dao.DomainObject.save = dao_proxy(dao.DomainObject.save, type="instance")
         dao.DomainObject.delete = dao_proxy(dao.DomainObject.delete, type="instance")

doajtest/matrices/background_task_status/background_task_status.matrix.csv

@@ -0,0 +1,36 @@
+test_id,in_queue,oldest_queued,error_count,error_age,lrs_success_or_error,queued,errors,lrs
+1,0,young,0,out_of_period,complete,stable,stable,stable
+2,0,young,0,out_of_period,error,stable,stable,unstable
+3,0,young,0,out_of_period,empty,stable,stable,unstable
+4,0,young,1,in_period,complete,stable,unstable,stable
+5,0,young,1,in_period,error,stable,unstable,unstable
+6,0,young,1,out_of_period,complete,stable,stable,stable
+7,0,young,1,out_of_period,error,stable,stable,unstable
+8,1,old,0,out_of_period,complete,unstable,stable,stable
+9,1,old,0,out_of_period,error,unstable,stable,unstable
+10,1,old,0,out_of_period,empty,unstable,stable,unstable
+11,1,old,1,in_period,complete,unstable,unstable,stable
+12,1,old,1,in_period,error,unstable,unstable,unstable
+13,1,old,1,out_of_period,complete,unstable,stable,stable
+14,1,old,1,out_of_period,error,unstable,stable,unstable
+15,1,young,0,out_of_period,complete,stable,stable,stable
+16,1,young,0,out_of_period,error,stable,stable,unstable
+17,1,young,0,out_of_period,empty,stable,stable,unstable
+18,1,young,1,in_period,complete,stable,unstable,stable
+19,1,young,1,in_period,error,stable,unstable,unstable
+20,1,young,1,out_of_period,complete,stable,stable,stable
+21,1,young,1,out_of_period,error,stable,stable,unstable
+22,2,old,0,out_of_period,complete,unstable,stable,stable
+23,2,old,0,out_of_period,error,unstable,stable,unstable
+24,2,old,0,out_of_period,empty,unstable,stable,unstable
+25,2,old,1,in_period,complete,unstable,unstable,stable
+26,2,old,1,in_period,error,unstable,unstable,unstable
+27,2,old,1,out_of_period,complete,unstable,stable,stable
+28,2,old,1,out_of_period,error,unstable,stable,unstable
+29,2,young,0,out_of_period,complete,unstable,stable,stable
+30,2,young,0,out_of_period,error,unstable,stable,unstable
+31,2,young,0,out_of_period,empty,unstable,stable,unstable
+32,2,young,1,in_period,complete,unstable,unstable,stable
+33,2,young,1,in_period,error,unstable,unstable,unstable
+34,2,young,1,out_of_period,complete,unstable,stable,stable
+35,2,young,1,out_of_period,error,unstable,stable,unstable

doajtest/matrices/background_task_status/background_task_status.settings.csv

@@ -0,0 +1,17 @@
+field,test_id,in_queue,oldest_queued,error_count,error_age,lrs_success_or_error,queued,errors,lrs
+type,index,generated,generated,generated,generated,generated,conditional,conditional,conditional
+default,,,,,,,stable,stable,stable
+,,,,,,,,,
+values,,0,old,0,in_period,complete,stable,stable,stable
+values,,1,young,1,out_of_period,error,unstable,unstable,unstable
+values,,2,,,,empty,,,
+,,,,,,,,,
+constraint in_queue,,0,young,,,,,,
+constraint error_count,,,,0,out_of_period,,,,
+constraint error_count,,,,1,,!empty,,,
+,,,,,,,,,
+conditional queued,,2,,,,,unstable,,
+conditional queued,,,old,,,,unstable,,
+conditional errors,,,,1,in_period,,,unstable,
+conditional lrs,,,,,,error,,,unstable
+conditional lrs,,,,,,empty,,,unstable

doajtest/matrices/background_task_status/background_task_status.settings.json

@@ -0,0 +1,150 @@
+{
+  "parameters": [
+    {
+      "name": "test_id",
+      "type": "index"
+    },
+    {
+      "name": "in_queue",
+      "type": "generated",
+      "values": {
+        "0": {
+          "constraints": {
+            "oldest_queued": {
+              "or": [
+                "young"
+              ]
+            }
+          }
+        },
+        "1": {},
+        "2": {}
+      }
+    },
+    {
+      "name": "oldest_queued",
+      "type": "generated",
+      "values": {
+        "old": {},
+        "young": {}
+      }
+    },
+    {
+      "name": "error_count",
+      "type": "generated",
+      "values": {
+        "0": {
+          "constraints": {
+            "error_age": {
+              "or": [
+                "out_of_period"
+              ]
+            }
+          }
+        },
+        "1": {
+          "constraints": {
+            "lrs_success_or_error": {
+              "nor": [
+                "empty"
+              ]
+            }
+          }
+        }
+      }
+    },
+    {
+      "name": "error_age",
+      "type": "generated",
+      "values": {
+        "in_period": {},
+        "out_of_period": {}
+      }
+    },
+    {
+      "name": "lrs_success_or_error",
+      "type": "generated",
+      "values": {
+        "complete": {},
+        "error": {},
+        "empty": {}
+      }
+    },
+    {
+      "name": "queued",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "in_queue": {
+                "or": [
+                  "2"
+                ]
+              }
+            },
+            {
+              "oldest_queued": {
+                "or": [
+                  "old"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "errors",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "error_count": {
+                "or": [
+                  "1"
+                ]
+              },
+              "error_age": {
+                "or": [
+                  "in_period"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "lrs",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "lrs_success_or_error": {
+                "or": [
+                  "error"
+                ]
+              }
+            },
+            {
+              "lrs_success_or_error": {
+                "or": [
+                  "empty"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}