Merge branch 'integrate-quotas' into 'master'

Integrate quotas

Closes #376

See merge request bp-barberini/bp-barberini!377

docker/Dockerfile

@@ -35,6 +35,8 @@ RUN apt-get update
 RUN $INSTALL postgresql-client-12
 
 # Install python
+# NB: We cannot upgrade to python3.7 or something newer at the moment because
+# psycopg2 seems to be incompatible.
 RUN $INSTALL python3.6 python3-pip python3-setuptools python3-dev \
   python3-wheel
 

docker/requirements.txt

@@ -38,12 +38,15 @@ sklearn==0.0
 stop-words==2018.7.23
 
 # Helpers
+backports-datetime-fromisoformat==1.0.0
 bs4==0.0.1
 dateparser==0.7.6
+js2py==0.70
 jsonpickle==1.4.1
 jstyleson==0.0.2
 lockfile==0.12.2
 mmh3==2.5.1
+nptime==1.1
 numpy==1.19.2
 oauth2client==4.1.3
 pandas==1.1.3

scripts/migrations/migration_052.sql

@@ -0,0 +1,25 @@
+-- Integrate gomus quotas and capacities (!377)
+
+BEGIN;
+
+    CREATE TABLE gomus_quota (
+        quota_id INT PRIMARY KEY,
+        name TEXT,
+        creation_date TIMESTAMP,
+        update_date TIMESTAMP
+    );
+
+    CREATE TABLE gomus_capacity (
+        quota_id INT REFERENCES gomus_quota,
+        date DATE,
+        time TIME,
+        max INT,
+        sold INT,
+        reserved INT,
+        available INT,
+        CHECK (max - sold - reserved = available),
+        last_updated TIMESTAMP,
+        PRIMARY KEY (quota_id, date, time)
+    );
+
+COMMIT;

src/_utils/__init__.py

@@ -3,8 +3,20 @@
 
 logger = logging.getLogger('luigi-interface')
 
-minimal_mode = os.getenv('MINIMAL') == 'True'
-OUTPUT_DIR = os.environ['OUTPUT_DIR']
+
+def minimal_mode() -> bool:
+    """
+    Answer a flag indicating whether task should process minimum of data only.
+
+    For more information on the minimal mining pipeline, see !102 and !109.
+    """
+    return os.getenv('MINIMAL') == 'True'
+
+
+def output_dir() -> str:
+    """Answer the path to the root of the output directory for all tasks."""
+    return os.environ['OUTPUT_DIR']
+
 
 from .utils import ObjectParameter, StreamToLogger             # noqa: E402
 from .data_preparation import ConcatCsvs, DataPreparationTask  # noqa: E402
@@ -25,5 +37,5 @@
     MuseumFacts,
     ObjectParameter, StreamToLogger,
 
-    db_connector
+    db_connector, minimal_mode, output_dir
 ]

src/_utils/_database.py

@@ -207,8 +207,8 @@ def _execute_query(
         assert not args or not kwargs, "cannot combine args and kwargs"
         all_args = next(
             filter(bool, [args, kwargs]),
-            # always pass args for consistent
-            # resolution of percent escapings
+            # always pass any args for consistent resolution of percent
+            # escapings
             None
         )
 

src/_utils/data_preparation.py

@@ -27,14 +27,14 @@ def __init__(self, *args, **kwargs):
         default=None)
 
     minimal_mode = luigi.parameter.BoolParameter(
-        default=_utils.minimal_mode,
+        default=_utils.minimal_mode(),
         description="If True, only a minimal amount of data will be prepared"
                     "in order to test the pipeline for structural problems")
 
     @property
     def output_dir(self):
 
-        return _utils.OUTPUT_DIR
+        return _utils.output_dir()
 
     def condense_performance_values(
             self,

src/_utils/database.py

@@ -25,7 +25,7 @@ class CsvToDb(CopyToTable):
     """
 
     minimal_mode = luigi.parameter.BoolParameter(
-        default=_utils.minimal_mode,
+        default=_utils.minimal_mode(),
         description="If True, only a minimal amount of data will be prepared"
                     "in order to test the pipeline for structural problems")
 
@@ -71,7 +71,7 @@ def __init__(self, *args, **kwargs):
     }
 
     """
-    Conversion functions to be applied before generating the ouput CSV for
+    Conversion functions to be applied before generating the output CSV for
     postgres.
     """
     converters_out = {

src/_utils/utils.py

@@ -7,6 +7,9 @@
 import jsonpickle
 import luigi
 
+from backports.datetime_fromisoformat import MonkeyPatch
+MonkeyPatch.patch_fromisoformat()
+
 
 class ObjectParameter(luigi.Parameter):
     """A luigi parameter that takes an arbitrary object."""

src/diagnostics/__init__.py

@@ -4,7 +4,7 @@
 
 import luigi
 
-from _utils import minimal_mode, OUTPUT_DIR
+from _utils import minimal_mode, output_dir
 from .log_report import SendLogReport
 
 
@@ -13,13 +13,13 @@ class Diagnostics(luigi.Task):
 
     def output(self):
 
-        return luigi.LocalTarget(f'{OUTPUT_DIR}/diagnostics.txt')
+        return luigi.LocalTarget(f'{output_dir()}/diagnostics.txt')
 
     def run(self):
 
         tasks = 0
 
-        if dt.date.today().isoweekday() == 7 or minimal_mode:  # sunday
+        if dt.date.today().isoweekday() == 7 or minimal_mode():  # sunday
             yield SendLogReport()
             tasks += 1
 

src/diagnostics/log_report.py

@@ -11,7 +11,7 @@
 import pandas as pd
 from tqdm import tqdm
 
-from _utils import utils, OUTPUT_DIR
+from _utils import utils, output_dir
 
 
 # Log patterns ---
@@ -98,7 +98,7 @@ def requires(self):
 
     def output(self):
 
-        return luigi.LocalTarget(f'{OUTPUT_DIR}/SendLogReport')
+        return luigi.LocalTarget(f'{output_dir()}/SendLogReport')
 
     def run(self):
 
@@ -148,7 +148,7 @@ class CollectLogReport(luigi.Task):
 
     def output(self):
 
-        return luigi.LocalTarget(f'{OUTPUT_DIR}/log_report.csv')
+        return luigi.LocalTarget(f'{output_dir()}/log_report.csv')
 
     def run(self):
 

src/extended_twitter_collection/keyword_intervals.py

@@ -13,7 +13,7 @@
 import numpy as np
 import pandas as pd
 from stop_words import get_stop_words
-from tqdm.auto import tqdm
+from tqdm import tqdm
 
 from _utils import DataPreparationTask, CsvToDb
 from twitter import TweetsToDb

src/gomus/__init__.py

@@ -3,12 +3,14 @@
 import luigi
 
 from .bookings import BookingsToDb
+from .capacities import CapacitiesToDb
 from .customers import CustomersToDb, GomusToCustomerMappingToDb
 from .daily_entries import DailyEntriesToDb, ExpectedDailyEntriesToDb
 from .exhibitions import ExhibitionTimesToDb
 from .events import EventsToDb
 from .order_contains import OrderContainsToDb
 from .orders import OrdersToDb
+from .quotas import QuotasToDb
 
 
 class GomusToDb(luigi.WrapperTask):
@@ -32,3 +34,6 @@ def requires(self):
             yield GomusToCustomerMappingToDb()
             yield OrderContainsToDb()
             yield OrdersToDb()
+
+            yield CapacitiesToDb()
+            yield QuotasToDb()

src/gomus/_utils/cleanse_data.py

@@ -7,7 +7,7 @@
 from luigi.format import UTF8
 import pandas as pd
 import pgeocode
-from tqdm.auto import tqdm
+from tqdm import tqdm
 
 from _utils import DataPreparationTask, logger
 from .extract_customers import ExtractCustomerData

src/gomus/_utils/edit_report.py

@@ -7,7 +7,7 @@
 import requests
 from bs4 import BeautifulSoup
 
-from _utils import OUTPUT_DIR
+from _utils import output_dir
 from gomus._utils.fetch_report_helper import REPORT_IDS
 
 # These lists map directly to various Gomus attributes used for editing
@@ -68,7 +68,7 @@ def __init__(self, *args, **kwargs):
     # obsoletes output() and requires()
     def output(self):
         return luigi.LocalTarget(
-            f'{OUTPUT_DIR}/gomus/edit_report_{self.task_id}')
+            f'{output_dir()}/gomus/edit_report_{self.task_id}')
 
     def run(self):
         self.add_to_body(f'_method={METHOD}')