Read purchase information from BioOffice and show monthly basket stat…

…istics.

pyproject.toml

@@ -32,6 +32,7 @@ ipython = "^8.15.0"
 django-chartjs = "^2.3.0"
 unidecode = "^1.3.6"
 python-barcode = "^0.15.1"
+fabric = "^3.2.2"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.9.1"

tapir/accounts/management/commands/update_purchase_tracking_list.py

@@ -1,12 +1,12 @@
 import csv
 import os
 
-import environ
 from django.core.management import BaseCommand
 
 from tapir import settings
 from tapir.accounts.models import TapirUser
 from tapir.settings import GROUP_VORSTAND
+from tapir.utils.shortcuts import setup_ssh_for_biooffice_storage
 from tapir.utils.user_utils import UserUtils
 
 
@@ -59,20 +59,7 @@ def send_file_to_server(cls):
             )
             return
 
-        env = environ.Env()
-
-        os.system("mkdir -p ~/.ssh")
-        os.system(
-            f'bash -c \'echo -e "{env("TAPIR_SSH_KEY_PRIVATE")}" > ~/.ssh/biooffice_id_rsa\''
-        )
-        os.system("chmod u=rw,g=,o= ~/.ssh/biooffice_id_rsa")
-        os.system(
-            f'bash -c \'echo -e "{env("TAPIR_SSH_KEY_PUBLIC")}" > ~/.ssh/biooffice_id_rsa.pub\''
-        )
-        os.system("chmod u=rw,g=r,o=r ~/.ssh/biooffice_id_rsa.pub")
-        os.system(
-            f'bash -c \'echo -e "{env("BIOOFFICE_SERVER_SSH_KEY_FINGERPRINT")}" > ~/.ssh/biooffice_known_hosts\''
-        )
+        setup_ssh_for_biooffice_storage()
         os.system(
             f"scp -o 'NumberOfPasswordPrompts=0' -o 'UserKnownHostsFile=~/.ssh/biooffice_known_hosts' -i ~/.ssh/biooffice_id_rsa -P 23 {cls.FILE_NAME} [email protected]:./"
         )

tapir/accounts/templates/accounts/user_detail.html

@@ -9,6 +9,7 @@
 {% load accounts %}
 {% load core %}
 {% load utils %}
+{% load statistics %}
 
 {% block head %}
     {{ block.super }}
@@ -164,6 +165,12 @@ <h5 class="card-header d-flex justify-content-between align-items-center">
         <div class="col-xl-6">
             {% purchase_tracking_card tapir_user=object %}
         </div>
+
+        {% if tapir_user.share_owner and tapir_user.allows_purchase_tracking %}
+            <div class="col-xl-6">
+                {% purchase_statistics_card tapir_user=object %}
+            </div>
+        {% endif %}
     </div>
 
 {% endblock %}

tapir/settings.py

@@ -165,6 +165,10 @@
         "task": "tapir.shifts.tasks.run_freeze_checks",
         "schedule": celery.schedules.crontab(minute=0, hour=1),
     },
+    "process_purchase_files": {
+        "task": "tapir.statistics.tasks.process_purchase_files",
+        "schedule": celery.schedules.crontab(minute=0, hour=2),
+    },
 }
 
 # Password validation

tapir/statistics/management/commands/process_purchase_files.py

@@ -0,0 +1,77 @@
+import csv
+import datetime
+import fnmatch
+import io
+from typing import Dict
+
+import environ
+import paramiko
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.utils import timezone
+from fabric import Connection
+from paramiko.sftp_file import SFTPFile
+
+from tapir.accounts.models import TapirUser
+from tapir.statistics.models import ProcessedPurchaseFiles, PurchaseBasket
+from tapir.utils.shortcuts import get_timezone_aware_datetime
+
+
+class Command(BaseCommand):
+    help = "If a new cycle has started, remove one shift point from all active members."
+
+    def handle(self, *args, **options):
+        env = environ.Env()
+        private_key = paramiko.RSAKey.from_private_key(
+            io.StringIO(env("TAPIR_SSH_KEY_PRIVATE"))
+        )
+        connection = Connection(
+            host="u326634-sub6.your-storagebox.de",
+            user="u326634-sub6",
+            connect_kwargs={"pkey": private_key},
+        )
+        sftp_client = connection.sftp()
+        ProcessedPurchaseFiles.objects.all().delete()
+        for file_name in fnmatch.filter(
+            sftp_client.listdir(), "Statistics_Members_*.csv"
+        ):
+            if ProcessedPurchaseFiles.objects.filter(file_name=file_name).exists():
+                continue
+            self.process_file(sftp_client.open(file_name), file_name)
+
+    @classmethod
+    @transaction.atomic
+    def process_file(cls, file: SFTPFile, file_name: str):
+        source_file = ProcessedPurchaseFiles.objects.create(
+            file_name=file_name[: ProcessedPurchaseFiles.MAX_FILE_NAME_LENGTH],
+            processed_on=timezone.now(),
+        )
+        for row in csv.DictReader(file, delimiter=",", quotechar='"'):
+            row: Dict
+            purchase_date = get_timezone_aware_datetime(
+                date=datetime.datetime.strptime(
+                    row['\ufeff"Datum"'], "%Y-%m-%d"
+                ).date(),
+                time=datetime.datetime.strptime(row["Zeit"], "%H:%M:%S").time(),
+            )
+
+            tapir_user = (
+                TapirUser.objects.filter(id=int(row["Kunde"][3:])).first()
+                if row["Kunde"].isnumeric() and len(row["Kunde"]) > 3
+                else None
+            )
+            PurchaseBasket.objects.create(
+                source_file=source_file,
+                purchase_date=purchase_date,
+                cashier=row["cKasse"],
+                purchase_counter=row["Bon"],
+                tapir_user=tapir_user,
+                gross_amount=cls.parse_german_number(row["VKBrutto_SUM"]),
+                first_net_amount=cls.parse_german_number(row["VKNetto_SUM"]),
+                second_net_amount=cls.parse_german_number(row["EKNetto_SUM"]),
+                discount=cls.parse_german_number(row["Rabatt_SUM"]),
+            )
+
+    @staticmethod
+    def parse_german_number(string: str) -> float:
+        return float(string.replace(",", "."))

tapir/statistics/migrations/0001_initial.py

@@ -0,0 +1,67 @@
+# Generated by Django 3.2.21 on 2023-10-20 13:41
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ProcessedPurchaseFiles",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("file_name", models.CharField(max_length=255)),
+                ("processed_on", models.DateTimeField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name="PurchaseBasket",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("purchase_date", models.DateTimeField()),
+                ("cashier", models.IntegerField()),
+                ("purchase_counter", models.IntegerField()),
+                ("gross_amount", models.FloatField()),
+                ("first_net_amount", models.FloatField()),
+                ("second_net_amount", models.FloatField()),
+                ("discount", models.FloatField()),
+                (
+                    "source_file",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="statistics.processedpurchasefiles",
+                    ),
+                ),
+                (
+                    "tapir_user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+    ]

tapir/statistics/migrations/0002_alter_purchasebasket_tapir_user.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.21 on 2023-10-20 16:09
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("statistics", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="purchasebasket",
+            name="tapir_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

tapir/statistics/migrations/0003_alter_purchasebasket_tapir_user.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.21 on 2023-10-20 16:24
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("statistics", "0002_alter_purchasebasket_tapir_user"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="purchasebasket",
+            name="tapir_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

tapir/statistics/models.py

@@ -1 +1,25 @@
 # Create your models here.
+from django.db import models
+
+from tapir.accounts.models import TapirUser
+
+
+class ProcessedPurchaseFiles(models.Model):
+    MAX_FILE_NAME_LENGTH = 255
+
+    file_name = models.CharField(max_length=255)
+    processed_on = models.DateTimeField()
+
+
+class PurchaseBasket(models.Model):
+    source_file = models.ForeignKey(ProcessedPurchaseFiles, on_delete=models.CASCADE)
+    purchase_date = models.DateTimeField()  # Datum & Zeit
+    cashier = models.IntegerField()  # cKasse (column names from the CSV source file)
+    purchase_counter = models.IntegerField()  # Bon
+    tapir_user = models.ForeignKey(
+        TapirUser, on_delete=models.SET_NULL, null=True
+    )  # Kunde
+    gross_amount = models.FloatField()  # VKBrutto_SUM
+    first_net_amount = models.FloatField()  # VKNetto_SUM
+    second_net_amount = models.FloatField()  # EKNetto_SUM
+    discount = models.FloatField()  # Rabatt_SUM

tapir/statistics/tasks.py

@@ -0,0 +1,7 @@
+from celery import shared_task
+from django.core.management import call_command
+
+
+@shared_task
+def process_purchase_files():
+    call_command("process_purchase_files")

tapir/statistics/templates/statistics/main_statistics.html

@@ -58,13 +58,33 @@ <h5 class="card-header">{% translate "Targets for break-even" %}</h5>
                 <ul class="list-group list-group-flush">
                     <li class="list-group-item">
                         <h5>{% translate 'Shopping basket' %}</h5>
-                        {% blocktranslate with target_basket=225 %}
-                            The current target food basket value per member and per month to reach the break-even is
-                            {{ target_basket }}€.
-                        {% endblocktranslate %}
+                        <p>
+                            {% blocktranslate with target_basket=target_average_monthly_basket %}
+                                The current target food basket value per member and per month to reach the break-even is
+                                {{ target_basket }}€.
+                            {% endblocktranslate %}
+                        </p>
                     </li>
-                </ul>
-                <ul class="list-group list-group-flush">
+                    {% if perms.coop.manage %}
+                        <li class="list-group-item text-bg-warning">
+                            <h5>WIP, visible for member office only</h5>
+                            <p>
+                                <a class="{% tapir_button_action %}">Update purchase data from BioOffice</a>
+                            </p>
+                            <p>
+                                {% blocktranslate with average_basket=current_average_monthly_basket %}
+                                    The current average food basket value per member and per month is
+                                    {{ current_average_monthly_basket }}€.
+                                {% endblocktranslate %}
+
+                            </p>
+                            <div class="progress" style="height: 25px;">
+                                <div class="progress-bar" role="progressbar"
+                                     style="width: {{ progress_monthly_basket }}%; ">{{ current_average_monthly_basket }}€
+                                </div>
+                            </div>
+                        </li>
+                    {% endif %}
                     <li class="list-group-item">
                         <h5>{% translate 'Members eligible to purchase' %}</h5>
                         <p>
@@ -200,7 +220,8 @@ <h5 class="card-header">{{ campaigns.0.name }}</h5>
                                 </p>
                                 {% if perms.coop.admin %}
                                     <p>
-                                        (Visible only to the Vorstand) the current number of extra shares is {{ extra_shares }}.
+                                        (Visible only to the Vorstand) the current number of extra shares
+                                        is {{ extra_shares }}.
                                     </p>
                                 {% endif %}
                             </li>

tapir/statistics/templates/statistics/tags/purchase_statistics_card.html

@@ -0,0 +1,34 @@
+{% load i18n %}
+{% load shifts %}
+{% load core %}
+
+<div class="card mb-2">
+    <h5 class="card-header d-flex justify-content-between align-items-center">
+        {% translate "Purchases" %}
+    </h5>
+    <ul class="list-group  list-group-flush">
+        <li class="list-group-item">
+            {% blocktranslate with average=average_basket_per_month %}
+                Your average basket per month is {{ average }}€.
+            {% endblocktranslate %}
+        </li>
+        <li class="list-group-item">
+            <table class="{% tapir_table_classes %}" aria-label="{% translate 'List of the last purchases' %}">
+                <thead>
+                <tr>
+                    <th>{% translate 'Date' %}</th>
+                    <th>{% translate 'Gross amount' %}</th>
+                </tr>
+                </thead>
+                <tbody>
+                {% for purchase in last_purchases %}
+                    <tr>
+                        <td>{{ purchase.purchase_date|date:"d.m.Y H:i" }}</td>
+                        <td>{{ purchase.gross_amount }}€</td>
+                    </tr>
+                {% endfor %}
+                </tbody>
+            </table>
+        </li>
+    </ul>
+</div>