Merge remote-tracking branch 'origin/main' into topic/vlad/gh_actions

Makefile

@@ -48,7 +48,7 @@ coverage.xml: pytest behave-all behave-translator
 .Phony: ci-test
 ci-test: | toggle-local build migrate run coverage.xml
 
-## cleanup: remove local containers and volumes
+## clean: remove local containers and volumes
 .Phony: clean
 clean: docker-compose.yaml
 	@docker compose rm -f -s

config/api_router.py

@@ -1,11 +1,6 @@
 from rest_framework.routers import DefaultRouter
 
-from scram.route_manager.api.views import (
-    ActionTypeViewSet,
-    ClientViewSet,
-    EntryViewSet,
-    IgnoreEntryViewSet,
-)
+from scram.route_manager.api.views import ActionTypeViewSet, ClientViewSet, EntryViewSet, IgnoreEntryViewSet
 from scram.users.api.views import UserViewSet
 
 router = DefaultRouter()

config/consumers.py

@@ -37,6 +37,8 @@ async def _send_event(self, event):
     translator_add = _send_event
     # Tell all translators of this actiontype of a withdrawal of a route.
     translator_remove = _send_event
+    # Tell all translators of this actiontype to withdraw ALL routes.
+    translator_remove_all = _send_event
     # Send a query to all translators if a route is announced.
     translator_check = _send_event
 

local.yml

@@ -70,6 +70,23 @@ services:
       - "7000"
     command: /start-docs
 
+  docs:
+    image: scram_local_docs
+    build:
+      context: .
+      dockerfile: ./compose/local/docs/Dockerfile
+    env_file:
+      - ./.envs/.local/.django
+    networks:
+      default: {}
+    volumes:
+      - $CI_PROJECT_DIR/docs:/docs:z
+      - $CI_PROJECT_DIR/config:/app/config:z
+      - $CI_PROJECT_DIR/scram:/app/scram:z
+    ports:
+      - "7000"
+    command: /start-docs
+
   redis:
     image: redis:5.0
     sysctls:

scram/route_manager/admin.py

@@ -1,7 +1,7 @@
 from django.contrib import admin
 from simple_history.admin import SimpleHistoryAdmin
 
-from .models import ActionType, Client, Entry, IgnoreEntry, Route
+from .models import ActionType, Client, Entry, IgnoreEntry, Route, WebSocketMessage, WebSocketSequenceElement
 
 
 @admin.register(ActionType)
@@ -14,3 +14,5 @@ class ActionTypeAdmin(SimpleHistoryAdmin):
 admin.site.register(IgnoreEntry, SimpleHistoryAdmin)
 admin.site.register(Route)
 admin.site.register(Client)
+admin.site.register(WebSocketMessage)
+admin.site.register(WebSocketSequenceElement)

scram/route_manager/api/views.py

@@ -12,14 +12,9 @@
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 
-from ..models import ActionType, Client, Entry, IgnoreEntry
+from ..models import ActionType, Client, Entry, IgnoreEntry, WebSocketSequenceElement
 from .exceptions import ActiontypeNotAllowed, IgnoredRoute, PrefixTooLarge
-from .serializers import (
-    ActionTypeSerializer,
-    ClientSerializer,
-    EntrySerializer,
-    IgnoreEntrySerializer,
-)
+from .serializers import ActionTypeSerializer, ClientSerializer, EntrySerializer, IgnoreEntrySerializer
 
 channel_layer = get_channel_layer()
 
@@ -106,11 +101,17 @@ def perform_create(self, serializer):
             logging.info(f"Cannot proceed adding {route}. The ignore list contains {ignore_entries}")
             raise IgnoredRoute
         else:
-            # Must match a channel name defined in asgi.py
-            async_to_sync(channel_layer.group_send)(
-                f"translator_{actiontype}",
-                {"type": "translator_add", "message": {"route": str(route)}},
-            )
+            elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num")
+            if not elements:
+                logging.warning(f"No elements found for actiontype={actiontype}.")
+
+            for element in elements:
+                msg = element.websocketmessage
+                msg.msg_data[msg.msg_data_route_field] = str(route)
+                # Must match a channel name defined in asgi.py
+                async_to_sync(channel_layer.group_send)(
+                    f"translator_{actiontype}", {"type": msg.msg_type, "message": msg.msg_data}
+                )
 
             serializer.save()
 

scram/route_manager/migrations/0027_websocketmessage_websocketsequenceelement.py

@@ -0,0 +1,56 @@
+# Generated by Django 3.2.13 on 2024-01-06 21:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("route_manager", "0026_alter_client_hostname"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="WebSocketMessage",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("msg_type", models.CharField(max_length=50, verbose_name="The type of the message")),
+                (
+                    "msg_data",
+                    models.JSONField(default=dict, verbose_name="The JSON payload. See also msg_data_route_field."),
+                ),
+                (
+                    "msg_data_route_field",
+                    models.CharField(
+                        max_length=25,
+                        verbose_name="The key in the JSON payload whose value will contain the route being acted on.",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="WebSocketSequenceElement",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                (
+                    "order_num",
+                    models.SmallIntegerField(
+                        default=0,
+                        verbose_name="Sequences are sent from the smallest order_num to the highest. Messages with the same order_num could be sent in any order",
+                    ),
+                ),
+                ("verb", models.CharField(choices=[("A", "Add"), ("C", "Check"), ("R", "Remove")], max_length=1)),
+                (
+                    "action_type",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="route_manager.actiontype"),
+                ),
+                (
+                    "websocketmessage",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="route_manager.websocketmessage"
+                    ),
+                ),
+            ],
+        ),
+    ]

scram/route_manager/migrations/0028_default_websocket_messages.py

@@ -0,0 +1,25 @@
+from django.db import migrations
+
+
+def create(apps, schema_editor):
+    ActionType = apps.get_model("route_manager", "ActionType")
+    at = ActionType.objects.get(name="block")
+
+    WebSocketMessage = apps.get_model("route_manager", "WebSocketMessage")
+    wsm = WebSocketMessage(msg_type="translator_add", msg_data_route_field="route")
+    wsm.save()
+
+    WebSocketSequenceElement = apps.get_model("route_manager", "WebSocketSequenceElement")
+    wsse = WebSocketSequenceElement(websocketmessage=wsm, verb="A", action_type=at)
+    wsse.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("route_manager", "0027_websocketmessage_websocketsequenceelement"),
+    ]
+
+    operations = [
+        migrations.RunPython(create),
+    ]

scram/route_manager/migrations/0029_alter_websocketmessage_msg_data_route_field.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.13 on 2024-01-09 16:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("route_manager", "0028_default_websocket_messages"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="websocketmessage",
+            name="msg_data_route_field",
+            field=models.CharField(
+                default="route",
+                max_length=25,
+                verbose_name="The key in the JSON payload whose value will contain the route being acted on.",
+            ),
+        ),
+    ]

scram/route_manager/models.py

@@ -35,6 +35,47 @@ def __str__(self):
         return self.name
 
 
+class WebSocketMessage(models.Model):
+    """Defines a single message sent to downstream translators via WebSocket."""
+
+    msg_type = models.CharField("The type of the message", max_length=50)
+    msg_data = models.JSONField("The JSON payload. See also msg_data_route_field.", default=dict)
+    msg_data_route_field = models.CharField(
+        "The key in the JSON payload whose value will contain the route being acted on.",
+        default="route",
+        max_length=25,
+    )
+
+    def __str__(self):
+        return f"{self.msg_type}: {self.msg_data} with the route in key {self.msg_data_route_field}"
+
+
+class WebSocketSequenceElement(models.Model):
+    """In a sequence of messages, defines a single element."""
+
+    websocketmessage = models.ForeignKey("WebSocketMessage", on_delete=models.CASCADE)
+    order_num = models.SmallIntegerField(
+        "Sequences are sent from the smallest order_num to the highest. "
+        + "Messages with the same order_num could be sent in any order",
+        default=0,
+    )
+
+    VERB_CHOICES = [
+        ("A", "Add"),
+        ("C", "Check"),
+        ("R", "Remove"),
+    ]
+    verb = models.CharField(max_length=1, choices=VERB_CHOICES)
+
+    action_type = models.ForeignKey("ActionType", on_delete=models.CASCADE)
+
+    def __str__(self):
+        return (
+            f"{self.websocketmessage} as order={self.order_num} for "
+            + f"{self.verb} actions on actiontype={self.action_type}"
+        )
+
+
 class Entry(models.Model):
     """An instance of an action taken on a route."""
 

scram/route_manager/tests/acceptance/features/add_automated_block_entry.feature

@@ -2,11 +2,12 @@ Feature: an automated source adds a block entry
   Automated clients (eg zeek) can add v4/v6 block entries
 
   Scenario: unauthenticated users get a 403
-    When we add the entry 127.0.0.1
+    When we add the entry 192.0.2.132
     Then we get a 403 status code
 
   Scenario Outline: add a block entry
-    Given a client with block authorization
+    Given a block actiontype is defined
+    And a client with block authorization
     When we're logged in
     And  we add the entry <ip>
     And  we list the entrys
@@ -18,21 +19,21 @@ Feature: an automated source adds a block entry
 
     Examples: v4 IPs
       | ip          | cidr           |
-      | 1.2.3.4     | 1.2.3.4/32     |
-      | 193.168.0.0 | 193.168.0.0/32 |
+      | 192.0.2.128 | 192.0.2.128/32 |
+      | 192.0.2.129 | 192.0.2.129/32 |
 
     Examples: v6 IPs
-      | ip     | cidr       |
-      | 2000:: | 2000::/128 |
-      | ::1    | ::1/128    |
+      | ip     | cidr                         |
+      | 2001:DB8:94BB::94BC  | 2001:DB8:94BB::94BC/128  |
+      | 2001:DB8:94BC:: | 2001:DB8:94BC::/128 |
 
   @history
   Scenario: add a block entry with a comment
     Given a client with block authorization
     When we're logged in
-    And  we add the entry 127.0.0.2 with comment it's coming from inside the house
+    And  we add the entry 192.0.2.133 with comment it's coming from inside the house
     Then we get a 201 status code
-    And  the change entry for 127.0.0.2 is it's coming from inside the house
+    And  the change entry for 192.0.2.133 is it's coming from inside the house
 
   Scenario Outline: add a block entry multiple times and it's accepted
     Given a client with block authorization
@@ -44,11 +45,11 @@ Feature: an automated source adds a block entry
     And the number of entrys is 1
 
     Examples: IPs
-      | ip          |
-      | 1.2.3.4     |
-      | 193.168.0.0 |
-      | 2000::      |
-      | ::1         |
+      | ip              |
+      | 192.0.2.130     |
+      | 198.51.100.130  |
+      | 2001:DB8:94BD::94BD  |
+      | 2001:DB8:94BE:: |
 
   Scenario Outline: invalid block entries can't be added
     Given a client with block authorization
@@ -69,7 +70,8 @@ Feature: an automated source adds a block entry
       | 2000::/129  |
 
   Scenario Outline: add a block entry as a cidr address
-    Given a client with block authorization
+    Given a block actiontype is defined
+    And a client with block authorization
     When we're logged in
     And the CIDR prefix limits are 8 and 32
     And we add the entry <ip>
@@ -80,7 +82,7 @@ Feature: an automated source adds a block entry
 
     Examples:
       | ip                 |
-      | 1.2.3.4/32         |
-      | 10.1.0.0/16        |
-      | 2000::1/128        |
-      | 2001:4f8:3:ba::/64 |
+      | 192.0.2.131/32     |
+      | 198.51.100.160/29  |
+      | 2001:DB8:94BA::94BD/128 |
+      | 2001:DB8:94BE::/64 |

scram/route_manager/tests/acceptance/features/client.feature

@@ -7,11 +7,11 @@ Feature: We can register and use clients
   Scenario: We can add a block using an authorized client
     Given a client with block authorization
     When we're logged in
-    And we add the entry 1.2.3.4
+    And we add the entry 192.0.2.216
     Then we get a 201 status code
 
   Scenario: We can't block with an unauthorized client even if we are logged in
     Given a client without block authorization
     When we're logged in
-    And we add the entry 1.2.3.4
+    And we add the entry 192.0.2.217
     Then we get a 403 status code