From eff8221198d5a24a9536536ef8f412e7c80c3957 Mon Sep 17 00:00:00 2001 From: jarosevcik Date: Wed, 13 Oct 2021 11:57:00 +0200 Subject: [PATCH] feat(entity): import services and libraries into entity model --- test/globalsearch/test_indexer.py | 4 +- zoo/entities/management/__init__.py | 0 zoo/entities/management/commands/__init__.py | 0 .../management/commands/import_entities.py | 110 ++++++++++++++++++ ...0002_allow_null_product_owner_on_group.py} | 0 .../migrations/0003_allow_blank_on_group.py | 24 ++++ zoo/entities/models.py | 3 +- 7 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 zoo/entities/management/__init__.py create mode 100644 zoo/entities/management/commands/__init__.py create mode 100644 zoo/entities/management/commands/import_entities.py rename zoo/entities/migrations/{0002_auto_20211007_1000.py => 0002_allow_null_product_owner_on_group.py} (100%) create mode 100644 zoo/entities/migrations/0003_allow_blank_on_group.py diff --git a/test/globalsearch/test_indexer.py b/test/globalsearch/test_indexer.py index 0f4e5034..79dfd7a7 100644 --- a/test/globalsearch/test_indexer.py +++ b/test/globalsearch/test_indexer.py @@ -11,9 +11,7 @@ def _initialize(client, models): for model, index in models: - client.create_index( - uid=index, options={"name": model.__name__, "primaryKey": "id"} - ) + client.create_index(uid=index, options={"primaryKey": "id"}) def test_index_model(): diff --git a/zoo/entities/management/__init__.py b/zoo/entities/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zoo/entities/management/commands/__init__.py b/zoo/entities/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zoo/entities/management/commands/import_entities.py b/zoo/entities/management/commands/import_entities.py new file mode 100644 index 00000000..69b12d1d --- /dev/null +++ b/zoo/entities/management/commands/import_entities.py @@ -0,0 +1,110 @@ +import structlog +from django.core.management.base import BaseCommand + +from zoo.entities.enums import Kind +from zoo.entities.models import Entity, Link +from zoo.libraries.models import Library +from zoo.services.models import Service + +log = structlog.get_logger() + + +class Command(BaseCommand): + help = "Create entities from services and libraries" + + def handle(self, *args, **options): + self.service_to_entity() + self.library_to_entity() + + @staticmethod + def service_to_entity(): + with open("service_exceptions.csv", "w+") as exceptions: + exceptions.write( + "Service name;Service owner;Service repository;Exception\n" + ) + for service in Service.objects.all(): + if service.repository: + log.info("entity.import_services.processing", service=service) + try: + entity = Entity.objects.create( + name=service.name, + owner=service.owner, + description=service.description, + kind=Kind.COMPONENT, + type="service", + source=service.repository, + tags=service.tags, + service=service, + ) + + if service.docs_url: + Link.objects.create( + name="Documentation", + url=service.docs_url, + entity=entity, + ) + + if service.slack_channel: + Link.objects.create( + name="Discussion", + url=f"https://app.slack.com/client/T024Z3H2Y/{service.slack_channel}", + entity=entity, + ) + except Exception as err: + exceptions.write( + f"{getattr(service, 'name', '')};{getattr(service, 'owner', '')};{getattr(service, 'repository', '')};{repr((err))}\n" + ) + else: + exceptions.write( + f"{getattr(service, 'name', '')};{getattr(service, 'owner', '')};;Missing repository for service\n" + ) + + @staticmethod + def library_to_entity(): + with open("library_exceptions.csv", "w+") as exceptions: + exceptions.write( + "Library name;Library owner;Library repository;Exception\n" + ) + for library in Library.objects.all(): + if library.repository: + log.info("entity.import_libraries.processing", library=library) + try: + entity = Entity.objects.create( + name=library.name, + owner=library.owner, + description=library.description, + kind=Kind.COMPONENT, + type="library", + source=library.repository, + tags=library.tags, + library=library, + ) + + if library.docs_url: + Link.objects.create( + name="Documentation", + url=library.docs_url, + entity=entity, + ) + + if library.slack_channel: + Link.objects.create( + name="Discussion", + url=f"https://app.slack.com/client/T024Z3H2Y/{library.slack_channel}", + entity=entity, + ) + + if library.library_url: + Link.objects.create( + name="Library URL", + url=library.library_url, + entity=entity, + ) + except Exception as err: + exceptions.write( + f"{getattr(library, 'name', '')};{getattr(library, 'owner', '')};{getattr(library, 'repository', '')};{repr((err))}\n" + ) + else: + exceptions.write( + f"{getattr(library, 'name', '')};{getattr(library, 'owner', '')};;Missing repository for library\n" + ) diff --git a/zoo/entities/migrations/0002_auto_20211007_1000.py b/zoo/entities/migrations/0002_allow_null_product_owner_on_group.py similarity index 100% rename from zoo/entities/migrations/0002_auto_20211007_1000.py rename to zoo/entities/migrations/0002_allow_null_product_owner_on_group.py diff --git a/zoo/entities/migrations/0003_allow_blank_on_group.py b/zoo/entities/migrations/0003_allow_blank_on_group.py new file mode 100644 index 00000000..39075ade --- /dev/null +++ b/zoo/entities/migrations/0003_allow_blank_on_group.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.19 on 2021-10-13 09:52 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("entities", "0002_allow_null_product_owner_on_group"), + ] + + operations = [ + migrations.AlterField( + model_name="entity", + name="group", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="entities.Group", + ), + ), + ] diff --git a/zoo/entities/models.py b/zoo/entities/models.py index c7c612e5..8013a114 100644 --- a/zoo/entities/models.py +++ b/zoo/entities/models.py @@ -57,7 +57,8 @@ class Meta: group = models.OneToOneField( "entities.Group", on_delete=models.CASCADE, - default=None, + null=True, + blank=True, ) source = models.ForeignKey( "repos.Repository",