task/DES-1828 Streetview (#46)

* Basic authentication for streetview services.

* Basic uploading

* Implement api for exposing mapillary sequences and fix file transfer.

* Improve mapillary uploads.

* Fix streetview tests

* Add progress notification tests.

* Upload sequence typo.

* Fix notification logs.

* Fix streetview sequence update request

* Add organization uploads.

* Add kube variable for mapillary.

* Add progress cleanup after aborting on error.

* Fix organization temporarily.

* Refactor and fix.

* Fixes.

* Resolve merge conflict.

* Fix notification route delete test.

* Add better logs for streetview related exceptions.

* Port to mapillary v4 and cleanup.

* Adjust changes to mapillary_tools.

* Fix comments and api docs

* New Migrations

* Revert changes for migration.

* Revert unnecessary changes.

* New migrations.

* string env variable

* Delete streetview if it exists.

* Remove uniqueness constraint from streetview service name

* Add merge migration

* Handle different mapillary keys and add project link for streetview instances.

* Make streetview sequences map features.

* Fix kube/Makefile.

* Add surrounding quotes for mapillary env vars.

* Asset uuid for streetview features.

* Use correct sequence_id for creating feature assets.

* Make add organization_id field for streetivew sequences.

* Remove unused route

* Update some docstr and variables for streetview service objects

* Remove public routes for streetview endpoints.

* Update unit tests related to streeetview service resources

* Rename variables and docstr to use streetview service resource wording

* Rename things related to resources

* Add tests for creating and delting organizations

* Skip two tests

* Fix formatting

* Fix linting issue

* Add image id for streetview assets.

* Task/DES-2240 access to streetview (#85)

* Remove unused route

* Update some docstr and variables for streetview service objects

* Update unit tests related to streeetview service resources

* Rename variables and docstr to use streetview service resource wording

* Rename things related to resources

* Add tests for creating and delting organizations

* Skip two tests

* Fix formatting

* Fix linting issue

* Add delete instance test

* Add additional tests

* Remove as unused

* Revert "Merge branch 'task/DES-2240-access-to-streetview2' into DES-1828-streetview-authentication"

This reverts commit 963b87d, reversing
changes made to 3544c56.

* Change routes to make easier maintenance.

* Fix organization routes.

* Groundwork for process task notifications.

* Move streetview tasks back to streetview task file.

* Change docstring for streetview exceptions.

* Remove unnecessary text.

* Resolve feedback.

* Remove file.

* Remove file.

* Remove file.

* Remove file.

* Remove file.

* Correct wording.

* Update tasks for streetview progress.

* Remove unused function.

* Remove unused get user route.

* Remove unused get progress methods.

* Move streetview api exception handling to app.py.

* Error in app.py.

* Update mapillary tools to pin hash

Co-authored-by: Nathan Franklin <[email protected]>

Dockerfile

@@ -2,8 +2,10 @@ FROM python:3.7-slim
 RUN apt-get update -q && apt-get install -q -y \
   software-properties-common \
   libgdal-dev \
-  ffmpeg
+  ffmpeg \
+  git
 COPY requirements.txt /
+
 RUN pip install -q -r /requirements.txt
 RUN pip install -q gunicorn
 RUN mkdir /app

Dockerfile.entwine

@@ -28,4 +28,4 @@ RUN pip3 install -r requirements.txt
 COPY geoapi /api
 ENV PYTHONPATH "${PYTHONPATH}:/api"
 
-WORKDIR /api
+WORKDIR /api

Dockerfile.potree

@@ -3,7 +3,7 @@ FROM pdal/pdal:2.1
 RUN apt-get update && apt-get install -y \
 libtiff-dev libgeotiff-dev libgdal-dev \
 libboost-system-dev libboost-thread-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libboost-iostreams-dev \
-git cmake build-essential wget python3.7 python3-pip python3-dev ffmpeg \
+git cmake build-essential wget python3.7 python3-pip python3-dev ffmpeg git \
 unzip
 
 WORKDIR /opt
@@ -35,4 +35,3 @@ RUN mkdir app
 COPY ./geoapi /app/geoapi
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 WORKDIR /app/geoapi
-

docker-compose.yml

@@ -55,6 +55,7 @@ services:
       - .:/app
       - assets:/assets
     environment:
+      - MAPILLARY_CLIENT_TOKEN=4866220476802272
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=development
       - ASSETS_BASE_DIR=/assets
@@ -75,6 +76,7 @@ services:
       - .:/app
       - assets:/assets
     environment:
+      - MAPILLARY_CLIENT_TOKEN=4866220476802272
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=development
       - ASSETS_BASE_DIR=/assets
@@ -99,6 +101,7 @@ services:
       - 8.8.8.8
       - 8.8.4.4
     environment:
+      - MAPILLARY_CLIENT_TOKEN=4866220476802272
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=development
       - ASSETS_BASE_DIR=/assets
@@ -107,4 +110,6 @@ services:
     tty: true
     container_name: geoapi
     hostname: geoapi
-    command: "gunicorn -w 4 -b 0.0.0.0:8000 geoapi.app:app -k gevent --reload --timeout 300"
+    command: "gunicorn -w 4 -b 0.0.0.0:8000 geoapi.app:app -k gevent --reload --timeout 300"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

geoapi/app.py

@@ -3,7 +3,7 @@
 from geoapi.routes import api
 from geoapi.settings import settings as app_settings
 from geoapi.db import db_session
-from geoapi.exceptions import InvalidGeoJSON, InvalidEXIFData, InvalidCoordinateReferenceSystem, ObservableProjectAlreadyExists, ApiException
+from geoapi.exceptions import InvalidGeoJSON, InvalidEXIFData, InvalidCoordinateReferenceSystem, ObservableProjectAlreadyExists, ApiException, StreetviewAuthException, StreetviewLimitException
 
 import logging
 
@@ -51,6 +51,16 @@ def handle_observable_project_already_exists_exception(error: Exception):
     return {'message': 'Conflict, a project for this storage system/path already exists'}, 409
 
 
+@api.errorhandler(StreetviewAuthException)
+def handle_streetview_auth_exception(error: Exception):
+    return {'message': 'Not logged in to streetview service'}, 401
+
+
+@api.errorhandler(StreetviewLimitException)
+def handle_streetview_limit_exception(error: Exception):
+    return {'message': 'Exceed concurrent streetview publish limit'}, 403
+
+
 @app.teardown_appcontext
 def shutdown_session(exception=None):
     db_session.remove()

geoapi/exceptions.py

@@ -21,4 +21,16 @@ class MissingServiceAccount(Exception):
 
 class ApiException(Exception):
     """ A generic exception from the api"""
-    pass
+    pass
+
+class StreetviewAuthException(Exception):
+    """ Not logged in to streetview service """
+    pass
+
+class StreetviewLimitException(Exception):
+    """ Exceed concurrent streetview publish limit """
+    pass
+
+class StreetviewExistsException(Exception):
+    """ Already published the streetview assets from a system/path """
+    pass

geoapi/migrations/versions/0a10e4e1ea0c_.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 0a10e4e1ea0c
+Revises: 8bb0c45d987d
+Create Date: 2022-04-18 20:52:25.172291
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0a10e4e1ea0c'
+down_revision = '8bb0c45d987d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('streetview_sequence', sa.Column('organization_id', sa.String(), nullable=True))
+    op.create_index(op.f('ix_streetview_sequence_organization_id'), 'streetview_sequence', ['organization_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_streetview_sequence_organization_id'), table_name='streetview_sequence')
+    op.drop_column('streetview_sequence', 'organization_id')
+    # ### end Alembic commands ###

geoapi/migrations/versions/63e1135553fe_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 63e1135553fe
+Revises: efde4e0dd485
+Create Date: 2022-02-25 19:02:52.639068
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '63e1135553fe'
+down_revision = 'efde4e0dd485'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('streetview_service_key', 'streetview', type_='unique')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_unique_constraint('streetview_service_key', 'streetview', ['service'])
+    # ### end Alembic commands ###

geoapi/migrations/versions/8bb0c45d987d_.py

@@ -0,0 +1,38 @@
+"""empty message
+
+Revision ID: 8bb0c45d987d
+Revises: b014b0e37b0b
+Create Date: 2022-04-13 21:31:26.876517
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8bb0c45d987d'
+down_revision = 'b014b0e37b0b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('streetview_sequence', sa.Column('feature_id', sa.Integer(), nullable=True))
+    op.add_column('streetview_sequence', sa.Column('task_id', sa.Integer(), nullable=True))
+    op.create_index(op.f('ix_streetview_sequence_feature_id'), 'streetview_sequence', ['feature_id'], unique=False)
+    op.create_index(op.f('ix_streetview_sequence_task_id'), 'streetview_sequence', ['task_id'], unique=False)
+    op.create_foreign_key(None, 'streetview_sequence', 'features', ['feature_id'], ['id'], onupdate='CASCADE', ondelete='SET NULL')
+    op.create_foreign_key(None, 'streetview_sequence', 'tasks', ['task_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'streetview_sequence', type_='foreignkey')
+    op.drop_constraint(None, 'streetview_sequence', type_='foreignkey')
+    op.drop_index(op.f('ix_streetview_sequence_task_id'), table_name='streetview_sequence')
+    op.drop_index(op.f('ix_streetview_sequence_feature_id'), table_name='streetview_sequence')
+    op.drop_column('streetview_sequence', 'task_id')
+    op.drop_column('streetview_sequence', 'feature_id')
+    # ### end Alembic commands ###

geoapi/migrations/versions/b014b0e37b0b_.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: b014b0e37b0b
+Revises: 03c3d62d3b11, 63e1135553fe
+Create Date: 2022-04-01 03:04:29.358825
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b014b0e37b0b'
+down_revision = ('03c3d62d3b11', '63e1135553fe')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

geoapi/migrations/versions/efde4e0dd485_.py

@@ -0,0 +1,100 @@
+"""empty message
+
+Revision ID: efde4e0dd485
+Revises: 77346f8d9025
+Create Date: 2022-02-07 17:28:22.767875
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'efde4e0dd485'
+down_revision = '77346f8d9025'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('progress_notifications',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=True),
+    sa.Column('username', sa.String(), nullable=True),
+    sa.Column('tenant_id', sa.String(), nullable=True),
+    sa.Column('created', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
+    sa.Column('progress', sa.Integer(), nullable=True),
+    sa.Column('status', sa.String(length=256), nullable=True),
+    sa.Column('message', sa.String(length=512), nullable=True),
+    sa.Column('logs', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('viewed', sa.Boolean(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_progress_notifications_username'), 'progress_notifications', ['username'], unique=False)
+    op.create_table('streetview',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('token', sa.String(), nullable=True),
+    sa.Column('service', sa.String(), nullable=True),
+    sa.Column('service_user', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['user_id'], ['users.id'], onupdate='CASCADE', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('service')
+    )
+    op.create_index(op.f('ix_streetview_user_id'), 'streetview', ['user_id'], unique=False)
+    op.create_table('streetview_instance',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('streetview_id', sa.Integer(), nullable=True),
+    sa.Column('system_id', sa.String(), nullable=True),
+    sa.Column('path', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['streetview_id'], ['streetview.id'], onupdate='CASCADE', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_streetview_instance_path'), 'streetview_instance', ['path'], unique=False)
+    op.create_index(op.f('ix_streetview_instance_streetview_id'), 'streetview_instance', ['streetview_id'], unique=False)
+    op.create_index(op.f('ix_streetview_instance_system_id'), 'streetview_instance', ['system_id'], unique=False)
+    op.create_table('streetview_organization',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('streetview_id', sa.Integer(), nullable=True),
+    sa.Column('key', sa.String(), nullable=True),
+    sa.Column('name', sa.String(), nullable=True),
+    sa.Column('slug', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['streetview_id'], ['streetview.id'], onupdate='CASCADE', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_streetview_organization_streetview_id'), 'streetview_organization', ['streetview_id'], unique=False)
+    op.create_table('streetview_sequence',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('streetview_instance_id', sa.Integer(), nullable=True),
+    sa.Column('start_date', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('end_date', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('bbox', sa.String(), nullable=True),
+    sa.Column('sequence_id', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['streetview_instance_id'], ['streetview_instance.id'], onupdate='CASCADE', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_streetview_sequence_bbox'), 'streetview_sequence', ['bbox'], unique=False)
+    op.create_index(op.f('ix_streetview_sequence_sequence_id'), 'streetview_sequence', ['sequence_id'], unique=False)
+    op.create_index(op.f('ix_streetview_sequence_streetview_instance_id'), 'streetview_sequence', ['streetview_instance_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_streetview_sequence_streetview_instance_id'), table_name='streetview_sequence')
+    op.drop_index(op.f('ix_streetview_sequence_sequence_id'), table_name='streetview_sequence')
+    op.drop_index(op.f('ix_streetview_sequence_bbox'), table_name='streetview_sequence')
+    op.drop_table('streetview_sequence')
+    op.drop_index(op.f('ix_streetview_organization_streetview_id'), table_name='streetview_organization')
+    op.drop_table('streetview_organization')
+    op.drop_index(op.f('ix_streetview_instance_system_id'), table_name='streetview_instance')
+    op.drop_index(op.f('ix_streetview_instance_streetview_id'), table_name='streetview_instance')
+    op.drop_index(op.f('ix_streetview_instance_path'), table_name='streetview_instance')
+    op.drop_table('streetview_instance')
+    op.drop_index(op.f('ix_streetview_user_id'), table_name='streetview')
+    op.drop_table('streetview')
+    op.drop_index(op.f('ix_progress_notifications_username'), table_name='progress_notifications')
+    op.drop_table('progress_notifications')
+    # ### end Alembic commands ###

geoapi/models/__init__.py

@@ -6,5 +6,6 @@
 from .overlay import Overlay
 from .tile_server import TileServer
 from .observable_data import ObservableDataProject
-from .notification import Notification
+from .notification import Notification, ProgressNotification
 from .imported_file import ImportedFile
+from .streetview import Streetview, StreetviewInstance, StreetviewSequence, StreetviewOrganization

geoapi/models/notification.py

@@ -1,8 +1,9 @@
-import uuid
 from sqlalchemy import (
     Column, Integer, String,
     Boolean, DateTime
 )
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.sql import func
 from geoapi.db import Base
 
@@ -17,3 +18,20 @@ class Notification(Base):
     status = Column(String(256))
     message = Column(String(512))
     viewed = Column(Boolean, default=False)
+
+
+class ProgressNotification(Base):
+
+    __tablename__ = 'progress_notifications'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer)
+    tenant_id = Column(String)
+    uuid = Column(UUID(as_uuid=True))
+    username = Column(String, nullable=True, index=True)
+    tenant_id = Column(String)
+    created = Column(DateTime(timezone=True), server_default=func.now())
+    progress = Column(Integer)
+    status = Column(String(256))
+    message = Column(String(512))
+    logs = Column(JSONB, default={})
+    viewed = Column(Boolean, default=False)

geoapi/models/project.py

@@ -31,6 +31,7 @@ class Project(Base):
     created = Column(DateTime(timezone=True), server_default=func.now())
     updated = Column(DateTime(timezone=True), onupdate=func.now())
     features = relationship('Feature', cascade="all, delete-orphan")
+
     users = relationship('User',
                          secondary='projects_users',
                          back_populates='projects')