diff --git a/.gitignore b/.gitignore index 83461d6..7cc3619 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,7 @@ setup.ipynb *~ /.idea/ +/.vscode/ secrets gtfs/*.zip gtfs/*.pbf @@ -142,4 +143,12 @@ data/failed/ data/trash/ data/gtfs/ data/tmp - +data/users/** +data/** + +#these files are under app/static but they get copied to the outside directory on startup +logging.conf +config +static/** +templates/** +conf/** diff --git a/Dockerfile b/Dockerfile index 6002b0a..9110514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,27 +15,29 @@ RUN \ # Remove package index obtained by `apt update`. && rm -rf /var/lib/apt/lists/* -ENV ADMIN_TOKEN='' -ENV RIDE2GO_TOKEN='' + ENV ADMIN_TOKEN='' + ENV RIDE2GO_TOKEN='' + ENV SECRET_KEY='' EXPOSE 80 -ARG PLUGINS - COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ${PLUGINS} +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt # set MODULE_NAME explicitly ENV MODULE_NAME=amarillo.main COPY ./amarillo /app/amarillo -COPY enhancer.py /app -COPY prestart.sh /app -COPY ./static /app/static -COPY ./templates /app/templates -COPY config /app -COPY logging.conf /app -COPY ./conf /app/conf +COPY ./amarillo/plugins /app/amarillo/plugins +COPY ./amarillo/static/static /app/static +COPY ./amarillo/static/templates /app/templates +COPY ./amarillo/static/config /app +COPY ./amarillo/static/logging.conf /app +COPY ./amarillo/static/data /app/data + +# Create the error.log, otherwise we get a permission error when we try to write to it +RUN touch /app/error.log +RUN chmod 777 /app/error.log # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Jenkinsfile b/Jenkinsfile index 2715959..7223a30 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,17 @@ pipeline { - agent any + agent { label 'builtin' } environment { GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER') - PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER') TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi" PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple" - DOCKER_REGISTRY_URL = 'https://git.gerhardt.io' + DOCKER_REGISTRY = 'git.gerhardt.io' + DERIVED_DOCKERFILE = 'standard.Dockerfile' OWNER = 'amarillo' + BASE_IMAGE_NAME = 'amarillo-base' IMAGE_NAME = 'amarillo' - AMARILLO_DISTRIBUTION = '0.2' - TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}" - PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export' - DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand' + AMARILLO_DISTRIBUTION = '0.3' + TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}" + DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/${env.BRANCH_NAME}" DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET') } stages { @@ -42,35 +42,56 @@ pipeline { sh 'python3 -m twine upload --skip-existing --verbose --repository-url $TWINE_REPO_URL --username $GITEA_CREDS_USR --password $GITEA_CREDS_PSW ./dist/*' } } - stage('Publish package to PyPI') { + stage('Build base docker image') { when { - branch 'release' + isDeployBranch() } steps { - sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*' + echo 'Building image' + script { + docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") + } + } + } + stage('Push base image to container registry') { + when { + isDeployBranch() + } + steps { + echo 'Pushing image to registry' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") + image.push() + image.push('latest') + } + } } } - stage('Build Mitanand docker image') { + stage('Build derived docker image') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Building image' script { - docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", - //--no-cache to make sure plugins are updated - "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", + //--no-cache to make sure plugins are updated + "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + } + } } } - stage('Push image to container registry') { + stage('Push derived image to container registry') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Pushing image to registry' script { - docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){ + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}") image.push() image.push('latest') @@ -80,7 +101,7 @@ pipeline { } stage('Notify CD script') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Triggering deploy webhook' @@ -93,3 +114,7 @@ pipeline { } } } + +def isDeployBranch() { + return anyOf { branch 'main'; branch 'dev'; branch 'mitanand' } +} diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0daae06 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include amarillo/static/ * +recursive-include amarillo/tests/ * \ No newline at end of file diff --git a/README.md b/README.md index 15f1fe9..ce62132 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,7 @@ An Amarillo is a [yellow-dressed person](https://www.dreamstime.com/sancti-spiri - Python 3.9.2 with pip - python3-venv -Create a virtual environment `python3 -m venv venv`. - -Activate the environment and install the dependencies `pip install -r requirements.txt`. +Create a virtual environment `python3 -m venv venv`. Activate the environment with `source venv/bin/activate` and install the dependencies `pip install -r requirements.txt`. Run `uvicorn amarillo.main:app`. @@ -22,6 +20,8 @@ In development, you can use `--reload`. - `env` - `ADMIN_TOKEN` +E.g. set the environment variable like this: `export ADMIN_TOKEN=YOUR_SECRET_TOKEN_HERE`. + ## Security All endpoints are protected by an API-Key in the HTTP header. @@ -29,16 +29,16 @@ There is a special *admin* user. For this user, the API-Key must be passed in as an environment variable when Amarillo is started. -The admin can create additional API-Keys in the `/agencyconf` endpoint. This +The admin can create additional API-Keys in the `/users` endpoint. This endpoint is always available but not always shown in `/docs`, especially not when running in production. -The Swagger docs for `/agencyconf` can be seen on the MFDZ demo server. +The Swagger docs for `/users` can be seen on the MFDZ demo server. Permissions work this way - the admin is allowed to call all operations on all resources. Only the admin - can create new API-Keys by POSTing an `AgencyConf` JSON object to `/agencyconf`. + can create new API-Keys by POSTing an `users` JSON object to `/users`. - API-Keys for agencies are allowed to POST/PUT/GET/DELETE their own - resources and GET some public resources. + resources and GET some public resources. ## Development diff --git a/amarillo/configuration.py b/amarillo/configuration.py index 669c988..6d2d66f 100644 --- a/amarillo/configuration.py +++ b/amarillo/configuration.py @@ -1,21 +1,14 @@ # separate file so that it can be imported without initializing FastAPI from amarillo.utils.container import container -import json import logging -from glob import glob -from amarillo.models.Carpool import Agency, Carpool, Region -from amarillo.services import stops -from amarillo.services import trips -from amarillo.services.agencyconf import AgencyConfService, agency_conf_directory -from amarillo.services.carpools import CarpoolService +from amarillo.services.users import UserService, user_conf_directory from amarillo.services.agencies import AgencyService from amarillo.services.regions import RegionService from amarillo.services.config import config from amarillo.utils.utils import assert_folder_exists -import amarillo.services.gtfs_generator as gtfs_generator logger = logging.getLogger(__name__) @@ -34,12 +27,12 @@ def create_required_directories(): assert_folder_exists(f'data/{subdir}/{agency_id}') # Agency configurations - assert_folder_exists(agency_conf_directory) + assert_folder_exists(user_conf_directory) def configure_services(): - container['agencyconf'] = AgencyConfService() - logger.info("Loaded %d agency configuration(s)", len(container['agencyconf'].agency_id_to_agency_conf)) + container['users'] = UserService() + logger.info("Loaded %d user configuration(s)", len(container['users'].user_id_to_user_conf)) container['agencies'] = AgencyService() logger.info("Loaded %d agencies", len(container['agencies'].agencies)) @@ -49,42 +42,6 @@ def configure_services(): create_required_directories() - -def configure_enhancer_services(): - configure_services() - - logger.info("Load stops...") - with open(config.stop_sources_file) as stop_sources_file: - stop_sources = json.load(stop_sources_file) - stop_store = stops.StopsStore(stop_sources) - - stop_store.load_stop_sources() - container['stops_store'] = stop_store - container['trips_store'] = trips.TripStore(stop_store) - container['carpools'] = CarpoolService(container['trips_store']) - - logger.info("Restore carpools...") - - for agency_id in container['agencies'].agencies: - for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): - try: - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].put(carpool.agency, carpool.id, carpool) - except Exception as e: - logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) - - # notify carpool about carpools in trash, as delete notifications must be sent - for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'): - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].delete(carpool.agency, carpool.id) - - logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) - logger.info("Starting scheduler") - gtfs_generator.start_schedule() - - def configure_admin_token(): if config.admin_token is None: message = "ADMIN_TOKEN environment variable not set" diff --git a/amarillo/main.py b/amarillo/main.py index cadafe3..70821ba 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -5,10 +5,15 @@ import mimetypes from starlette.staticfiles import StaticFiles +from amarillo.utils.utils import copy_static_files +#this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet +copy_static_files(["data", "static", "templates", "logging.conf", "config"]) + import amarillo.plugins from amarillo.services.config import config from amarillo.configuration import configure_services, configure_admin_token -from amarillo.routers import carpool, agency, agencyconf, region +from amarillo.routers import carpool, agency, users, region +import amarillo.services.oauth2 as oauth2 from fastapi import FastAPI # https://pydantic-docs.helpmanual.io/usage/settings/ @@ -76,8 +81,9 @@ app.include_router(carpool.router) app.include_router(agency.router) -app.include_router(agencyconf.router) +app.include_router(users.router) app.include_router(region.router) +app.include_router(oauth2.router) def iter_namespace(ns_pkg): diff --git a/amarillo/models/AgencyConf.py b/amarillo/models/AgencyConf.py deleted file mode 100644 index 29ac2f5..0000000 --- a/amarillo/models/AgencyConf.py +++ /dev/null @@ -1,26 +0,0 @@ -from pydantic import ConfigDict, BaseModel, Field - - -class AgencyConf(BaseModel): - agency_id: str = Field( - description="ID of the agency that uses this token.", - min_length=1, - max_length=20, - pattern='^[a-zA-Z0-9]+$', - examples=["mfdz"]) - - api_key: str = Field( - description="The agency's API key for using the API", - min_length=20, - max_length=256, - pattern=r'^[a-zA-Z0-9]+$', - examples=["d8yLuY4DqMEUCLcfJASi"]) - model_config = ConfigDict(json_schema_extra={ - "title": "Agency Configuration", - "description": "Configuration for an agency.", - "example": - { - "agency_id": "mfdz", - "api_key": "d8yLuY4DqMEUCLcfJASi" - } - }) diff --git a/amarillo/models/Carpool.py b/amarillo/models/Carpool.py index ffbb995..b9c019c 100644 --- a/amarillo/models/Carpool.py +++ b/amarillo/models/Carpool.py @@ -4,7 +4,7 @@ from datetime import time from pydantic import BaseModel, Field from geojson_pydantic.geometries import LineString -from enum import Enum +from enum import Enum, IntEnum NumType = Union[float, int] @@ -24,6 +24,15 @@ class PickupDropoffType(str, Enum): only_pickup = "only_pickup" only_dropoff = "only_dropoff" +class YesNoEnum(IntEnum): + yes = 1 + no = 2 + +class LuggageSize(IntEnum): + small = 1 + medium = 2 + large = 3 + class StopTime(BaseModel): id: Optional[str] = Field( None, @@ -111,7 +120,83 @@ class Region(BaseModel): bbox: Tuple[NumType, NumType, NumType, NumType] = Field( description="Bounding box of the region. Format is [minLon, minLat, maxLon, maxLat]", examples=[[10.5,49.2,11.3,51.3]]) + +class RidesharingInfo(BaseModel): + number_free_seats: int = Field( + description="Number of free seats", + ge=0, + examples=[3]) + + same_gender: Optional[YesNoEnum] = Field( + None, + description="Trip only for same gender:" + "1: Yes" + "2: No", + examples=[1]) + luggage_size: Optional[LuggageSize] = Field( + None, + description="Size of the luggage:" + "1: small" + "2: medium" + "3: large", + examples=[3]) + animal_car: Optional[YesNoEnum] = Field( + None, + description="Animals in Car allowed:" + "1: Yes" + "2: No", + examples=[2]) + + car_model: Optional[str] = Field( + None, + description="Car model", + min_length=1, + max_length=48, + examples=["Golf"]) + car_brand: Optional[str] = Field( + None, + description="Car brand", + min_length=1, + max_length=48, + examples=["VW"]) + + creation_date: datetime = Field( + description="Date when trip was created", + examples=["2022-02-13T20:20:39+00:00"]) + + smoking: Optional[YesNoEnum] = Field( + None, + description="Smoking allowed:" + "1: Yes" + "2: No", + examples=[2]) + + payment_method: Optional[str] = Field( + None, + description="Method of payment", + min_length=1, + max_length=48) +class Driver(BaseModel): + driver_id: Optional[str] = Field( + None, + description="Identifies the driver.", + min_length=1, + max_length=256, + pattern='^[a-zA-Z0-9_-]+$', + examples=["789"]) + profile_picture: Optional[HttpUrl] = Field( + None, + description="URL that contains the profile picture", + examples=["https://mfdz.de/driver/789/picture"]) + rating: Optional[int] = Field( + None, + description="Rating of the driver from 1 to 5." + "0 no rating yet", + ge=0, + le=5, + examples=[5]) + class Agency(BaseModel): id: str = Field( description="ID of the agency.", @@ -196,6 +281,17 @@ class Carpool(BaseModel): max_length=20, pattern='^[a-zA-Z0-9]+$', examples=["mfdz"]) + + driver: Optional[Driver] = Field( + None, + description="Driver data", + examples=[""" + { + "driver_id": "123", + "profile_picture": "https://mfdz.de/driver/789/picture", + "rating": 5 + } + """]) deeplink: HttpUrl = Field( description="Link to an information page providing detail information " @@ -246,7 +342,22 @@ class Carpool(BaseModel): "published.", examples=['A single date 2022-04-04 or a list of weekdays ["saturday", ' '"sunday"]']) - + route_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Route color designation that matches public facing material. " + "The color difference between route_color and route_text_color " + "should provide sufficient contrast when viewed on a black and " + "white screen.", + examples=["0039A6"]) + route_text_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Legible color to use for text drawn against a background of " + "route_color. The color difference between route_color and " + "route_text_color should provide sufficient contrast when " + "viewed on a black and white screen.", + examples=["D4D2D2"]) path: Optional[LineString] = Field( None, description="Optional route geometry as json LineString.") @@ -258,6 +369,18 @@ class Carpool(BaseModel): "purge outdated offers (e.g. older than 180 days). If not " "passed, the service may assume 'now'", examples=["2022-02-13T20:20:39+00:00"]) + additional_ridesharing_info: Optional[RidesharingInfo] = Field( + None, + description="Extension of GRFS to the GTFS standard", + examples=[""" + { + "number_free_seats": 2, + "creation_date": "2022-02-13T20:20:39+00:00", + "same_gender": 2, + "smoking": 1, + "luggage_size": 3 + } + """]) model_config = ConfigDict(json_schema_extra={ "title": "Carpool", # description ... diff --git a/amarillo/models/User.py b/amarillo/models/User.py new file mode 100644 index 0000000..c48d49e --- /dev/null +++ b/amarillo/models/User.py @@ -0,0 +1,38 @@ +from typing import Annotated, Optional, List +from pydantic import ConfigDict, BaseModel, Field +class User(BaseModel): + #TODO: add attributes admin, permissions, fullname, email + + user_id: str = Field( + description="ID of the agency that uses this token.", + min_length=1, + max_length=20, + pattern='^[a-zA-Z0-9]+$', + examples=["mfdz"]) + + api_key: Optional[str] = Field(None, + description="The agency's API key for using the API", + min_length=20, + max_length=256, + pattern=r'^[a-zA-Z0-9]+$', + examples=["d8yLuY4DqMEUCLcfJASi"]) + password: Optional[str] = Field(None, + description="The agency's password for generating JWT tokens", + min_length=8, + max_length=256, + examples=["$2b$12$EixZaYVK1fsbw1ZfbX3OXePaWxn96p36WQoeG6Lruj3vjPGga31lW"]) + permissions: Optional[List[Annotated[str, Field(pattern=r'^[a-z0-9-]+(:[a-z]+)?$')]]] = Field([], + description="The permissions of this user, a list of strings in the format or ", + max_length=256, + # pattern=r'^[a-zA-Z0-9]+(:[a-zA-Z]+)?$', #TODO + examples=["ride2go:read", "all:read", "admin", "geojson"]) + model_config = ConfigDict(json_schema_extra={ + "title": "Agency Configuration", + "description": "Configuration for an agency.", + "example": + { + "agency_id": "mfdz", + "api_key": "d8yLuY4DqMEUCLcfJASi", + "password": "$2b$12$EixZaYVK1fsbw1ZfbX3OXePaWxn96p36WQoeG6Lruj3vjPGga31lW" + } + }) diff --git a/amarillo/models/gtfs.py b/amarillo/models/gtfs.py deleted file mode 100644 index ee7f701..0000000 --- a/amarillo/models/gtfs.py +++ /dev/null @@ -1,29 +0,0 @@ -from collections import namedtuple -from datetime import timedelta - -GtfsFeedInfo = namedtuple('GtfsFeedInfo', 'feed_id feed_publisher_name feed_publisher_url feed_lang feed_version') -GtfsAgency = namedtuple('GtfsAgency', 'agency_id agency_name agency_url agency_timezone agency_lang agency_email') -GtfsRoute = namedtuple('GtfsRoute', 'agency_id route_id route_long_name route_type route_url route_short_name') -GtfsStop = namedtuple('GtfsStop', 'stop_id stop_lat stop_lon stop_name') -GtfsStopTime = namedtuple('GtfsStopTime', 'trip_id departure_time arrival_time stop_id stop_sequence pickup_type drop_off_type timepoint') -GtfsTrip = namedtuple('GtfsTrip', 'route_id trip_id service_id shape_id trip_headsign bikes_allowed') -GtfsCalendar = namedtuple('GtfsCalendar', 'service_id start_date end_date monday tuesday wednesday thursday friday saturday sunday') -GtfsCalendarDate = namedtuple('GtfsCalendarDate', 'service_id date exception_type') -GtfsShape = namedtuple('GtfsShape','shape_id shape_pt_lon shape_pt_lat shape_pt_sequence') - -# TODO Move to utils -class GtfsTimeDelta(timedelta): - def __str__(self): - seconds = self.total_seconds() - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - seconds = seconds % 60 - str = '{:02d}:{:02d}:{:02d}'.format(int(hours), int(minutes), int(seconds)) - return (str) - - def __add__(self, other): - if isinstance(other, timedelta): - return self.__class__(self.days + other.days, - self.seconds + other.seconds, - self.microseconds + other.microseconds) - return NotImplemented \ No newline at end of file diff --git a/amarillo/plugins/__init__.py b/amarillo/plugins/__init__.py index 0260537..69e3be5 100644 --- a/amarillo/plugins/__init__.py +++ b/amarillo/plugins/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/amarillo/routers/agency.py b/amarillo/routers/agency.py index cf50c47..fd2e1b5 100644 --- a/amarillo/routers/agency.py +++ b/amarillo/routers/agency.py @@ -5,7 +5,8 @@ from fastapi import APIRouter, HTTPException, status, Depends from amarillo.models.Carpool import Carpool, Agency -from amarillo.routers.agencyconf import verify_api_key, verify_admin_api_key, verify_permission_for_same_agency_or_admin +from amarillo.models.User import User +from amarillo.services.oauth2 import get_current_user, verify_permission # TODO should move this to service from amarillo.routers.carpool import store_carpool, delete_agency_carpools_older_than from amarillo.services.agencies import AgencyService @@ -32,7 +33,7 @@ status.HTTP_404_NOT_FOUND: {"description": "Agency not found"}, }, ) -async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key)) -> Agency: +async def get_agency(agency_id: str, requesting_user: User = Depends(get_current_user)) -> Agency: agencies: AgencyService = container['agencies'] agency = agencies.get_agency(agency_id) agency_exists = agency is not None @@ -52,6 +53,7 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key operation_id="sync", summary="Synchronizes all carpool offers", response_model=List[Carpool], + response_model_exclude_none=True, responses={ status.HTTP_200_OK: { "description": "Carpool created"}, @@ -60,8 +62,8 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key status.HTTP_500_INTERNAL_SERVER_ERROR: { "description": "Import error"} }) -async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: - await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) +async def sync(agency_id: str, requesting_user: User = Depends(get_current_user)) -> List[Carpool]: + verify_permission(f"{agency_id}:sync") if agency_id == "ride2go": importer = Ride2GoImporter() diff --git a/amarillo/routers/agencyconf.py b/amarillo/routers/agencyconf.py deleted file mode 100644 index 036cd95..0000000 --- a/amarillo/routers/agencyconf.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -from typing import List - -from fastapi import APIRouter, HTTPException, status, Header, Depends - -from amarillo.models.AgencyConf import AgencyConf -from amarillo.services.agencyconf import AgencyConfService -from amarillo.services.config import config -from amarillo.utils.container import container - -logger = logging.getLogger(__name__) - -router = APIRouter( - prefix="/agencyconf", - tags=["agencyconf"] -) - -# This endpoint is not shown in PROD installations, only in development -# TODO make this an explicit config option -include_in_schema = config.env != 'PROD' - - -# noinspection PyPep8Naming -# X_API_Key is upper case for OpenAPI -async def verify_admin_api_key(X_API_Key: str = Header(...)): - if X_API_Key != config.admin_token: - message="X-API-Key header invalid" - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - return "admin" - - -# noinspection PyPep8Naming -# X_API_Key is upper case for OpenAPI -async def verify_api_key(X_API_Key: str = Header(...)): - agency_conf_service: AgencyConfService = container['agencyconf'] - - return agency_conf_service.check_api_key(X_API_Key) - -# TODO Return code 403 Unauthoized (in response_status_codes as well...) -async def verify_permission_for_same_agency_or_admin(agency_id_in_path_or_body, agency_id_from_api_key): - """Verifies that an agency is accessing something it owns or the user is admin - - The agency_id is part of some paths, or when not in the path it is in the body, e.g. in PUT /carpool. - - This function encapsulates the formula 'working with own stuff, or admin'. - """ - is_permitted = agency_id_in_path_or_body == agency_id_from_api_key or agency_id_from_api_key == "admin" - - if not is_permitted: - message = f"Working with {agency_id_in_path_or_body} resources is not permitted for {agency_id_from_api_key}." - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - -@router.get("/", - include_in_schema=include_in_schema, - operation_id="getAgencyIdsWhichHaveAConfiguration", - summary="Get agency_ids which have a configuration", - response_model=List[str], - description="Returns the agency_ids but not the details.", - status_code=status.HTTP_200_OK) -async def get_agency_ids(admin_api_key: str = Depends(verify_api_key)) -> [str]: - return container['agencyconf'].get_agency_ids() - - -@router.post("/", - include_in_schema=include_in_schema, - operation_id="postNewAgencyConf", - summary="Post a new AgencyConf") -async def post_agency_conf(agency_conf: AgencyConf, admin_api_key: str = Depends(verify_admin_api_key)): - agency_conf_service: AgencyConfService = container['agencyconf'] - agency_conf_service.add(agency_conf) - -# TODO 400->403 -@router.delete("/{agency_id}", - include_in_schema=include_in_schema, - operation_id="deleteAgencyConf", - status_code=status.HTTP_200_OK, - summary="Delete configuration of an agency. Returns true if the token for the agency existed, " - "false if it didn't exist." - ) -async def delete_agency_conf(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)): - agency_may_delete_own = requesting_agency_id == agency_id - admin_may_delete_everything = requesting_agency_id == "admin" - is_permitted = agency_may_delete_own or admin_may_delete_everything - - if not is_permitted: - message = f"The API key for {requesting_agency_id} can not delete the configuration for {agency_id}" - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - agency_conf_service: AgencyConfService = container['agencyconf'] - - agency_exists = agency_id in agency_conf_service.get_agency_ids() - - if not agency_exists: - message = f"No config for {agency_id}" - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - agency_conf_service.delete(agency_id) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 1eeb96a..908983b 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -5,13 +5,17 @@ import re from glob import glob -from fastapi import APIRouter, Body, Header, HTTPException, status, Depends +from fastapi import APIRouter, Body, HTTPException, status, Depends, BackgroundTasks +import requests from datetime import datetime from amarillo.models.Carpool import Carpool -from amarillo.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin +from amarillo.models.User import User +from amarillo.services.oauth2 import get_current_user, verify_permission from amarillo.tests.sampledata import examples - +from amarillo.services.hooks import run_on_create, run_on_delete +from amarillo.services.config import config +from amarillo.utils.utils import assert_folder_exists logger = logging.getLogger(__name__) @@ -20,26 +24,43 @@ tags=["carpool"] ) +#TODO: housekeeping for outdated trips + +def enhance_trip(carpool: Carpool): + response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json()) + enhanced_carpool = Carpool(**json.loads(response.content)) + + #TODO: use data/enhanced directory + folder = f'data/enhanced/{carpool.agency}' + filename = f'{folder}/{carpool.id}.json' + + assert_folder_exists(folder) + with open(filename, 'w', encoding='utf-8') as f: + f.write(enhanced_carpool.model_dump_json()) + @router.post("/", operation_id="addcarpool", summary="Add a new or update existing carpool", description="Carpool object to be created or updated", response_model=Carpool, + response_model_exclude_none=True, responses={ status.HTTP_404_NOT_FOUND: { "description": "Agency does not exist"}, }) -async def post_carpool(carpool: Carpool = Body(..., examples=examples), - requesting_agency_id: str = Depends(verify_api_key)) -> Carpool: - await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id) +async def post_carpool(background_tasks: BackgroundTasks, carpool: Carpool = Body(..., examples=examples), + requesting_user: User = Depends(get_current_user)) -> Carpool: + verify_permission(f"{carpool.agency}:write", requesting_user) + + background_tasks.add_task(run_on_create, carpool) logger.info(f"POST trip {carpool.agency}:{carpool.id}.") await assert_agency_exists(carpool.agency) - await set_lastUpdated_if_unset(carpool) + await store_carpool(carpool) - await save_carpool(carpool) + background_tasks.add_task(enhance_trip, carpool) return carpool @@ -48,12 +69,15 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples), operation_id="getcarpoolById", summary="Find carpool by ID", response_model=Carpool, + response_model_exclude_none=True, description="Find carpool by ID", responses={ status.HTTP_404_NOT_FOUND: {"description": "Carpool not found"}, }, ) -async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(verify_api_key)) -> Carpool: +async def get_carpool(agency_id: str, carpool_id: str, requesting_user: User = Depends(get_current_user)) -> Carpool: + verify_permission(f"{agency_id}:read", requesting_user) + logger.info(f"Get trip {agency_id}:{carpool_id}.") await assert_agency_exists(agency_id) await assert_carpool_exists(agency_id, carpool_id) @@ -72,12 +96,14 @@ async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(ve "description": "Carpool or agency not found"}, }, ) -async def delete_carpool(agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): - await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) +async def delete_carpool(background_tasks: BackgroundTasks, agency_id: str, carpool_id: str, requesting_user: User = Depends(get_current_user)): + verify_permission(f"{agency_id}:write", requesting_user) logger.info(f"Delete trip {agency_id}:{carpool_id}.") await assert_agency_exists(agency_id) await assert_carpool_exists(agency_id, carpool_id) + cp = await load_carpool(agency_id, carpool_id) + background_tasks.add_task(run_on_delete, cp) return await _delete_carpool(agency_id, carpool_id) @@ -88,9 +114,16 @@ async def _delete_carpool(agency_id: str, carpool_id: str): # load and store, to receive pyinotify events and have file timestamp updated await save_carpool(cp, 'data/trash') logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.") - os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + try: + os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + os.remove(f"data/enhanced/{agency_id}/{carpool_id}.json", ) + except FileNotFoundError: + pass + async def store_carpool(carpool: Carpool) -> Carpool: + carpool_exists = os.path.exists(f"data/carpool/{carpool.agency}/{carpool.id}.json") + await set_lastUpdated_if_unset(carpool) await save_carpool(carpool) @@ -114,7 +147,7 @@ async def save_carpool(carpool, folder: str = 'data/carpool'): async def assert_agency_exists(agency_id: str): - agency_exists = os.path.exists(f"conf/agency/{agency_id}.json") + agency_exists = os.path.exists(f"data/agency/{agency_id}.json") if not agency_exists: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -134,4 +167,6 @@ async def delete_agency_carpools_older_than(agency_id, timestamp): if os.path.getmtime(carpool_file_name) < timestamp: m = re.search(r'([a-zA-Z0-9_-]+)\.json$', carpool_file_name) # TODO log deletion + cp = await load_carpool(agency_id, m[1]) + run_on_delete(cp) await _delete_carpool(agency_id, m[1]) diff --git a/amarillo/routers/region.py b/amarillo/routers/region.py index 50da511..86984ac 100644 --- a/amarillo/routers/region.py +++ b/amarillo/routers/region.py @@ -5,10 +5,8 @@ from fastapi import APIRouter, HTTPException, status, Depends from amarillo.models.Carpool import Region -from amarillo.routers.agencyconf import verify_admin_api_key from amarillo.services.regions import RegionService from amarillo.utils.container import container -from fastapi.responses import FileResponse logger = logging.getLogger(__name__) @@ -45,7 +43,7 @@ async def get_region(region_id: str) -> Region: return region def _assert_region_exists(region_id: str) -> Region: - regions: regionService = container['regions'] + regions: RegionService = container['regions'] region = regions.get_region(region_id) region_exists = region is not None @@ -55,34 +53,3 @@ def _assert_region_exists(region_id: str) -> Region: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) return region - -@router.get("/{region_id}/gtfs", - summary="Return GTFS Feed for this region", - response_description="GTFS-Feed (zip-file)", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - } - ) -async def get_file(region_id: str, user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip') - -@router.get("/{region_id}/gtfs-rt", - summary="Return GTFS-RT Feed for this region", - response_description="GTFS-RT-Feed", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - status.HTTP_400_BAD_REQUEST: {"description": "Bad request, e.g. because format is not supported, i.e. neither protobuf nor json."} - } - ) -async def get_file(region_id: str, format: str = 'protobuf', user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - if format == 'json': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json') - elif format == 'protobuf': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf') - else: - message = "Specified format is not supported, i.e. neither protobuf nor json." - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) diff --git a/amarillo/routers/users.py b/amarillo/routers/users.py new file mode 100644 index 0000000..4c416d8 --- /dev/null +++ b/amarillo/routers/users.py @@ -0,0 +1,71 @@ +import logging +from typing import List + +from fastapi import APIRouter, HTTPException, status, Header, Depends + +from amarillo.models.User import User +from amarillo.services.users import UserService +from amarillo.services.oauth2 import get_current_user, verify_permission +from amarillo.services.config import config +from amarillo.utils.container import container + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/users", + tags=["users"] +) + +# This endpoint is not shown in PROD installations, only in development +# TODO make this an explicit config option +include_in_schema = config.env != 'PROD' + + +@router.get("/", + include_in_schema=include_in_schema, + operation_id="getUserIdsWhichHaveAConfiguration", + summary="Get user which have a configuration", + response_model=List[str], + description="Returns the user_ids but not the details.", + status_code=status.HTTP_200_OK) +async def get_user_ids(requesting_user: User = Depends(get_current_user)) -> [str]: + return container['users'].get_user_ids() + + +@router.post("/", + include_in_schema=include_in_schema, + operation_id="postNewUserConf", + summary="Post a new User") +async def post_user_conf(user_conf: User, requesting_user: User = Depends(get_current_user)): + verify_permission("admin", requesting_user) + user_service: UserService = container['users'] + user_service.add(user_conf) + +# TODO 400->403 +@router.delete("/{user_id}", + include_in_schema=include_in_schema, + operation_id="deleteUser", + status_code=status.HTTP_200_OK, + summary="Delete configuration of a user. Returns true if the token for the user existed, " + "false if it didn't exist." + ) +async def delete_user(user_id: str, requesting_user: User = Depends(get_current_user)): + user_may_delete_own = requesting_user.user_id == user_id + admin_may_delete_everything = "admin" in requesting_user.permissions + is_permitted = user_may_delete_own or admin_may_delete_everything + + if not is_permitted: + message = f"User '{requesting_user.user_id} can not delete the configuration for {user_id}" + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + user_service: UserService = container['users'] + + agency_exists = user_id in user_service.get_user_ids() + + if not agency_exists: + message = f"No config for {user_id}" + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + user_service.delete(user_id) diff --git a/amarillo/services/agencies.py b/amarillo/services/agencies.py index 47baa5a..e7450aa 100644 --- a/amarillo/services/agencies.py +++ b/amarillo/services/agencies.py @@ -12,8 +12,7 @@ class AgencyService: def __init__(self): self.agencies: Dict[str, Agency] = {} - - for agency_file_name in glob('conf/agency/*.json'): + for agency_file_name in glob('data/agency/*.json'): with open(agency_file_name) as agency_file: dict = json.load(agency_file) agency = Agency(**dict) diff --git a/amarillo/services/agencyconf.py b/amarillo/services/agencyconf.py deleted file mode 100644 index 30431dc..0000000 --- a/amarillo/services/agencyconf.py +++ /dev/null @@ -1,111 +0,0 @@ -import json -import os -from glob import glob -from typing import Dict, List -import logging - -from fastapi import HTTPException, status - -from amarillo.models.AgencyConf import AgencyConf -from amarillo.services.config import config - -logger = logging.getLogger(__name__) - -agency_conf_directory = 'data/agencyconf' - - -class AgencyConfService: - - def __init__(self): - # Both Dicts to be kept in sync always. The second api_key_to_agency_id is like a reverse - # cache for the first for fast lookup of valid api keys, which happens on *every* request. - self.agency_id_to_agency_conf: Dict[str, AgencyConf] = {} - self.api_key_to_agency_id: Dict[str, str] = {} - - for agency_conf_file_name in glob(f'{agency_conf_directory}/*.json'): - with open(agency_conf_file_name) as agency_conf_file: - dictionary = json.load(agency_conf_file) - - agency_conf = AgencyConf(**dictionary) - - agency_id = agency_conf.agency_id - api_key = agency_conf.api_key - - self.agency_id_to_agency_conf[agency_id] = agency_conf - self.api_key_to_agency_id[api_key] = agency_conf.agency_id - - def get_agency_conf(self, agency_id: str) -> AgencyConf: - agency_conf = self.agency_id_to_agency_conf.get(agency_id) - return agency_conf - - def check_api_key(self, api_key: str) -> str: - """Check if the API key is valid - - The agencies' api keys are checked first, and the admin's key. - - The agency_id or "admin" is returned for further checks in the caller if the - request is permitted, like {agency_id} == agency_id. - """ - - agency_id = self.api_key_to_agency_id.get(api_key) - - is_agency = agency_id is not None - - if is_agency: - return agency_id - - is_admin = api_key == config.admin_token - - if is_admin: - return "admin" - - message = "X-API-Key header invalid" - logger.error(message) - raise HTTPException(status_code=400, detail=message) - - def add(self, agency_conf: AgencyConf): - - agency_id = agency_conf.agency_id - api_key = agency_conf.api_key - - agency_id_exists_already = self.agency_id_to_agency_conf.get(agency_id) is not None - - if agency_id_exists_already: - message = f"Agency {agency_id} exists already. To update, delete it first." - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - agency_using_this_api_key_already = self.api_key_to_agency_id.get(api_key) - a_different_agency_is_using_this_api_key_already = \ - agency_using_this_api_key_already is not None and \ - agency_using_this_api_key_already != agency_id - - if a_different_agency_is_using_this_api_key_already: - message = f"Duplicate API Key for {agency_id} not permitted. Use a different key." - logger.error(message) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) - - with open(f'{agency_conf_directory}/{agency_id}.json', 'w', encoding='utf-8') as f: - f.write(agency_conf.json()) - - self.agency_id_to_agency_conf[agency_id] = agency_conf - self.api_key_to_agency_id[api_key] = agency_id - - logger.info(f"Added configuration for agency {agency_id}.") - - def get_agency_ids(self) -> List[str]: - return list(self.agency_id_to_agency_conf.keys()) - - def delete(self, agency_id): - - agency_conf = self.agency_id_to_agency_conf.get(agency_id) - - api_key = agency_conf.api_key - - del self.api_key_to_agency_id[api_key] - - del self.agency_id_to_agency_conf[agency_id] - - os.remove(f'{agency_conf_directory}/{agency_id}.json') - - logger.info(f"Deleted configuration for agency {agency_id}.") diff --git a/amarillo/services/config.py b/amarillo/services/config.py index 1dd620a..fe8fffb 100644 --- a/amarillo/services/config.py +++ b/amarillo/services/config.py @@ -8,6 +8,7 @@ class Config(BaseSettings): ride2go_query_data: str env: str = 'DEV' graphhopper_base_url: str = 'https://api.mfdz.de/gh' - stop_sources_file: str = 'conf/stop_sources.json' + stop_sources_file: str = 'data/stop_sources.json' + enhancer_url: str = 'http://localhost:8001' config = Config(_env_file='config', _env_file_encoding='utf-8') diff --git a/amarillo/services/gtfs.py b/amarillo/services/gtfs.py deleted file mode 100644 index c45a626..0000000 --- a/amarillo/services/gtfs.py +++ /dev/null @@ -1,137 +0,0 @@ -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.services.gtfs_constants import * -from google.protobuf.json_format import MessageToDict -from google.protobuf.json_format import ParseDict -from datetime import datetime, timedelta -import json -import re -import time - -class GtfsRtProducer(): - - def __init__(self, trip_store): - self.trip_store = trip_store - - def generate_feed(self, time, format='protobuf', bbox=None): - # See https://developers.google.com/transit/gtfs-realtime/reference - # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java - gtfsrt_dict = { - 'header': { - 'gtfsRealtimeVersion': '1.0', - 'timestamp': int(time) - }, - 'entity': self._get_trip_updates(bbox) - } - feed = gtfs_realtime_pb2.FeedMessage() - ParseDict(gtfsrt_dict, feed) - - if "message" == format.lower(): - return feed - elif "json" == format.lower(): - return MessageToDict(feed) - else: - return feed.SerializeToString() - - def export_feed(self, timestamp, file_path, bbox=None): - """ - Exports gtfs-rt feed as .json and .pbf file to file_path - """ - feed = self.generate_feed(timestamp, "message", bbox) - with open(f"{file_path}.pbf", "wb") as f: - f.write(feed.SerializeToString()) - with open(f"{file_path}.json", "w") as f: - json.dump(MessageToDict(feed), f) - - def _get_trip_updates(self, bbox = None): - trips = [] - trips.extend(self._get_added(bbox)) - trips.extend(self._get_deleted(bbox)) - trip_updates = [] - for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) - return trip_updates - - def _get_deleted(self, bbox = None): - return self._get_updates( - self.trip_store.recently_deleted_trips(), - self._as_delete_updates, - bbox) - - def _get_added(self, bbox = None): - return self._get_updates( - self.trip_store.recently_added_trips(), - self._as_added_updates, - bbox) - - def _get_updates(self, trips, update_func, bbox = None): - updates = [] - today = datetime.today() - for t in trips: - if bbox == None or t.intersects(bbox): - updates.extend(update_func(t, today)) - return updates - - def _as_delete_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'CANCELED', - 'routeId': trip.trip_id - } - } for trip_date in trip.next_trip_dates(fromdate)] - - def _to_seconds(self, fromdate, stop_time): - startdate = datetime.strptime(fromdate, '%Y%m%d') - m = re.search(r'(\d+):(\d+):(\d+)', stop_time) - delta = timedelta( - hours=int(m.group(1)), - minutes=int(m.group(2)), - seconds=int(m.group(3))) - return time.mktime((startdate + delta).timetuple()) - - def _to_stop_times(self, trip, fromdate): - return [{ - 'stopSequence': stoptime.stop_sequence, - 'arrival': { - 'time': self._to_seconds(fromdate, stoptime.arrival_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'departure': { - 'time': self._to_seconds(fromdate, stoptime.departure_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'stopId': stoptime.stop_id, - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' - } - } - } - for stoptime in trip.stop_times] - - def _as_added_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'ADDED', - 'routeId': trip.trip_id, - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : trip.url, - 'agencyId' : trip.agency, - 'route_long_name' : trip.route_long_name(), - 'route_type': RIDESHARING_ROUTE_TYPE - } - }, - 'stopTimeUpdate': self._to_stop_times(trip, trip_date) - } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/services/gtfs_constants.py b/amarillo/services/gtfs_constants.py deleted file mode 100644 index 1e8f3af..0000000 --- a/amarillo/services/gtfs_constants.py +++ /dev/null @@ -1,14 +0,0 @@ -# Constants - -NO_BIKES_ALLOWED = 2 -RIDESHARING_ROUTE_TYPE = 1551 -CALENDAR_DATES_EXCEPTION_TYPE_ADDED = 1 -CALENDAR_DATES_EXCEPTION_TYPE_REMOVED = 2 -STOP_TIMES_STOP_TYPE_REGULARLY = 0 -STOP_TIMES_STOP_TYPE_NONE = 1 -STOP_TIMES_STOP_TYPE_PHONE_AGENCY = 2 -STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER = 3 -STOP_TIMES_TIMEPOINT_APPROXIMATE = 0 -STOP_TIMES_TIMEPOINT_EXACT = 1 - -MFDZ_DEFAULT_UNCERTAINITY = 600 \ No newline at end of file diff --git a/amarillo/services/gtfs_export.py b/amarillo/services/gtfs_export.py deleted file mode 100644 index fb39425..0000000 --- a/amarillo/services/gtfs_export.py +++ /dev/null @@ -1,229 +0,0 @@ - -from collections.abc import Iterable -from datetime import datetime, timedelta -from zipfile import ZipFile -import csv -import gettext -import logging -import re - -from amarillo.utils.utils import assert_folder_exists -from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.services.stops import is_carpooling_stop -from amarillo.services.gtfs_constants import * - - -logger = logging.getLogger(__name__) - -class GtfsExport: - - stops_counter = 0 - trips_counter = 0 - routes_counter = 0 - - stored_stops = {} - - def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): - self.stops = {} - self.routes = [] - self.calendar_dates = [] - self.calendar = [] - self.trips = [] - self.stop_times = [] - self.calendar = [] - self.shapes = [] - self.agencies = agencies - self.feed_info = feed_info - self.localized_to = " nach " - self.localized_short_name = "Mitfahrgelegenheit" - self.stopstore = stopstore - self.ridestore = ridestore - self.bbox = bbox - - def export(self, gtfszip_filename, gtfsfolder): - assert_folder_exists(gtfsfolder) - self._prepare_gtfs_feed(self.ridestore, self.stopstore) - self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) - self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) - self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) - self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) - self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) - self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) - self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) - self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) - self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) - self._zip_files(gtfszip_filename, gtfsfolder) - - def _zip_files(self, gtfszip_filename, gtfsfolder): - gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', - 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] - with ZipFile(gtfszip_filename, 'w') as gtfszip: - for gtfsfile in gtfsfiles: - gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) - - def _prepare_gtfs_feed(self, ridestore, stopstore): - """ - Prepares all gtfs objects in memory before they are written - to their respective streams. - - For all wellknown stops a GTFS stop is created and - afterwards all ride offers are transformed into their - gtfs equivalents. - """ - for stopSet in stopstore.stopsDataFrames: - for stop in stopSet["stops"].itertuples(): - self._load_stored_stop(stop) - cloned_trips = dict(ridestore.trips) - for url, trip in cloned_trips.items(): - if self.bbox is None or trip.intersects(self.bbox): - self._convert_trip(trip) - - def _convert_trip(self, trip): - self.routes_counter += 1 - self.routes.append(self._create_route(trip)) - self.calendar.append(self._create_calendar(trip)) - if not trip.runs_regularly: - self.calendar_dates.append(self._create_calendar_date(trip)) - self.trips.append(self._create_trip(trip, self.routes_counter)) - self._append_stops_and_stop_times(trip) - self._append_shapes(trip, self.routes_counter) - - def _trip_headsign(self, destination): - destination = destination.replace('(Deutschland)', '') - destination = destination.replace(', Deutschland', '') - appendix = '' - if 'Schweiz' in destination or 'Switzerland' in destination: - appendix = ', Schweiz' - destination = destination.replace('(Schweiz)', '') - destination = destination.replace(', Schweiz', '') - destination = destination.replace('(Switzerland)', '') - - try: - matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) - - match = matches.group(3).strip() if matches != None else destination.strip() - if match[-1]==')' and not '(' in match: - match = match[0:-1] - - return match + appendix - except Exception as ex: - logger.error("error for "+destination ) - logger.exception(ex) - return destination - - def _create_route(self, trip): - return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") - - def _create_calendar(self, trip): - # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. - # We could apply some heuristics like requesting multiple days and extrapolate - # if multiple trips are found, but better would be to have these provided by the - # offical interface. Then validity periods should be provided as well (not - # sure if these are available) - # For fahrgemeinschaft.de, regurlar trips are recognizable via their url - # which contains "regelmaessig". However, we don't know on which days of the week, - # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, - # if it's sa/su, only this... - - feed_start_date = datetime.today() - stop_date = self._convert_stop_date(feed_start_date) - return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) - - def _create_calendar_date(self, trip): - return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) - - def _create_trip(self, trip, shape_id): - return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) - - def _convert_stop(self, stop): - """ - Converts a stop represented as pandas row to a gtfs stop. - Expected attributes of stop: id, stop_name, x, y (in wgs84) - """ - if stop.id: - id = stop.id - else: - self.stops_counter += 1 - id = "tmp-{}".format(self.stops_counter) - - stop_name = "k.A." if stop.stop_name is None else stop.stop_name - return GtfsStop(id, stop.y, stop.x, stop_name) - - def _append_stops_and_stop_times(self, trip): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - for stop_time in trip.stop_times: - # retrieve stop from stored_stops and mark it to be exported - wkn_stop = self.stored_stops.get(stop_time.stop_id) - if not wkn_stop: - logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) - else: - self.stops[stop_time.stop_id] = wkn_stop - # Append stop_time - self.stop_times.append(stop_time) - - def _append_shapes(self, trip, shape_id): - counter = 0 - for point in trip.path.coordinates: - counter += 1 - self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) - - def _stop_hash(self, stop): - return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) - - def _should_always_export(self, stop): - """ - Returns true, if the given stop shall be exported to GTFS, - regardless, if it's part of a trip or not. - - This is necessary, as potential stops are required - to be part of the GTFS to be referenced later on - by dynamicly added trips. - """ - if self.bbox: - return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and - self.bbox[1] <= stop.stop_lat <= self.bbox[3]) - else: - return is_carpooling_stop(stop.stop_id, stop.stop_name) - - def _load_stored_stop(self, stop): - gtfsstop = self._convert_stop(stop) - stop_hash = self._stop_hash(stop) - self.stored_stops[gtfsstop.stop_id] = gtfsstop - if self._should_always_export(gtfsstop): - self.stops[gtfsstop.stop_id] = gtfsstop - - def _get_stop_by_hash(self, stop_hash): - return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) - - def _get_or_create_stop(self, stop): - stop_hash = self._stop_hash(stop) - gtfsstop = self.stops.get(stop_hash) - if gtfsstop is None: - gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) - self.stops[stop_hash] = gtfsstop - return gtfsstop - - def _convert_stop_date(self, date_time): - return date_time.strftime("%Y%m%d") - - def _write_csvfile(self, gtfsfolder, filename, content): - with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: - self._write_csv(csvfile, content) - - def _write_csv(self, csvfile, content): - if hasattr(content, '_fields'): - writer = csv.DictWriter(csvfile, content._fields) - writer.writeheader() - writer.writerow(content._asdict()) - else: - if content: - writer = csv.DictWriter(csvfile, next(iter(content))._fields) - writer.writeheader() - for record in content: - writer.writerow(record._asdict()) - - \ No newline at end of file diff --git a/amarillo/services/gtfs_generator.py b/amarillo/services/gtfs_generator.py deleted file mode 100644 index 113d5db..0000000 --- a/amarillo/services/gtfs_generator.py +++ /dev/null @@ -1,71 +0,0 @@ -from amarillo.models.Carpool import Region -from amarillo.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.utils.container import container -from glob import glob -import json -import schedule -import threading -import time -import logging -from datetime import date, timedelta - -logger = logging.getLogger(__name__) - -regions = {} -for region_file_name in glob('conf/region/*.json'): - with open(region_file_name) as region_file: - dict = json.load(region_file) - region = Region(**dict) - region_id = region.id - regions[region_id] = region - -agencies = [] -for agency_file_name in glob('conf/agency/*.json'): - with open(agency_file_name) as agency_file: - dict = json.load(agency_file) - agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) - agency_id = agency.agency_id - agencies.append(agency) - -def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) - -def midnight(): - container['stops_store'].load_stop_sources() - container['trips_store'].unflag_unrecent_updates() - container['carpools'].purge_outdated_offers() - generate_gtfs() - -def generate_gtfs(): - logger.info("Generate GTFS") - - for region in regions.values(): - # TODO make feed producer infos configurable - feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) - exporter = GtfsExport( - agencies, - feed_info, - container['trips_store'], - container['stops_store'], - region.bbox) - exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") - -def generate_gtfs_rt(): - logger.info("Generate GTFS-RT") - producer = GtfsRtProducer(container['trips_store']) - for region in regions.values(): - rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) - -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() \ No newline at end of file diff --git a/amarillo/services/gtfsrt/__init__.py b/amarillo/services/gtfsrt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/services/gtfsrt/gtfs_realtime_pb2.py deleted file mode 100644 index 4e10463..0000000 --- a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: gtfs-realtime.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' - _FEEDMESSAGE._serialized_start=41 - _FEEDMESSAGE._serialized_end=162 - _FEEDHEADER._serialized_start=165 - _FEEDHEADER._serialized_end=380 - _FEEDHEADER_INCREMENTALITY._serialized_start=312 - _FEEDHEADER_INCREMENTALITY._serialized_end=364 - _FEEDENTITY._serialized_start=383 - _FEEDENTITY._serialized_end=593 - _TRIPUPDATE._serialized_start=596 - _TRIPUPDATE._serialized_end=1622 - _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 - _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 - _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 - _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 - _VEHICLEPOSITION._serialized_start=1625 - _VEHICLEPOSITION._serialized_end=2872 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 - _ALERT._serialized_start=2875 - _ALERT._serialized_end=4027 - _ALERT_CAUSE._serialized_start=3497 - _ALERT_CAUSE._serialized_end=3713 - _ALERT_EFFECT._serialized_start=3716 - _ALERT_EFFECT._serialized_end=3937 - _ALERT_SEVERITYLEVEL._serialized_start=3939 - _ALERT_SEVERITYLEVEL._serialized_end=4011 - _TIMERANGE._serialized_start=4029 - _TIMERANGE._serialized_end=4084 - _POSITION._serialized_start=4086 - _POSITION._serialized_end=4199 - _TRIPDESCRIPTOR._serialized_start=4202 - _TRIPDESCRIPTOR._serialized_end=4535 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 - _VEHICLEDESCRIPTOR._serialized_start=4537 - _VEHICLEDESCRIPTOR._serialized_end=4622 - _ENTITYSELECTOR._serialized_start=4625 - _ENTITYSELECTOR._serialized_end=4801 - _TRANSLATEDSTRING._serialized_start=4804 - _TRANSLATEDSTRING._serialized_end=4970 - _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 - _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/services/gtfsrt/realtime_extension_pb2.py b/amarillo/services/gtfsrt/realtime_extension_pb2.py deleted file mode 100644 index 5db1fda..0000000 --- a/amarillo/services/gtfsrt/realtime_extension_pb2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: realtime_extension.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) - gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\007de.mfdz' - _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 - _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/services/hooks.py b/amarillo/services/hooks.py new file mode 100644 index 0000000..5713585 --- /dev/null +++ b/amarillo/services/hooks.py @@ -0,0 +1,27 @@ +from typing import List +from amarillo.models.Carpool import Carpool + +class CarpoolEvents: + def on_create(cp : Carpool): + pass + def on_update(cp : Carpool): + pass + def on_delete(cp : Carpool): + pass + +carpool_event_listeners : List[CarpoolEvents] = [] + +def register_carpool_event_listener(cpe : CarpoolEvents): + carpool_event_listeners.append(cpe) + +def run_on_create(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_create(cp) + +def run_on_update(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_update(cp) + +def run_on_delete(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_delete(cp) \ No newline at end of file diff --git a/amarillo/services/oauth2.py b/amarillo/services/oauth2.py new file mode 100644 index 0000000..7c99ec7 --- /dev/null +++ b/amarillo/services/oauth2.py @@ -0,0 +1,165 @@ +# OAuth2 authentication based on https://fastapi.tiangolo.com/tutorial/security/oauth2-jwt/#__tabbed_4_2 + +from datetime import datetime, timedelta, timezone +from typing import Annotated, Optional, Union +import logging +import logging.config + +from fastapi import Depends, HTTPException, Header, status, APIRouter +from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm +from jose import JWTError, jwt +from pydantic import BaseModel +from amarillo.models.User import User +from amarillo.services.passwords import verify_password +from amarillo.utils.container import container +from amarillo.services.agencies import AgencyService +from amarillo.services.users import UserService +from amarillo.models.Carpool import Agency + +from amarillo.services.secrets import secrets + +SECRET_KEY = secrets.secret_key +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 30 + +logging.config.fileConfig('logging.conf', disable_existing_loggers=False) +logger = logging.getLogger("main") + +router = APIRouter() + +class Token(BaseModel): + access_token: str + token_type: str + +class TokenData(BaseModel): + user_id: Union[str, None] = None + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) +async def verify_optional_api_key(X_API_Key: Optional[str] = Header(None)): + if X_API_Key == None: return None + return await verify_api_key(X_API_Key) + +def authenticate_user(user_id: str, password: str): + user_service : UserService = container['users'] + user_conf = user_service.user_id_to_user_conf.get(user_id, None) + if not user_conf: + return False + + if not verify_password(password, user_conf.password): + return False + return user_id + + +def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None): + to_encode = data.copy() + if expires_delta: + expire = datetime.now(timezone.utc) + expires_delta + else: + expire = datetime.now(timezone.utc) + timedelta(minutes=15) + to_encode.update({"exp": expire}) + encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + return encoded_jwt + + +async def get_current_user(token: str = Depends(oauth2_scheme), user_from_api_key: str = Depends(verify_optional_api_key)) -> User: + if token: + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate OAuth2 credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + user_id: str = payload.get("sub") + if user_id is None: + raise credentials_exception + token_data = TokenData(user_id=user_id) + except JWTError: + raise credentials_exception + user_id = token_data.user_id + if user_id is None: + raise credentials_exception + + user_service : UserService = container['users'] + return user_service.get_user(user_id) + elif user_from_api_key: + logger.info(f"API Key provided: {user_from_api_key}") + user_service : UserService = container['users'] + return user_service.get_user(user_from_api_key) + else: + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": "Bearer"}, + ) + raise credentials_exception + +def verify_permission(permission: str, user: User): + + def permissions_exception(): + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"User '{user.user_id}' does not have the permission '{permission}'", + headers={"WWW-Authenticate": "Bearer"}, + ) + + #user is admin + if "admin" in user.permissions: return + + #permission is an operation + if ":" not in permission: + if permission not in user.permissions: + raise permissions_exception() + + return + + #permission is in agency:operation format + def permission_matches(permission, user_permission): + prescribed_agency, prescribed_operation = permission.split(":") + given_agency, given_operation = user_permission.split(":") + + return (prescribed_agency == given_agency or given_agency == "all") and (prescribed_operation == given_operation or given_operation == "all") + + if any(permission_matches(permission, p) for p in user.permissions if ":" in p): return + + raise permissions_exception() + + +# noinspection PyPep8Naming +# X_API_Key is upper case for OpenAPI +async def verify_api_key(X_API_Key: str = Header(...)): + user_service: UserService = container['users'] + + return user_service.check_api_key(X_API_Key) + +@router.post("/token") +async def login_for_access_token( + form_data: Annotated[OAuth2PasswordRequestForm, Depends()] +) -> Token: + agency = authenticate_user(form_data.username, form_data.password) + if not agency: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Bearer"}, + ) + access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) + access_token = create_access_token( + data={"sub": agency}, expires_delta=access_token_expires + ) + return Token(access_token=access_token, token_type="bearer") + +# TODO: eventually remove this +@router.get("/users/me/", response_model=Agency) +async def read_users_me( + current_agency: Annotated[Agency, Depends(get_current_user)] +): + agency_service : AgencyService = container['agencies'] + return agency_service.get_agency(agency_id=current_agency) + +# TODO: eventually remove this +@router.get("/users/me/items/") +async def read_own_items( + current_agency: Annotated[str, Depends(get_current_user)] +): + return [{"item_id": "Foo", "owner": current_agency}] diff --git a/amarillo/services/passwords.py b/amarillo/services/passwords.py new file mode 100644 index 0000000..fa010dc --- /dev/null +++ b/amarillo/services/passwords.py @@ -0,0 +1,10 @@ +from passlib.context import CryptContext + +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + +def verify_password(plain_password, hashed_password): + return pwd_context.verify(plain_password, hashed_password) + + +def get_password_hash(password): + return pwd_context.hash(password) \ No newline at end of file diff --git a/amarillo/services/regions.py b/amarillo/services/regions.py index 8be79e0..425b3ac 100644 --- a/amarillo/services/regions.py +++ b/amarillo/services/regions.py @@ -9,8 +9,7 @@ class RegionService: def __init__(self): self.regions: Dict[str, Region] = {} - - for region_file_name in glob('conf/region/*.json'): + for region_file_name in glob('data/region/*.json'): with open(region_file_name) as region_file: dict = json.load(region_file) region = Region(**dict) diff --git a/amarillo/services/routing.py b/amarillo/services/routing.py deleted file mode 100644 index 96f6229..0000000 --- a/amarillo/services/routing.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests -import logging - -logger = logging.getLogger(__name__) - -class RoutingException(Exception): - def __init__(self, message): - # Call Exception.__init__(message) - # to use the same Message header as the parent class - super().__init__(message) - -class RoutingService(): - def __init__(self, gh_url = 'https://api.mfdz.de/gh'): - self.gh_service_url = gh_url - - def path_for_stops(self, points): - # Retrieve graphhopper route traversing given points - directions = self._get_directions(points) - if directions and len(directions.get("paths"))>0: - return directions.get("paths")[0] - else: - return {} - - def _get_directions(self, points): - req_url = self._create_url(points, True, True) - logger.debug("Get directions via: {}".format(req_url)) - response = requests.get(req_url) - status = response.status_code - if status == 200: - # Found route between points - return response.json() - else: - try: - message = response.json().get('message') - except: - raise RoutingException("Get directions failed with status code {}".format(status)) - else: - raise RoutingException(message) - - def _create_url(self, points, calc_points = False, instructions = False): - """ Creates GH request URL """ - locations = "" - for point in points: - locations += "point={0}%2C{1}&".format(point.y, point.x) - - return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false&profile=car".format( - self.gh_service_url, locations, instructions, calc_points) diff --git a/amarillo/services/secrets.py b/amarillo/services/secrets.py index 756d21c..c77e8d8 100644 --- a/amarillo/services/secrets.py +++ b/amarillo/services/secrets.py @@ -2,9 +2,9 @@ from pydantic_settings import BaseSettings # Example: secrets = { "mfdz": "some secret" } class Secrets(BaseSettings): - model_config = ConfigDict(extra='allow') + model_config = ConfigDict(extra='allow') # Allow plugins to add extra values ride2go_token: str = Field(None, env = 'RIDE2GO_TOKEN') - + secret_key: str = Field(None, env = 'SECRET_KEY') # Read if file exists, otherwise no error (it's in .gitignore) secrets = Secrets(_env_file='secrets', _env_file_encoding='utf-8') diff --git a/amarillo/services/stops.py b/amarillo/services/stops.py deleted file mode 100644 index 1c00791..0000000 --- a/amarillo/services/stops.py +++ /dev/null @@ -1,131 +0,0 @@ -import codecs -import csv -import logging -import re -from contextlib import closing -from io import TextIOWrapper - -import geopandas as gpd -import pandas as pd -import requests -from pyproj import Proj, Transformer -from shapely.geometry import LineString, Point -from shapely.ops import transform - -from amarillo.models.Carpool import StopTime - -from .stop_importer import CsvStopsImporter, GeojsonStopsImporter, GtfsStopsImporter, OverpassStopsImporter - -logger = logging.getLogger(__name__) - - -def is_carpooling_stop(stop_id, name): - stop_name = name.lower() - # mfdz: or bbnavi: prefixed stops are custom stops which are explicitly meant to be carpooling stops - return stop_id.startswith('mfdz:') or stop_id.startswith('bbnavi:') or 'mitfahr' in stop_name or 'p&m' in stop_name - - -class StopsStore: - def __init__(self, stop_sources=None, internal_projection='EPSG:32632'): - self.internal_projection = internal_projection - self.projection = Transformer.from_crs('EPSG:4326', internal_projection, always_xy=True).transform - self.stopsDataFrames = [] - self.stop_sources = stop_sources if stop_sources is not None else [] - - def load_stop_sources(self): - """Imports stops from stop_sources and registers them with - the distance they are still associated with a trip. - E.g. bus stops should be registered with a distance of e.g. 30m, - while larger carpool parkings might be registered with e.g. 500m. - - Subsequent calls of load_stop_sources will reload all stop_sources - but replace the current stops only if all stops could be loaded successfully. - """ - stopsDataFrames = [] - error_occured = False - - for stops_source in self.stop_sources: - try: - source_url = stops_source.get('url') - source_type = stops_source.get('type') or ( - 'geojson' - if source_url is not None and source_url.startswith('http') and source_url.endswith('json') - else 'csv' - ) - logger.info('Loading stop source %s...', stops_source.get('id')) - match source_type: - case 'geojson': - stopsDataFrame = GeojsonStopsImporter().load_stops(source_url) - case 'csv': - stopsDataFrame = CsvStopsImporter().load_stops(source_url) - case 'overpass': - stopsDataFrame = OverpassStopsImporter().load_stops(**stops_source) - case 'gtfs': - stopsDataFrame = GtfsStopsImporter().load_stops(**stops_source) - case _: - logger.error('Failed to load stops, source type %s not supported', source_type) - continue - stopsDataFrame.to_crs(crs=self.internal_projection, inplace=True) - stopsDataFrames.append({'distanceInMeter': stops_source['vicinity'], 'stops': stopsDataFrame}) - except Exception: - error_occured = True - logger.error('Failed to load stops from %s to StopsStore.', stops_source, exc_info=True) - - if not error_occured: - self.stopsDataFrames = stopsDataFrames - - def find_additional_stops_around(self, line, stops=None): - """Returns a GeoDataFrame with all stops in vicinity of the - given line, sorted by distance from origin of the line. - Note: for internal projection/distance calculations, the - lat/lon geometries of line and stops are converted to - """ - stops_frames = [] - if stops: - stops_frames.append(self._convert_to_dataframe(stops)) - transformedLine = transform(self.projection, LineString(line.coordinates)) - for stops_to_match in self.stopsDataFrames: - stops_frames.append( - self._find_stops_around_transformed( - stops_to_match['stops'], transformedLine, stops_to_match['distanceInMeter'] - ) - ) - stops = gpd.GeoDataFrame(pd.concat(stops_frames, ignore_index=True, sort=True)) - if not stops.empty: - self._sort_by_distance(stops, transformedLine) - return stops - - def find_closest_stop(self, carpool_stop, max_search_distance): - transformedCoord = Point(self.projection(carpool_stop.lon, carpool_stop.lat)) - best_dist = max_search_distance + 1 - best_stop = None - for stops_with_dist in self.stopsDataFrames: - stops = stops_with_dist['stops'] - s, d = stops.sindex.nearest( - transformedCoord, return_all=True, return_distance=True, max_distance=max_search_distance - ) - if len(d) > 0 and d[0] < best_dist: - best_dist = d[0] - row = s[1][0] - best_stop = StopTime(name=stops.at[row, 'stop_name'], lat=stops.at[row, 'y'], lon=stops.at[row, 'x']) - - return best_stop if best_stop else carpool_stop - - def _find_stops_around_transformed(self, stopsDataFrame, transformedLine, distance): - bufferedLine = transformedLine.buffer(distance) - sindex = stopsDataFrame.sindex - possible_matches_index = list(sindex.intersection(bufferedLine.bounds)) - possible_matches = stopsDataFrame.iloc[possible_matches_index] - - return possible_matches[possible_matches.intersects(bufferedLine)] - - def _convert_to_dataframe(self, stops): - return gpd.GeoDataFrame( - [[stop.name, stop.lon, stop.lat, stop.id, Point(self.projection(stop.lon, stop.lat))] for stop in stops], - columns=['stop_name', 'x', 'y', 'id', 'geometry'], - crs=self.internal_projection, - ) - - def _sort_by_distance(self, stops, transformedLine): - stops['distance'] = stops.apply(lambda row: transformedLine.project(row['geometry']), axis=1) - stops.sort_values('distance', inplace=True) diff --git a/amarillo/services/trips.py b/amarillo/services/trips.py deleted file mode 100644 index fcbc21b..0000000 --- a/amarillo/services/trips.py +++ /dev/null @@ -1,375 +0,0 @@ -from amarillo.services.config import config -from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime -from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType -from amarillo.services.gtfs_constants import * -from amarillo.services.routing import RoutingService, RoutingException -from amarillo.services.stops import is_carpooling_stop -from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m -from shapely.geometry import Point, LineString, box -from geojson_pydantic.geometries import LineString as GeoJSONLineString -from datetime import datetime, timedelta -import numpy as np -import os -import json -import logging - -logger = logging.getLogger(__name__) - -class Trip: - - def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, bbox): - if isinstance(calendar, set): - self.runs_regularly = True - self.weekdays = [ - 1 if Weekday.monday in calendar else 0, - 1 if Weekday.tuesday in calendar else 0, - 1 if Weekday.wednesday in calendar else 0, - 1 if Weekday.thursday in calendar else 0, - 1 if Weekday.friday in calendar else 0, - 1 if Weekday.saturday in calendar else 0, - 1 if Weekday.sunday in calendar else 0, - ] - start_in_day = self._total_seconds(departureTime) - else: - self.start = datetime.combine(calendar, departureTime) - self.runs_regularly = False - self.weekdays = [0,0,0,0,0,0,0] - - self.start_time = departureTime - self.path = path - self.trip_id = trip_id - self.url = url - self.agency = agency - self.stops = [] - self.lastUpdated = lastUpdated - self.stop_times = stop_times - self.bbox = bbox - self.route_name = route_name - self.trip_headsign = headsign - - def path_as_line_string(self): - return path - - def _total_seconds(self, instant): - return instant.hour * 3600 + instant.minute * 60 + instant.second - - def start_time_str(self): - return self.start_time.strftime("%H:%M:%S") - - def next_trip_dates(self, start_date, day_count=14): - if self.runs_regularly: - for single_date in (start_date + timedelta(n) for n in range(day_count)): - if self.weekdays[single_date.weekday()]==1: - yield single_date.strftime("%Y%m%d") - else: - yield self.start.strftime("%Y%m%d") - - def route_long_name(self): - return self.route_name - - def intersects(self, bbox): - return self.bbox.intersects(box(*bbox)) - - -class TripStore(): - """ - TripStore maintains the currently valid trips. A trip is a - carpool offer enhanced with all stops this - - Attributes: - trips Dict of currently valid trips. - deleted_trips Dict of recently deleted trips. - """ - - def __init__(self, stops_store): - self.transformer = TripTransformer(stops_store) - self.stops_store = stops_store - self.trips = {} - self.deleted_trips = {} - self.recent_trips = {} - - - def put_carpool(self, carpool: Carpool): - """ - Adds carpool to the TripStore. - """ - id = "{}:{}".format(carpool.agency, carpool.id) - filename = f'data/enhanced/{carpool.agency}/{carpool.id}.json' - try: - existing_carpool = self._load_carpool_if_exists(carpool.agency, carpool.id) - if existing_carpool and existing_carpool.lastUpdated == carpool.lastUpdated: - enhanced_carpool = existing_carpool - else: - if len(carpool.stops) < 2 or self.distance_in_m(carpool) < 1000: - logger.warning("Failed to add carpool %s:%s to TripStore, distance too low", carpool.agency, carpool.id) - self.handle_failed_carpool_enhancement(carpool) - return - enhanced_carpool = self.transformer.enhance_carpool(carpool) - # TODO should only store enhanced_carpool, if it has 2 or more stops - assert_folder_exists(f'data/enhanced/{carpool.agency}/') - with open(filename, 'w', encoding='utf-8') as f: - f.write(enhanced_carpool.json()) - logger.info("Added enhanced carpool %s:%s", carpool.agency, carpool.id) - - return self._load_as_trip(enhanced_carpool) - except RoutingException as err: - logger.warning("Failed to add carpool %s:%s to TripStore due to RoutingException %s", carpool.agency, carpool.id, getattr(err, 'message', repr(err))) - self.handle_failed_carpool_enhancement(carpool) - except Exception as err: - logger.error("Failed to add carpool %s:%s to TripStore.", carpool.agency, carpool.id, exc_info=True) - self.handle_failed_carpool_enhancement(carpool) - - def handle_failed_carpool_enhancement(sellf, carpool: Carpool): - assert_folder_exists(f'data/failed/{carpool.agency}/') - with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: - f.write(carpool.json()) - - def distance_in_m(self, carpool): - if len(carpool.stops) < 2: - return 0 - s1 = carpool.stops[0] - s2 = carpool.stops[-1] - return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) - - def recently_added_trips(self): - return list(self.recent_trips.values()) - - def recently_deleted_trips(self): - return list(self.deleted_trips.values()) - - def _load_carpool_if_exists(self, agency_id: str, carpool_id: str): - if carpool_exists(agency_id, carpool_id, 'data/enhanced'): - try: - return load_carpool(agency_id, carpool_id, 'data/enhanced') - except Exception as e: - # An error on restore could be caused by model changes, - # in such a case, it need's to be recreated - logger.warning("Could not restore enhanced trip %s:%s, reason: %s", agency_id, carpool_id, repr(e)) - - return None - - def _load_as_trip(self, carpool: Carpool): - trip = self.transformer.transform_to_trip(carpool) - id = trip.trip_id - self.trips[id] = trip - if not is_older_than_days(carpool.lastUpdated, 1): - self.recent_trips[id] = trip - logger.debug("Added trip %s", id) - - return trip - - def delete_carpool(self, agency_id: str, carpool_id: str): - """ - Deletes carpool from the TripStore. - """ - agencyScopedCarpoolId = f"{agency_id}:{carpool_id}" - trip_to_be_deleted = self.trips.get(agencyScopedCarpoolId) - if trip_to_be_deleted: - self.deleted_trips[agencyScopedCarpoolId] = trip_to_be_deleted - del self.trips[agencyScopedCarpoolId] - - if self.recent_trips.get(agencyScopedCarpoolId): - del self.recent_trips[agencyScopedCarpoolId] - - if carpool_exists(agency_id, carpool_id): - remove_carpool_file(agency_id, carpool_id) - - logger.debug("Deleted trip %s", id) - - def unflag_unrecent_updates(self): - """ - Trips that were last updated before yesterday, are not recent - any longer. As no updates need to be sent for them any longer, - they will be removed from recent recent_trips and deleted_trips. - """ - for key in list(self.recent_trips): - t = self.recent_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.recent_trips[key] - - for key in list(self.deleted_trips): - t = self.deleted_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.deleted_trips[key] - - -class TripTransformer: - REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS = True - REPLACEMENT_STOPS_SERACH_RADIUS_IN_M = 1000 - SIMPLIFY_TOLERANCE = 0.0001 - - router = RoutingService(config.graphhopper_base_url) - - def __init__(self, stops_store): - self.stops_store = stops_store - - def transform_to_trip(self, carpool): - stop_times = self._convert_stop_times(carpool) - route_name = carpool.stops[0].name + " nach " + carpool.stops[-1].name - headsign= carpool.stops[-1].name - trip_id = self._trip_id(carpool) - path = carpool.path - bbox = box( - min([pt[0] for pt in path.coordinates]), - min([pt[1] for pt in path.coordinates]), - max([pt[0] for pt in path.coordinates]), - max([pt[1] for pt in path.coordinates])) - - trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, bbox) - - return trip - - def _trip_id(self, carpool): - return f"{carpool.agency}:{carpool.id}" - - def _replace_stops_by_transit_stops(self, carpool, max_search_distance): - new_stops = [] - for carpool_stop in carpool.stops: - new_stops.append(self.stops_store.find_closest_stop(carpool_stop, max_search_distance)) - return new_stops - - def enhance_carpool(self, carpool): - if self.REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS: - carpool.stops = self._replace_stops_by_transit_stops(carpool, self.REPLACEMENT_STOPS_SERACH_RADIUS_IN_M) - - path = self._path_for_ride(carpool) - lineString_shapely_wgs84 = LineString(coordinates = path["points"]["coordinates"]).simplify(0.0001) - lineString_wgs84 = GeoJSONLineString(type="LineString", coordinates=list(lineString_shapely_wgs84.coords)) - virtual_stops = self.stops_store.find_additional_stops_around(lineString_wgs84, carpool.stops) - if not virtual_stops.empty: - virtual_stops["time"] = self._estimate_times(path, virtual_stops['distance']) - logger.debug("Virtual stops found: {}".format(virtual_stops)) - if len(virtual_stops) > MAX_STOPS_PER_TRIP: - # in case we found more than MAX_STOPS_PER_TRIP, we retain first and last - # half of MAX_STOPS_PER_TRIP - virtual_stops = virtual_stops.iloc[np.r_[0:int(MAX_STOPS_PER_TRIP/2), int(MAX_STOPS_PER_TRIP/2):]] - - trip_id = f"{carpool.agency}:{carpool.id}" - stop_times = self._stops_and_stop_times(carpool.departureTime, trip_id, virtual_stops) - - enhanced_carpool = carpool.copy() - enhanced_carpool.stops = stop_times - enhanced_carpool.path = lineString_wgs84 - return enhanced_carpool - - def _convert_stop_times(self, carpool): - - stop_times = [GtfsStopTime( - self._trip_id(carpool), - stop.arrivalTime, - stop.departureTime, - stop.id, - seq_nr+1, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_dropoff else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_pickup else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_TIMEPOINT_APPROXIMATE) - for seq_nr, stop in enumerate(carpool.stops)] - return stop_times - - def _path_for_ride(self, carpool): - points = self._stop_coords(carpool.stops) - return self.router.path_for_stops(points) - - def _stop_coords(self, stops): - # Retrieve coordinates of all officially announced stops (start, intermediate, target) - return [Point(stop.lon, stop.lat) for stop in stops] - - def _estimate_times(self, path, distances_from_start): - cumulated_distance = 0 - cumulated_time = 0 - stop_times = [] - instructions = path["instructions"] - - cnt = 0 - instr_distance = instructions[cnt]["distance"] - instr_time = instructions[cnt]["time"] - - for distance in distances_from_start: - while cnt < len(instructions) and cumulated_distance + instructions[cnt]["distance"] < distance: - cumulated_distance = cumulated_distance + instructions[cnt]["distance"] - cumulated_time = cumulated_time + instructions[cnt]["time"] - cnt = cnt + 1 - - if cnt < len(instructions): - if instructions[cnt]["distance"] ==0: - raise RoutingException("Origin and destinaction too close") - percent_dist = (distance - cumulated_distance) / instructions[cnt]["distance"] - stop_time = cumulated_time + percent_dist * instructions[cnt]["time"] - stop_times.append(stop_time) - else: - logger.debug("distance {} exceeds total length {}, using max arrival time {}".format(distance, cumulated_distance, cumulated_time)) - stop_times.append(cumulated_time) - return stop_times - - def _stops_and_stop_times(self, start_time, trip_id, stops_frame): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - number_of_stops = len(stops_frame.index) - total_distance = stops_frame.iloc[number_of_stops-1]["distance"] - - first_stop_time = GtfsTimeDelta(hours = start_time.hour, minutes = start_time.minute, seconds = start_time.second) - stop_times = [] - seq_nr = 0 - for i in range(0, number_of_stops): - current_stop = stops_frame.iloc[i] - - if not current_stop.id: - continue - elif i == 0: - if (stops_frame.iloc[1].time-current_stop.time) < 1000: - # skip custom stop if there is an official stop very close by - logger.debug("Skipped stop %s", current_stop.id) - continue - else: - if (current_stop.time-stops_frame.iloc[i-1].time) < 5000 and not i==1 and not is_carpooling_stop(current_stop.id, current_stop.stop_name): - # skip latter stop if it's very close (<5 seconds drive) by the preceding - logger.debug("Skipped stop %s", current_stop.id) - continue - trip_time = timedelta(milliseconds=int(current_stop.time)) - is_dropoff = self._is_dropoff_stop(current_stop, total_distance) - is_pickup = self._is_pickup_stop(current_stop, total_distance) - # TODO would be nice if possible to publish a minimum shared distance - pickup_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_pickup else STOP_TIMES_STOP_TYPE_NONE - dropoff_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_dropoff else STOP_TIMES_STOP_TYPE_NONE - - if is_pickup and not is_dropoff: - pickup_dropoff = PickupDropoffType.only_pickup - elif not is_pickup and is_dropoff: - pickup_dropoff = PickupDropoffType.only_dropoff - else: - pickup_dropoff = PickupDropoffType.pickup_and_dropoff - - next_stop_time = first_stop_time + trip_time - seq_nr += 1 - stop_times.append(StopTime(**{ - 'arrivalTime': str(next_stop_time), - 'departureTime': str(next_stop_time), - 'id': current_stop.id, - 'pickup_dropoff': pickup_dropoff, - 'name': str(current_stop.stop_name), - 'lat': current_stop.y, - 'lon': current_stop.x - })) - - return stop_times - - def _is_dropoff_stop(self, current_stop, total_distance): - return current_stop["distance"] >= 0.5 * total_distance - - def _is_pickup_stop(self, current_stop, total_distance): - return current_stop["distance"] < 0.5 * total_distance - -def load_carpool(agency_id: str, carpool_id: str, folder: str ='data/enhanced') -> Carpool: - with open(f'{folder}/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - return carpool - -def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") - -def remove_carpool_file(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.remove(f"{folder}/{agency_id}/{carpool_id}.json") diff --git a/amarillo/services/users.py b/amarillo/services/users.py new file mode 100644 index 0000000..9866c2f --- /dev/null +++ b/amarillo/services/users.py @@ -0,0 +1,116 @@ +import json +import os +from glob import glob +from typing import Dict, List +import logging + +from fastapi import HTTPException, status + +from amarillo.models.User import User +from amarillo.services.config import config +from amarillo.services.passwords import get_password_hash + +logger = logging.getLogger(__name__) + +user_conf_directory = 'data/users' + + +class UserService: + + def __init__(self): + # Both Dicts to be kept in sync always. The second api_key_to_agency_id is like a reverse + # cache for the first for fast lookup of valid api keys, which happens on *every* request. + self.user_id_to_user_conf: Dict[str, User] = {} + self.api_key_to_user_id: Dict[str, str] = {} + + for user_conf_file_name in glob(f'{user_conf_directory}/*.json'): + with open(user_conf_file_name) as user_conf_file: + dictionary = json.load(user_conf_file) + + user_conf = User(**dictionary) + + agency_id = user_conf.user_id + api_key = user_conf.api_key + + self.user_id_to_user_conf[agency_id] = user_conf + if api_key is not None: + self.api_key_to_user_id[api_key] = user_conf.user_id + + def get_user(self, user_id: str) -> User: + user_conf = self.user_id_to_user_conf.get(user_id) + return user_conf + + def check_api_key(self, api_key: str) -> str: + """Check if the API key is valid + + The agencies' api keys are checked first, and the admin's key. + + The agency_id or "admin" is returned for further checks in the caller if the + request is permitted, like {agency_id} == agency_id. + """ + + agency_id = self.api_key_to_user_id.get(api_key) + + is_agency = agency_id is not None + + if is_agency: + return agency_id + + is_admin = api_key == config.admin_token + + if is_admin: + return "admin" + + message = "X-API-Key header invalid" + logger.error(message) + raise HTTPException(status_code=400, detail=message) + + def add(self, user_conf: User): + + user_id = user_conf.user_id + api_key = user_conf.api_key + + agency_id_exists_already = self.user_id_to_user_conf.get(user_id) is not None + + if agency_id_exists_already: + message = f"Agency {user_id} exists already. To update, delete it first." + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + agency_using_this_api_key_already = self.api_key_to_user_id.get(api_key) + a_different_agency_is_using_this_api_key_already = \ + agency_using_this_api_key_already is not None and \ + agency_using_this_api_key_already != user_id + + if a_different_agency_is_using_this_api_key_already: + message = f"Duplicate API Key for {user_id} not permitted. Use a different key." + logger.error(message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + + user_conf.password = get_password_hash(user_conf.password) + + with open(f'{user_conf_directory}/{user_id}.json', 'w', encoding='utf-8') as f: + f.write(user_conf.json()) + + self.user_id_to_user_conf[user_id] = user_conf + if api_key: + self.api_key_to_user_id[api_key] = user_id + + logger.info(f"Added configuration for user {user_id}.") + + def get_user_ids(self) -> List[str]: + return list(self.user_id_to_user_conf.keys()) + + def delete(self, user_id): + + user_conf = self.user_id_to_user_conf.get(user_id) + + api_key = user_conf.api_key + + del self.api_key_to_user_id[api_key] + + del self.user_id_to_user_conf[user_id] + + os.remove(f'{user_conf_directory}/{user_id}.json') + + logger.info(f"Deleted configuration for {user_id}.") diff --git a/config b/amarillo/static/config similarity index 100% rename from config rename to amarillo/static/config diff --git a/conf/agency/mfdz.json b/amarillo/static/data/agency/mfdz.json similarity index 100% rename from conf/agency/mfdz.json rename to amarillo/static/data/agency/mfdz.json diff --git a/conf/agency/mifaz.json b/amarillo/static/data/agency/mifaz.json similarity index 100% rename from conf/agency/mifaz.json rename to amarillo/static/data/agency/mifaz.json diff --git a/conf/agency/ride2go.json b/amarillo/static/data/agency/ride2go.json similarity index 100% rename from conf/agency/ride2go.json rename to amarillo/static/data/agency/ride2go.json diff --git a/conf/agency/ummadum.json b/amarillo/static/data/agency/ummadum.json similarity index 100% rename from conf/agency/ummadum.json rename to amarillo/static/data/agency/ummadum.json diff --git a/conf/region/bb.json b/amarillo/static/data/region/bb.json similarity index 100% rename from conf/region/bb.json rename to amarillo/static/data/region/bb.json diff --git a/conf/region/bw.json b/amarillo/static/data/region/bw.json similarity index 100% rename from conf/region/bw.json rename to amarillo/static/data/region/bw.json diff --git a/conf/region/by.json b/amarillo/static/data/region/by.json similarity index 100% rename from conf/region/by.json rename to amarillo/static/data/region/by.json diff --git a/conf/region/nrw.json b/amarillo/static/data/region/nrw.json similarity index 100% rename from conf/region/nrw.json rename to amarillo/static/data/region/nrw.json diff --git a/conf/stop_sources.json b/amarillo/static/data/stop_sources.json similarity index 100% rename from conf/stop_sources.json rename to amarillo/static/data/stop_sources.json diff --git a/amarillo/static/logging.conf b/amarillo/static/logging.conf new file mode 100644 index 0000000..142287a --- /dev/null +++ b/amarillo/static/logging.conf @@ -0,0 +1,28 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler, fileHandler +propagate=yes + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=handlers.RotatingFileHandler +level=ERROR +formatter=simpleFormatter +args=('error.log', 'a', 1000000, 3) # Filename, mode, maxBytes, backupCount + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/static/css/docs.css b/amarillo/static/static/css/docs.css similarity index 100% rename from static/css/docs.css rename to amarillo/static/static/css/docs.css diff --git a/static/css/theme.css b/amarillo/static/static/css/theme.css similarity index 100% rename from static/css/theme.css rename to amarillo/static/static/css/theme.css diff --git a/static/img/cloud.png b/amarillo/static/static/img/cloud.png similarity index 100% rename from static/img/cloud.png rename to amarillo/static/static/img/cloud.png diff --git a/static/img/favicon.ico b/amarillo/static/static/img/favicon.ico similarity index 100% rename from static/img/favicon.ico rename to amarillo/static/static/img/favicon.ico diff --git a/templates/home/index.html b/amarillo/static/templates/home/index.html similarity index 100% rename from templates/home/index.html rename to amarillo/static/templates/home/index.html diff --git a/templates/shared/layout.html b/amarillo/static/templates/shared/layout.html similarity index 100% rename from templates/shared/layout.html rename to amarillo/static/templates/shared/layout.html diff --git a/amarillo/tests/stops.csv b/amarillo/tests/stops.csv deleted file mode 100644 index ed419bb..0000000 --- a/amarillo/tests/stops.csv +++ /dev/null @@ -1,5 +0,0 @@ -stop_id;stop_code;stop_lat;stop_lon;stop_name -mfdz:x;x;52.11901;14.2;Stop x -mfdz:y;y;53.1;14.01;Stop y -mfdz:z;z;54.11;14.0;Stop z -mfdz:Ang001;Ang001;53.11901;14.015776;Mitfahrbank Biesenbrow diff --git a/amarillo/tests/stops.json b/amarillo/tests/stops.json deleted file mode 100644 index 5744c13..0000000 --- a/amarillo/tests/stops.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "data": { - "pointsOfInterest": [ - { - "id": "14622", - "externalId": "bbnavi:12073:0001", - "name": "Parkbank", - "description": "Parkbank", - "dataProvider": { - "id": "1", - "name": "Administrator" - }, - "addresses": [ - { - "street": "Hauptstrasse", - "city": "Wittenberge", - "zip": "12345", - "geoLocation": { - "latitude": 52.9932971109789, - "longitude": 11.767383582547 - } - } - ], - "openStreetMap": { - "capacity": 112, - "capacityCharging": "2", - "capacityDisabled": "", - "fee": "No", - "lit": "Yes", - "parking": "", - "shelter": "No", - "surface": "", - "utilization": "", - "website": "" - } - } - ] - } -} \ No newline at end of file diff --git a/amarillo/tests/test_gtfs.py b/amarillo/tests/test_gtfs.py deleted file mode 100644 index 61a8e5a..0000000 --- a/amarillo/tests/test_gtfs.py +++ /dev/null @@ -1,142 +0,0 @@ -from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue -from amarillo.services.gtfs_export import GtfsExport -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.services.stops import StopsStore -from amarillo.services.trips import TripStore -from amarillo.models.Carpool import Carpool -from datetime import datetime -import time -import pytest - - -def test_gtfs_generation(): - cp = Carpool(**data1) - stops_store = StopsStore() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - - exporter = GtfsExport(None, None, trips_store, stops_store) - exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") - -def test_correct_stops(): - cp = Carpool(**stop_issue) - stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) - stops_store.load_stop_sources() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - assert len(trips_store.trips) == 1 - - -class TestTripConverter: - - def setup_method(self, method): - self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) - self.trips_store = TripStore(self.stops_store) - - def test_as_one_time_trip_as_delete_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Eins' - } - }] - - def test_as_one_time_trip_as_added_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_added_updates(trip, datetime(2022,4,11)) - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'ADDED', - 'routeId': 'mfdz:Eins', - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : 'https://mfdz.de/trip/123', - 'agencyId' : 'mfdz', - 'route_long_name' : 'abc nach xyz', - 'route_type': 1551 - } - }, - 'stopTimeUpdate': [{ - 'stopSequence': 1, - 'arrival': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'stopId': 'mfdz:12073:001', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' - } - } - }, - { - 'stopSequence': 2, - 'arrival': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - - 'stopId': 'de:12073:900340137::3', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER', - 'pickupType': 'NONE' - } - } - }] - }] - - def test_as_periodic_trip_as_delete_update(self): - cp = Carpool(**carpool_repeating_json) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220411', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - }, - { - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220418', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - } - ] \ No newline at end of file diff --git a/amarillo/tests/test_permissions.py b/amarillo/tests/test_permissions.py new file mode 100644 index 0000000..5e2ded2 --- /dev/null +++ b/amarillo/tests/test_permissions.py @@ -0,0 +1,37 @@ +from fastapi import HTTPException +import pytest +from amarillo.services.oauth2 import verify_permission +from amarillo.models.User import User + +test_user = User(user_id="test", password="testpassword", permissions=["all:read", "mfdz:write", "ride2go:all", "gtfs"]) +admin_user = User(user_id="admin", password="testpassword", permissions=["admin"]) + +def test_operation(): + verify_permission("gtfs", test_user) + + with pytest.raises(HTTPException): + verify_permission("geojson", test_user) + +def test_agency_permission(): + verify_permission("mvv:read", test_user) + verify_permission("mfdz:read", test_user) + verify_permission("mfdz:write", test_user) + verify_permission("ride2go:write", test_user) + + with pytest.raises(HTTPException): + verify_permission("mvv:write", test_user) + verify_permission("mvv:all", test_user) + + +def test_admin(): + verify_permission("admin", admin_user) + verify_permission("gtfs", admin_user) + verify_permission("all:all", admin_user) + verify_permission("mvv:all", admin_user) + verify_permission("mfdz:read", admin_user) + verify_permission("mfdz:write", admin_user) + verify_permission("ride2go:write", admin_user) + + with pytest.raises(HTTPException): + verify_permission("admin", test_user) + verify_permission("all:all", test_user) \ No newline at end of file diff --git a/amarillo/tests/test_stops_store.py b/amarillo/tests/test_stops_store.py deleted file mode 100644 index 931a562..0000000 --- a/amarillo/tests/test_stops_store.py +++ /dev/null @@ -1,28 +0,0 @@ -from amarillo.models.Carpool import StopTime -from amarillo.services import stops - - -def test_load_stops_from_file(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_csv_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://data.mfdz.de/mfdz/stops/custom.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_geojson_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://datahub.bbnavi.de/export/rideshare_points.geojson', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_find_closest_stop(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - carpool_stop = StopTime(name='start', lat=53.1191, lon=14.01577) - stop = store.find_closest_stop(carpool_stop, 1000) - assert stop.name == 'Mitfahrbank Biesenbrow' diff --git a/amarillo/tests/test_trip_store.py b/amarillo/tests/test_trip_store.py deleted file mode 100644 index 96c9616..0000000 --- a/amarillo/tests/test_trip_store.py +++ /dev/null @@ -1,23 +0,0 @@ -from amarillo.tests.sampledata import cp1, carpool_repeating -from amarillo.services.trips import TripStore -from amarillo.services.stops import StopsStore - - -import logging -logger = logging.getLogger(__name__) - -def test_trip_store_put_one_time_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(cp1) - assert t != None - assert len(t.stop_times) >= 2 - assert t.stop_times[0].stop_id == 'mfdz:12073:001' - assert t.stop_times[-1].stop_id == 'de:12073:900340137::3' - -def test_trip_store_put_repeating_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(carpool_repeating) - assert t != None - assert len(t.stop_times) >= 2 diff --git a/amarillo/utils/utils.py b/amarillo/utils/utils.py index c7c1075..9d060b3 100644 --- a/amarillo/utils/utils.py +++ b/amarillo/utils/utils.py @@ -1,8 +1,16 @@ import os import re +import shutil +from pathlib import Path +import logging + from datetime import datetime, date, timedelta from pyproj import Geod +logger = logging.getLogger(__name__) +#logging.conf may not exist yet, so we need to configure the logger to show infos +logging.basicConfig(level=logging.INFO) + def assert_folder_exists(foldername): if not os.path.isdir(foldername): os.makedirs(foldername, exist_ok=True) @@ -37,4 +45,30 @@ def geodesic_distance_in_m(coord1, coord2): geod = Geod(ellps="WGS84") lons = [coord1[0], coord2[0]] lats = [coord1[1], coord2[1]] - return geod.line_lengths(lons, lats)[0] \ No newline at end of file + return geod.line_lengths(lons, lats)[0] + + +def copy_static_files(files_and_dirs_to_copy): + amarillo_dir = Path(__file__).parents[1] + source_dir = os.path.join(amarillo_dir, "static") + + destination_dir = os.getcwd() + + for item in files_and_dirs_to_copy: + source_path = os.path.join(source_dir, item) + destination_path = os.path.join(destination_dir, item) + + if not os.path.exists(source_path): + raise FileNotFoundError(source_path) + + if os.path.exists(destination_path): + # logger.info(f"{item} already exists") + continue + + if os.path.isfile(source_path): + shutil.copy2(source_path, destination_path) + logger.info(f"Copied {item} to {destination_path}") + + if os.path.isdir(source_path): + shutil.copytree(source_path, destination_path) + logger.info(f"Copied directory {item} and its contents to {destination_path}") \ No newline at end of file diff --git a/enhancer.py b/enhancer.py deleted file mode 100644 index 1bdbb87..0000000 --- a/enhancer.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import time -import logging -import logging.config -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -from amarillo.configuration import configure_enhancer_services -from amarillo.utils.container import container -from amarillo.models.Carpool import Carpool -from amarillo.utils.utils import agency_carpool_ids_from_filename - -logging.config.fileConfig('logging.conf', disable_existing_loggers=False) -logger = logging.getLogger("enhancer") - -logger.info("Hello Enhancer") - -configure_enhancer_services() - -observer = Observer() # Watch Manager - - -class EventHandler(FileSystemEventHandler): - # TODO FG HB should watch for both carpools and agencies - # in data/agency, data/agencyconf, see AgencyConfService - - def on_closed(self, event): - - logger.info("CLOSE_WRITE: Created %s", event.src_path) - try: - with open(event.src_path, 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - - container['carpools'].put(carpool.agency, carpool.id, carpool) - except FileNotFoundError as e: - logger.error("Carpool could not be added, as already deleted (%s)", event.src_path) - except: - logger.exception("Eventhandler on_closed encountered exception") - - def on_deleted(self, event): - try: - logger.info("DELETE: Removing %s", event.src_path) - (agency_id, carpool_id) = agency_carpool_ids_from_filename(event.src_path) - container['carpools'].delete(agency_id, carpool_id) - except: - logger.exception("Eventhandler on_deleted encountered exception") - - -observer.schedule(EventHandler(), 'data/carpool', recursive=True) -observer.start() - -import time - -try: - # TODO FG Is this really needed? - cnt = 0 - ENHANCER_LOG_INTERVAL_IN_S = 600 - while True: - if cnt == ENHANCER_LOG_INTERVAL_IN_S: - logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) - cnt = 0 - - time.sleep(1) - cnt += 1 -finally: - observer.stop() - observer.join() - - logger.info("Goodbye Enhancer") diff --git a/logging.conf b/logging.conf deleted file mode 100644 index 429da8e..0000000 --- a/logging.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler -propagate=yes - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index af5be75..761fe35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "amarillo" -version = "0.0.15a3" +version = "0.0.17a1" description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" readme = "README.md" license = {file = "LICENSE"} @@ -16,6 +16,9 @@ dependencies = [ "pyproj==3.6.1", "geojson-pydantic==1.0.1", "watchdog==3.0.0", + "python-jose[cryptography]", + "bcrypt==4.0.1", + "passlib[bcrypt]" ] [tool.setuptools.packages] diff --git a/requirements.txt b/requirements.txt index 292a8f4..0c65e02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,7 @@ Shapely==2.0.2 pygeos==0.14 pyproj==3.6.1 geojson-pydantic==1.0.1 -watchdog==3.0.0 \ No newline at end of file +watchdog==3.0.0 +python-jose[cryptography] +bcrypt==4.0.1 +passlib[bcrypt] \ No newline at end of file diff --git a/standard.Dockerfile b/standard.Dockerfile new file mode 100644 index 0000000..3066d0b --- /dev/null +++ b/standard.Dockerfile @@ -0,0 +1,16 @@ +ARG DOCKER_REGISTRY +FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base + +ARG PLUGINS=\ +"amarillo-metrics "\ +"amarillo-gtfs-exporter " + +ARG PACKAGE_REGISTRY_URL + +ENV METRICS_USER='' +ENV METRICS_PASSWORD='' + +# RUN pip install $PLUGINS + +RUN --mount=type=secret,id=AMARILLO_REGISTRY_CREDENTIALS \ +pip install --no-cache-dir --upgrade --extra-index-url https://$(cat /run/secrets/AMARILLO_REGISTRY_CREDENTIALS)@${PACKAGE_REGISTRY_URL} ${PLUGINS}