diff --git a/.gitignore b/.gitignore index 3d975f6c..95871420 100644 --- a/.gitignore +++ b/.gitignore @@ -6,22 +6,22 @@ # data uploads tmp_files -b2stage/* # virtualenv - -# Skip self-signed certificates +b2stage/* data/certs/nginx-selfsigned* data/certs/*cineca* data/secrets/secret_* +data/b2handle/*.pem +data/files/* ################# ## Submodules ## ################# submodules/.backup -submodules/utilities -submodules/builds_base -submodules/backend +submodules/build-templates +submodules/utils +submodules/http-api submodules/frontend submodules/prc @@ -29,8 +29,8 @@ submodules/prc ## Dynamically created file and folders ## ############################################ -.projectrc -frontend/eudat/libs/bower_components +#.projectrc +frontend/b2stage/libs/bower_components .env projects/*/backend/confs/project_configuration.yaml diff --git a/.projectrc b/.projectrc new file mode 100644 index 00000000..14475b3c --- /dev/null +++ b/.projectrc @@ -0,0 +1,6 @@ +#log-level: VERY_VERBOSE +#log-level: VERBOSE +mode: debug +project: b2stage +#mode: production +#hostname: b2stage.cineca.it diff --git a/.travis.yml b/.travis.yml index ec996a78..abef50fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: services: - docker env: -- PROJECT=eudat +- PROJECT=b2stage # # installation # before_install: diff --git a/CHANGELOG.md b/CHANGELOG.md index 4db8dff9..3d6364e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,30 @@ -# v0.7.0-rc4 (*near future*) -**due date**: October 15th, 2017 +# v1.0.1 (*near future*) +**due date**: November 30th, 2017 ## features -- Connect prototype to production services #90 -- Checklist for good projects #62 -- to be defined +- to be defined in the next developer meeting -# v0.6.2-rc3 (*currently developed*) -**due date**: October 1th, 2017 -**project**: [webinar](https://github.com/EUDAT-B2STAGE/http-api/projects/3) +# v1.0.0 (*first stable release*) +**due date**: October 24th, 2017 +**milestone**: [stable release](https://github.com/EUDAT-B2STAGE/http-api/milestones/8) ## features -- Define a protocol to release production to be tested automatically #55 -- to be defined +- Benchmarks #20 +- Unittests upgrade (from `nose2` to `py.test`) #105 +- A publish endpoint to make data available to anyone/anonymous #72 +- PIDS refactor #106 +- Automatically recover from failures #103 +- Landing page #42 +- Bug fixes + +## known issues + +Currently B2ACCESS and B2SAFE are facing problems in sharing certificated credentials. This is under investigation and will be discussed in the upcoming developer meeting. It is not an issue related to B2STAGE HTTP-API code. -# v0.6.1-rc2 (*latest stable*) +# v0.6.1-rc2 **due date**: September 9th, 2017 **project**: [release candidate 2](https://github.com/EUDAT-B2STAGE/http-api/milestone/9) diff --git a/LICENSE b/LICENSE index 3db509ab..77bfce9a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2016 B2STAGE service core code for EUDAT project +Copyright 2011-2017 EUDAT CDI - www.eudat.eu Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 42dfed89..8f35ec52 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,9 @@ # B2STAGE HTTP-API for EUDAT project -| build | coverage | quality | uptime | swagger | -| --- | --- | --- | --- | --- | -| [![Build Status](https://travis-ci.org/EUDAT-B2STAGE/http-api.svg?branch=master)](https://travis-ci.org/EUDAT-B2STAGE/http-api) | [![Coverage Status](https://coveralls.io/repos/github/EUDAT-B2STAGE/http-api/badge.svg?branch=HEAD)](https://coveralls.io/github/EUDAT-B2STAGE/http-api?branch=master) | [![Code Health](https://landscape.io/github/EUDAT-B2STAGE/http-api/master/landscape.svg?style=flat)](https://landscape.io/github/EUDAT-B2STAGE/http-api/master) | [![Uptime Robot](https://img.shields.io/uptimerobot/ratio/m778586640-4e31f2b00e90bce508dcdf33.svg?maxAge=2592000)](https://stats.uptimerobot.com/xGG9gTK3q) | [![Swagger validation](https://img.shields.io/swagger/valid/2.0/https/b2stage.cineca.it/api/specs.svg)](http://petstore.swagger.io/?url=https://b2stage.cineca.it/api/specs&docExpansion=none) | - -NOTE: the B2STAGE HTTP-API is an interface towards other EUDAT services which are subject to modifications and new developments, therefore the functionalities and the implementation will have to be changed accordingly. - -Feedback on our first Release Candidate: [![Gitter](https://badges.gitter.im/EUDAT-B2STAGE/http-api.svg)](https://gitter.im/EUDAT-B2STAGE/http-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +| build | coverage | quality | uptime | swagger | feedback | +| --- | --- | --- | --- | --- | --- | +| [![Build Status](https://travis-ci.org/EUDAT-B2STAGE/http-api.svg?branch=master)](https://travis-ci.org/EUDAT-B2STAGE/http-api) | [![Coverage Status](https://coveralls.io/repos/github/EUDAT-B2STAGE/http-api/badge.svg?branch=master)](https://coveralls.io/github/EUDAT-B2STAGE/http-api?branch=master) | [![Code Health](https://landscape.io/github/EUDAT-B2STAGE/http-api/master/landscape.svg?style=flat)](https://landscape.io/github/EUDAT-B2STAGE/http-api/master) | [![Uptime Robot](https://img.shields.io/uptimerobot/ratio/m778586640-4e31f2b00e90bce508dcdf33.svg?maxAge=2592000)](https://stats.uptimerobot.com/xGG9gTK3q) | [![Swagger validation](https://img.shields.io/swagger/valid/2.0/https/b2stage.cineca.it/api/specs.svg)](http://petstore.swagger.io/?url=https://b2stage.cineca.it/api/specs&docExpansion=none) | [![Gitter](https://badges.gitter.im/EUDAT-B2STAGE/http-api.svg)](https://gitter.im/EUDAT-B2STAGE/http-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) | --- @@ -23,22 +19,37 @@ The primary goal is to allow users to ingest and retrieve data via a standard RE - lower the entry barrier to using EUDAT services, - simplify integration into existing workflows, - allow direct access to data assets held with the EUDAT CDI. - -The first development is focused on the interaction with B2SAFE, allowing users to transfer and manage data on the "registered" domain. + ## Documentation -Documentation for the latest stable release: - -- [Quick start](docs/user/user.md) for deploy and development -- [Deploy](docs/deploy/deploy.md) the server -- [User](docs/user/user.md) guide -- [Developer](docs/development/development.md) instructions +The first stable release is focused on the interaction with the `B2SAFE` service, allowing users to transfer and manage data on the `registered` **domain**. + +- Get started + + try a [quick start](docs/user/user.md) + + see the latest [webinar](https://pdonorio.github.io/chapters/webinars/b2stage) + + a demo [prototype](docs/prototype.md) of the current functionalities +- User guide + + [use](docs/user/user.md) an existing instance of the `HTTP-API` + + [main authentication](docs/user/authentication.md) based on `B2ACCESS` + + [alternative authentication](docs/user/authentication_b2safe.md) based on `B2SAFE` + + [endpoints](docs/user/endpoints.md) description and examples +- Administration + + read the [pre-requisites](docs/deploy/preq.md) prior to installation + + [startup](docs/deploy/startup.md) a working copy + + deploy in two [modes](docs/deploy/modes.md) + + [debug](docs/deploy/debugging.md) the server operating on top of `B2SAFE` and fix eventual issues +- Development + + [add](docs/development/development.md) new features + + extra [operations](docs/development/operations.md) + ## Implementation @@ -48,13 +59,11 @@ This project is based on the open-source [RAPyDo framework](https://github.com/r The API server is implemented with the latest Python 3 release using the Flask core. To facilitate and speed the development Docker is the base environment to automate the creation and configuration of the software stack needed. The API interconnects with EUDAT services' native APIs or libraries. -To exchange data and metadata within the B2SAFE service (built on iRODS system as backend storage) the interface is implemented using the official [python-irodsclient](https://github.com/irods/python-irodsclient). +To exchange data and metadata within the B2SAFE service (built on iRODS system as backend storage) the interface is implemented using (and contributing to) the official [python-irodsclient](https://github.com/irods/python-irodsclient). - +## Copyright + +Copyright 2011-2017 EUDAT CDI - www.eudat.eu diff --git a/confs/backend.yml b/confs/backend.yml index f9f6be35..ea19db97 100644 --- a/confs/backend.yml +++ b/confs/backend.yml @@ -7,16 +7,16 @@ version: '3' volumes: jwt_tokens: driver: local - rabbitdata: - driver: local + # rabbitdata: + # driver: local sqldata: driver: local - graphdata: - driver: local - mongodata: - driver: local - pureftpd: - driver: local + # graphdata: + # driver: local + # mongodata: + # driver: local + # pureftpd: + # driver: local letsencrypt_certs: driver: local ssl_dhparam: @@ -31,11 +31,11 @@ volumes: networks: # default: + # worker_net: + # ftp_net: i_net: app_net: proxy_net: - worker_net: - ftp_net: db_net: # # driver: bridge # ipam: @@ -54,7 +54,7 @@ services: ######################## backend: # command: sleep infinity - build: ../submodules/builds_base/backend + build: ../submodules/build-templates/backend image: ${COMPOSE_PROJECT_NAME}/backend:template # hostname: restapi hostname: rapydo_server @@ -67,6 +67,8 @@ services: DEBUG_ENDPOINTS: 0 # base the user/role mechanism on some database AUTH_ENABLE: 0 + # disable the basic /api/login method + MAIN_LOGIN_ENABLE: 0 # AUTH_SERVICE: ${AUTH_SERVICE} # # if Unicode is complaining with print # PYTHONIOENCODING: UTF-8 @@ -77,9 +79,6 @@ services: - ../projects/${COMPOSE_PROJECT_NAME}/backend:/code/${COMPOSE_PROJECT_NAME} # JWT tokens secret - jwt_tokens:${JWT_APP_SECRETS} - # Unit tests - - ../submodules/backend/tests:/code/tests - - ../projects/${COMPOSE_PROJECT_NAME}/backend/tests:/code/tests/custom networks: app_net: aliases: @@ -89,7 +88,7 @@ services: ############################## proxy: - build: ../submodules/builds_base/proxy + build: ../submodules/build-templates/proxy image: ${COMPOSE_PROJECT_NAME}/proxy:template hostname: reverseproxy volumes: @@ -117,10 +116,12 @@ services: ### DATABASES ### ################### postgres: - image: postgres:9.6 + # image: postgres:9.6 + # image: postgres:9.6.5-alpine + image: postgres:10.0-alpine volumes: - sqldata:/var/lib/postgresql/data - - ../submodules/builds_base/postgres/pgs_init.sh:/docker-entrypoint-initdb.d/setup-my-schema.sh:ro + - ../submodules/build-templates/postgres/pgs_init.sh:/docker-entrypoint-initdb.d/init-db-and-users.sh:ro environment: ACTIVATE: 0 POSTGRES_USER: "${PLACEHOLDER}" @@ -131,168 +132,176 @@ services: aliases: - ${ALCHEMY_HOST} - neo4j: - image: neo4j:3.2.1 - volumes: - - graphdata:/data - networks: - db_net: - aliases: - - ${GRAPHDB_HOST} - environment: - ACTIVATE: 0 - NEO4J_AUTH: neo4j/${PLACEHOLDER} - # NEO4J_dbms_memory_pagecache_size: 2048M - # NEO4J_dbms_memory_heap_maxSize: 4096M - # ports: - # - 9090:7474 - # - 7687:7687 + # neo4j: + # image: neo4j:3.2.2 + # # image: neo4j:3.2.3 + # # volumes: + # # - graphdata:/data + # # networks: + # # db_net: + # # aliases: + # # - ${GRAPHDB_HOST} + # environment: + # ACTIVATE: 0 + # NEO4J_AUTH: neo4j/${PLACEHOLDER} + # # NEO4J_dbms_memory_pagecache_size: 2048M + # # NEO4J_dbms_memory_heap_maxSize: 4096M + # # ports: + # # - 9090:7474 + # # - 7687:7687 - mongodb: - image: mongo:3.4 - volumes: - - mongodata:/data/db - networks: - db_net: - aliases: - - ${MONGO_HOST} - environment: - ACTIVATE: 0 + # mongodb: + # image: mongo:3.4 + # # volumes: + # # - mongodata:/data/db + # # networks: + # # db_net: + # # aliases: + # # - ${MONGO_HOST} + # environment: + # ACTIVATE: 0 ######################### ### QUEUE MANAGEMENT ### ######################### - celery: - build: ../submodules/builds_base/celery - image: ${COMPOSE_PROJECT_NAME}/celery:template - # hostname: celworker - # command: celery worker -c 1 -A rapydo.flask_ext.flask_celery.worker.celery_app - user: root - working_dir: /code - environment: - ACTIVATE: 0 - VANILLA_PACKAGE: ${COMPOSE_PROJECT_NAME} - JWT_APP_SECRETS: ${JWT_APP_SECRETS} - volumes: - # FIXME: UHM, what is this @mattia? - # Base code - - ../submodules/backend:/code - # Vanilla code - - ../projects/${COMPOSE_PROJECT_NAME}/backend:/code/${COMPOSE_PROJECT_NAME} - # JWT tokens secret - - jwt_tokens:${JWT_APP_SECRETS} - networks: - db_net: - worker_net: - depends_on: - - rabbit - # - flower + # celery: + # build: ../submodules/build-templates/celery + # image: ${COMPOSE_PROJECT_NAME}/celery:template + # # hostname: celworker + # # command: celery worker -c 1 -A rapydo.flask_ext.flask_celery.worker.celery_app + # user: root + # working_dir: /code + # environment: + # ACTIVATE: 0 + # VANILLA_PACKAGE: ${COMPOSE_PROJECT_NAME} + # JWT_APP_SECRETS: ${JWT_APP_SECRETS} + # volumes: + # # Base code + # - ../submodules/http-api:/code + # # Vanilla code + # - ../projects/${COMPOSE_PROJECT_NAME}/backend:/code/${COMPOSE_PROJECT_NAME} + # # JWT tokens secret + # - jwt_tokens:${JWT_APP_SECRETS} + # networks: + # db_net: + # worker_net: + # depends_on: + # - rabbit + # # - flower - rabbit: - image: rabbitmq:latest - hostname: rabbit - volumes: - - rabbitdata:/var/lib/rabbitmq - networks: - worker_net: - aliases: - - ${CELERY_BROKER_HOST} - environment: - ACTIVATE: 0 + # rabbit: + # image: rabbitmq:latest + # hostname: rabbit + # volumes: + # - rabbitdata:/var/lib/rabbitmq + # networks: + # worker_net: + # aliases: + # - ${CELERY_BROKER_HOST} + # environment: + # ACTIVATE: 0 + # RABBITMQ_DEFAULT_USER + # RABBITMQ_DEFAULT_PASS + # https://hub.docker.com/_/rabbitmq/ + # http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html ########################## ### SERVICE INTERFACES ### ########################## - celeryui: - build: ../submodules/builds_base/celery - image: ${COMPOSE_PROJECT_NAME}/celery:template - hostname: flower - user: root - working_dir: /code - command: start_flower - expose: - - 5555 - ports: - - 5555:5555 - environment: - ACTIVATE: 0 - VANILLA_PACKAGE: ${COMPOSE_PROJECT_NAME} - JWT_APP_SECRETS: ${JWT_APP_SECRETS} - volumes: - - ..:/code/${COMPOSE_PROJECT_NAME} - - ../tmp_irods_certificates:/opt/certificates - - ../tmp_files:/uploads - networks: - db_net: - worker_net: - depends_on: - - rabbit + # http://mher.github.io/flower/reverse-proxy.html + # celeryui: + # build: ../submodules/build-templates/celery + # image: ${COMPOSE_PROJECT_NAME}/celery:template + # hostname: flower + # user: root + # working_dir: /code + # command: start_flower + # expose: + # - 5555 + # ports: + # - 5555:5555 + # environment: + # ACTIVATE: 0 + # VANILLA_PACKAGE: ${COMPOSE_PROJECT_NAME} + # JWT_APP_SECRETS: ${JWT_APP_SECRETS} + # volumes: + # - ..:/code/${COMPOSE_PROJECT_NAME} + # - ../tmp_irods_certificates:/opt/certificates + # - ../tmp_files:/uploads + # networks: + # db_net: + # worker_net: + # depends_on: + # - rabbit swaggerui: - build: ../submodules/builds_base/swaggerui + build: ../submodules/build-templates/swaggerui image: ${COMPOSE_PROJECT_NAME}/swaggerui:template environment: ACTIVATE: 0 sqlalchemyui: - build: ../submodules/builds_base/sqlalchemyui - image: ${COMPOSE_PROJECT_NAME}/sqlalchemyui:template # NOTE: this image let you access sqllite/postgres/mysql # with a phpmyadmin-like web page + image: adminer:4.3.1-standalone + # build: ../submodules/build-templates/sqlalchemyui + # image: ${COMPOSE_PROJECT_NAME}/sqlalchemyui:template networks: db_net: environment: ACTIVATE: 0 + # https://github.com/vrana/adminer/tree/master/designs + # ADMINER_DESIGN: brade - mongoui: - image: mongo-express:0.40.0 - environment: - ACTIVATE: 0 - ME_CONFIG_MONGODB_SERVER: ${MONGO_HOST} - networks: - db_net: + # mongoui: + # image: mongo-express:0.40.0 + # environment: + # ACTIVATE: 0 + # ME_CONFIG_MONGODB_SERVER: ${MONGO_HOST} + # networks: + # db_net: ############# # REST API client (wget, curl, httpie, http-prompt) restclient: - build: ../submodules/builds_base/restclient + build: ../submodules/build-templates/restclient image: ${COMPOSE_PROJECT_NAME}/restclient:template hostname: rapydo_client environment: MAIN_ENDPOINT: ${API_MAIN_ENDPOINT} ACTIVATE: 0 volumes: - # - ../submodules/utilities/utilities/projects_defaults.yaml:/code/base.yml + # - ../submodules/utils/utilities/projects_defaults.yaml:/code/base.yml - ../projects/${COMPOSE_PROJECT_NAME}/project_configuration.yaml:/code/custom.yml ################### ### FTP SERVER ### ################### - ftp: - # image: stilliard/pure-ftpd:latest - build: ../submodules/builds_base/ftp - image: ${COMPOSE_PROJECT_NAME}/ftp:template - volumes: - - pureftpd:/etc/pure-ftpd/passwd - # /etc/ssl/private/ A directory containing a single pure-ftpd.pem file - # with the server's SSL certificates for TLS support. Optional TLS is - # automatically enabled when the container finds this file on startup. - environment: - ACTIVATE: 0 - PUBLICHOST: ${PROJECT_DOMAIN} - ADDED_FLAGS: -d -d - networks: - ftp_net: + # ftp: + # # image: stilliard/pure-ftpd:latest + # build: ../submodules/build-templates/ftp + # image: ${COMPOSE_PROJECT_NAME}/ftp:template + # volumes: + # - pureftpd:/etc/pure-ftpd/passwd + # # /etc/ssl/private/ A directory containing a single pure-ftpd.pem file + # # with the server's SSL certificates for TLS support. Optional TLS is + # # automatically enabled when the container finds this file on startup. + # environment: + # ACTIVATE: 0 + # PUBLICHOST: ${PROJECT_DOMAIN} + # ADDED_FLAGS: -d -d + # networks: + # ftp_net: ###################################################### ### iRODS iCAT server (usually for EUDAT B2safe) ### ###################################################### icat: # B2safe instance on irods - build: ../submodules/builds_base/icat + build: ../submodules/build-templates/icat image: ${COMPOSE_PROJECT_NAME}/icat:template # image: b2safe:rapydo hostname: ${IRODS_HOST} @@ -308,6 +317,7 @@ services: IRODS_DB: ${PLACEHOLDER} IRODS_USER: ${PLACEHOLDER} IRODS_PASSWORD: ${PLACEHOLDER} + IRODS_ANONYMOUS: ${IRODS_ANONYMOUS} networks: i_net: aliases: diff --git a/confs/coverage.yml b/confs/coverage.yml index d8aa6a57..926f1bc8 100644 --- a/confs/coverage.yml +++ b/confs/coverage.yml @@ -9,7 +9,7 @@ services: coverage: command: coveralls hostname: coveraging - build: ../submodules/builds_base/backend + build: ../submodules/build-templates/backend image: ${COMPOSE_PROJECT_NAME}/backend:template working_dir: /repo environment: diff --git a/confs/nginx/production.conf b/confs/nginx/production.conf index 0c59c2c7..566920ce 100644 --- a/confs/nginx/production.conf +++ b/confs/nginx/production.conf @@ -59,7 +59,17 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-IP $server_addr; - # It should be already added by FLASK + # client_body_buffer_size 8k; + proxy_buffering off; + proxy_request_buffering off; + + # 90000 seconds = 25 hours + proxy_connect_timeout 90000; + proxy_send_timeout 90000; + proxy_read_timeout 90000; + send_timeout 90000; + + # This should be already added by FLASK: # add_header Access-Control-Allow-Origin "*"; } diff --git a/data/b2handle/credentials.json b/data/b2handle/credentials.json index cbb074c3..fd74ba2c 100644 --- a/data/b2handle/credentials.json +++ b/data/b2handle/credentials.json @@ -2,9 +2,9 @@ "handle_server_url": "https://epic3.storage.surfsara.nl:8001", "private_key": "/opt/certificates/b2handle/private.pem", "certificate_only": "/opt/certificates/b2handle/public.pem", - "prefix": "-", - "handleowner": "-", - "reverselookup_username": "-", - "reverselookup_password": "-", + "prefix": "000", + "handleowner": "001:0.NA/000", + "reverselookup_username": "000", + "reverselookup_password": "some-password", "HTTPS_verify": "/opt/certificates/b2handle/ca_authority.pem" } \ No newline at end of file diff --git a/data/scripts/templates/client.py b/data/scripts/templates/client.py index 0b687180..bdc04b79 100755 --- a/data/scripts/templates/client.py +++ b/data/scripts/templates/client.py @@ -29,6 +29,7 @@ LOCAL_HTTPAPI_URI = 'http://localhost:8080' log = apiclient.setup_logger(__name__, level_name=LOG_LEVEL) +remote_calls = apiclient.check_cli_arg('remote') ############# # MAIN # @@ -38,7 +39,10 @@ if __name__ == '__main__': # decide which HTTP API server you should query - if apiclient.check_cli_arg('remote'): + if remote_calls: + from utilities.checks import internet_connection_available + if not internet_connection_available(): + log.exit("No internet connection") uri = REMOTE_HTTPAPI_URI else: uri = LOCAL_HTTPAPI_URI @@ -77,7 +81,7 @@ # other operations # avoid more operations if the user only requested listing - apiclient.check_cli_arg('list', exit=True) + apiclient.check_cli_arg('list', exit=True) # , reverse=True) # push files found in config dir files = apiclient.folder_content(FILES_PATH) @@ -111,8 +115,15 @@ # ACTION: upload one file ################ if not os.path.basename(file_path) in new_dir_content: + + # NOTE: you can wait for the PID registration + # (assuming the connected B2SAFE is configured so) + # by adding payload={'pid': True} to the call + # (probably also a longer timetout, e.g. 20 seconds) + response = apiclient.call( - uri, endpoint=apiclient.BASIC_ENDPOINT + new_dir_path, + uri, + endpoint=apiclient.BASIC_ENDPOINT + new_dir_path, token=token, method='put', file=file_path ) log.info("Uploaded file: %s", file_path) @@ -134,7 +145,9 @@ response = apiclient.call( uri, endpoint=os.path.join(new_dir_endpoint, some_file), token=token) - pid = response[0][some_file]['metadata']['PID'] + data = response.pop() + metadata = data.get(some_file, {}).get('metadata', {}) + pid = metadata.get('PID') if pid: response = apiclient.call( uri, token=token, diff --git a/data/scripts/templates/client_banchmark.py b/data/scripts/templates/client_banchmark.py new file mode 100755 index 00000000..00251dce --- /dev/null +++ b/data/scripts/templates/client_banchmark.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Example of a simple Python code uploading in parallel all files in ./data/files folder +Warning: Uploaded files are not automatically removed + + +requirements: +- requests +- rapydo/utils +""" + +import os +import threading +import better_exceptions as be +from utilities import helpers +from utilities import apiclient + + +########################### +# Configuration variables # +########################### + +USERNAME = 'someuser' +PASSWORD = 'somepassword' +FILES_PATH = './data/files' +LOG_LEVEL = 'info' # or 'debug', 'verbose', 'very_verbose' + +REMOTE_DOMAIN = 'b2stage.cineca.it' # you may change to another server +REMOTE_HTTPAPI_URI = 'https://%s' % REMOTE_DOMAIN +LOCAL_HTTPAPI_URI = 'http://localhost:8080' + +log = apiclient.setup_logger(__name__, level_name=LOG_LEVEL) + + + +def upload(file): + if not os.path.basename(file_path) in new_dir_content: + log.info("Uploading file: %s", file_path) + response = apiclient.call( + uri, endpoint=apiclient.BASIC_ENDPOINT + new_dir_path, + token=token, method='put', file=file, timeout=500 + ) + log.info("Uploaded file: %s", file_path) + else: + log.warning("%s already exists in %s", file_path, new_dir_path) + + + +if __name__ == '__main__': + + # decide which HTTP API server you should query + if apiclient.check_cli_arg('remote'): + uri = REMOTE_HTTPAPI_URI + else: + uri = LOCAL_HTTPAPI_URI + log.very_verbose('init log: %s\nURI [%s]', be, uri) + + ################ + # ACTION: check status + ################ + + # check if HTTP API are alive and/or our connection is working + apiclient.call(uri) + + ################ + # ACTION: LOGIN + ################ + # login to HTTP API with B2SAFE credentials + token, home_path = apiclient.login(uri, USERNAME, PASSWORD) + # # or set your token from B2ACCESS web login + # token = 'SOMEHASHSTRINGFROMB2ACCESS' + # path = '/tempZone/home/YOURUSER' + log.debug('Home directory is: %s', home_path) + log.info("Logged in with token: %s...", token[:20]) + + + # avoid more operations if the user only requested listing + apiclient.check_cli_arg('list', exit=True) + + # push files found in config dir + files = apiclient.folder_content(FILES_PATH) + log.debug("Files to be pushed: %s", files) + + new_dir = 'testing_httpapi' + # new_dir = helpers.random_name() + new_dir_path = os.path.join(home_path, new_dir) + + # list current files + response = apiclient.call( + uri, endpoint=apiclient.BASIC_ENDPOINT + home_path, token=token) + home_content = apiclient.parse_irods_listing(response, home_path) + if len(home_content) < 1 and apiclient.check_cli_arg('list'): + log.warning("Home directory is empty") + + if new_dir not in home_content: + + ################ + # ACTION: create directory + ################ + response = apiclient.call( + uri, endpoint=apiclient.BASIC_ENDPOINT, token=token, method='post', + payload={'path': new_dir_path} + ) + log.info("Created directory: %s", response.get('path')) + else: + log.warning("Directory already exists: %s", new_dir) + + # list new dir + new_dir_endpoint = apiclient.BASIC_ENDPOINT + new_dir_path + response = apiclient.call(uri, endpoint=new_dir_endpoint, token=token) + new_dir_content = apiclient.parse_irods_listing(response, new_dir_path) + + # upload files in paralle + threads = [] + for file_path in files: + t = threading.Thread(target=upload, args=(file_path,)) + threads.append(t) + t.start() diff --git a/docs/deploy/debugging.md b/docs/deploy/debugging.md new file mode 100644 index 00000000..07ec5099 --- /dev/null +++ b/docs/deploy/debugging.md @@ -0,0 +1,55 @@ + +# Other operations + +Here we have more informations for further debugging/developing the project + + +## launch interfaces + +To explore data and query parameters there are few other services as options: + +```bash +SERVER=localhost # or your IP / Domain +PORT=8080 + +# access a swagger web ui +rapydo interfaces swagger +# access the webpage: +open http://$SERVER/swagger-ui/?url=http://$SERVER:$PORT/api/specs + +# SQL admin web ui +rapydo interfaces sqlalchemy +# access the webpage: +open http://$SERVER:81/adminer +``` + +## add valid b2handle credentials + +To resolve non-public `PID`s prefixes for the `EPIC HANDLE` project we may leverage the `B2HANDLE` library providing existing credentials. + +You can do so by copying such files into the dedicated directory: + +```bash +cp PATH/TO/YOUR/CREDENTIALS/FILES/* data/b2handle/ +``` + +## destroy all + +If you need to clean everything you have stored in docker from this project: + +```bash +# BE CAREFUL! +rapydo clean --rm # very DANGEROUS +# With this command you lose all your current data! +``` + +## hack the certificates volume + +This hack is necessary if you want to raw copy a Certification Authority credential. +It may be needed if your current B2SAFE host DN was produced with an internal certification authority which is not recognized from other clients. + +```bash +path=`docker inspect $(docker volume ls -q | grep sharedcerts) | jq -r ".[0].Mountpoint"` +sudo cp /PATH/TO/YOUR/CA/FILES/CA-CODE.* $path/simple_ca + +``` \ No newline at end of file diff --git a/docs/deploy/deploy.md b/docs/deploy/deploy.md index ca854b9c..9b5f0b4e 100644 --- a/docs/deploy/deploy.md +++ b/docs/deploy/deploy.md @@ -1,3 +1,14 @@ -# Admin documentation ->To be written +# Deploy your own server + + + +See [the video instructions](https://www.youtube.com/watch?time_continue=734&v=KkH4oM1pdbE) in our webinar to download the repo. diff --git a/docs/deploy/modes.md b/docs/deploy/modes.md new file mode 100644 index 00000000..e75ad4ee --- /dev/null +++ b/docs/deploy/modes.md @@ -0,0 +1,110 @@ + +# MODES + +There are two main modes to work with the HTTP-API server developed with `rapydo`. + +The main one - called `debug` - is for developers: you are expected to test, debug and develop new code. The other options is mode `production`, suited for deploying your server in a real use case scenario on top of your already running `B2SAFE` instance. + +## debug mode + +NOTE: follow this paragraph only if you plan to develop new features on the HTTP API. + +```bash +################ +# bring up the docker containers in `debug` mode +$ rapydo start +# NOTE: this above is equivalent to default value `do --mode debug start` + +# laungh the restful http-api server +$ rapydo shell backend --command 'restapi launch' +# or +$ rapydo shell backend +[container shell]$ restapi launch +``` + +NOTE: the block of commands above can be used at once with: + +```bash +$ rapydo start --mode development +# drawback: when the server fails at reloading, it crashes +``` + + +And now you may access a client for the API, from another shell and test the server: + +```bash +$ rapydo shell restclient +``` + +The client shell will give you further instructions on how to test the server. In case you want to log with the only existing admin user: + +- username: user@nomail.org +- password: test + +NOTE: on the client image you have multiple tools installed for testing: +- curl +- httpie +- http-prompt + + +## production mode + +Some important points before going further: + +1. Please follow this paragraph only if you plan to deploy the HTTP API server in production +2. Usually in production you have a domain name associated to your host IP (e.g. `b2stage.cineca.it` to 240.bla.bla.bla). But you can just use 'localhost' if this is not the case. +3. You need a `B2ACCESS` account on the development server for the HTTP API application. Set the credentials [here](https://github.com/EUDAT-B2STAGE/http-api/blob/0.6.1/projects/b2stage/project_configuration.yaml#L22-L26) otherwise the endpoint `/auth/askauth` would not work. + +Deploying is very simple: + +```bash +# define your domain +$ DOMAIN='b2stage.cineca.it' +# launch production mode +$ rapydo --hostname $DOMAIN --mode production start +``` + +Now may access your IP or your domain and the HTTP API endpoints are online, protected by a proxy server. You can test this with: + +```bash +open $DOMAIN/api/status +``` + +Up to now the current SSL certificate is self signed and is 'not secure' for all applications. Your browser is probably complaining for this. This is why we need to produce one with the free `letsencrypt` service. + +```bash +$ rapydo --hostname $DOMAIN --mode production ssl-certificate +#NOTE: this will work only if you have a real domain associated to your IP +``` + +If you check again the server should now be correctly certificated. At this point the service should be completely functional. + +In production it might be difficult to get informations if something goes wrong. If the server failed you have a couple of options to find out why: + +```bash +# check any service on any mode +$ rapydo --mode YOURMODE --service YOURSERVICE log +# e.g. check all the logs from production, following new updates +# $ rapydo --mode production log --follow + +## if this is not enough: + +# check the backend as admin of the container +$ rapydo shell --user root backend +# look at the production WSGI logs +less /var/log/uwsgi/*log +# check processes +ps aux --forest +# here uwsgi (as developer) and nginx (as www-data) should be running + +## if you only get 'no app loaded' from uWSGI, + +$ rapydo shell backend +# launch by hand a server instance +$ DEBUG_LEVEL=VERY_VERBOSE restapi launch +# check if you get any error in your output + +``` + +Also please take a look at how to launch interfaces in the upcoming paragraph, in case there is a problem with the database or swagger. + diff --git a/docs/deploy/preq.md b/docs/deploy/preq.md new file mode 100644 index 00000000..01f43560 --- /dev/null +++ b/docs/deploy/preq.md @@ -0,0 +1,50 @@ + +# Deployment pre-requisites + +In order to deploy this project you need to install the `RAPyDO controller` and use it to startup the services on a `Docker` environment with containers. + +The following instruction are based on the hyphotesis that you will work on a `UNIX`-based OS. Any `Linux` distributions (Ubuntu, CentOS, Fedora, etc.) or any version of `Mac OSX` will do. Command line examples were heavily tested on `bash` terminal (version `4.4.0`, but also version `3.x` should work). + +Please note that for installing tools into your machine the suggested option is through your preferred OS package manager (e.g. `apt`, `yum`, `brew`, etc.). + + +## Base tools + +- The `git` client. + +Most of UNIX distributions have it already installed. If that is not that case then refer to the [official documentation]() + +- The `python 3.4+` interpreter installed together with its main package manager `pip3`. + +Most of distributions comes bundled with `python 2.7+`, which is not suitable for our project. Once again use a package manager, for example in ubuntu you would run: + +```bash +$ apt-get update && apt-get install python3-pip +``` + + +## Containers environment + +### docker engine + +To install docker on a unix terminal you may use the [get docker script](https://get.docker.com): + +``` +# Install docker +$ curl -fsSL get.docker.com -o get-docker.sh +$ sh get-docker.sh +``` + +For Mac and Windows users dedicated applications were written: + +- [Docker for Mac](https://www.docker.com/docker-mac) +- [Docker for Windows](https://www.docker.com/docker-windows) + +As alternative, the best way to get Docker ALL tools working +is by using their [toolbox](https://www.docker.com/toolbox). + +### docker compose + +`Compose` is a tool for docker written in Python. See the [official instructions](https://docs.docker.com/compose/install/) to install it. + +NOTE: compose comes bundled with the toolbox. diff --git a/docs/deploy/startup.md b/docs/deploy/startup.md new file mode 100644 index 00000000..e737c549 --- /dev/null +++ b/docs/deploy/startup.md @@ -0,0 +1,70 @@ + +# Start-up the project + + +## 1. cloning + +To clone the working code: + +```bash +$ VERSION=0.6.1 \ + && git clone https://github.com/EUDAT-B2STAGE/http-api.git \ + && cd http-api \ + && git checkout $VERSION + +# now you will have the current latest release (RC1) +``` + + +## 2. configure + +Now that you have all necessary software installed, before launching services you should consider editing the main configuration: + +[`projects/b2stage/project_configuration.yaml`](projects/b2stage/project_configuration.yaml) + +Here you can change at least the basic passwords, or configure access to external service (e.g. your own instance of iRODS/B2SAFE) for production. + + +## 3. controller + +The controller is what let you manage the project without much effort. +Here's what you need to use it: + +```bash +# install and use the rapydo controller +$ data/scripts/prerequisites.sh +# you have now the executable 'rapydo' +$ rapydo --version +# If you use a shell different from bash (e.g. zsh) +# you can try also the short alias 'do' +$ do --help +``` + +NOTE: python install binaries in `/usr/local/bin`. If you are not the admin/`root` user then the virtual environment is created and you may find the binary in `$HOME/.local/bin`. Make sure that the right one of these paths is in your `$PATH` variable, otherwise you end up with `command not found`. + + +## 4. deploy initialization + +Your current project needs to be initialized. This step is needed only the first time you use the cloned repository. + +```bash +$ rapydo init +``` + +NOTE: with `RC1` there is no working `upgrade` process in place to make life easier if you already have this project cloned from a previous release. This is something important already in progress [here](https://github.com/EUDAT-B2STAGE/http-api/issues/87). + +If you wish to __**manually upgrade**__: + +```bash +VERSION="0.6.1" +git checkout $VERSION + +# supposely the rapydo framework has been updated, so you need to check: +rm -rf submodules/* +data/scripts/prerequisites.sh +rapydo init + +# update docker images with the new build templates in rapydo +# NOTE: select your mode based on the next paragraph +rapydo --mode YOURMODE build -r -f +``` diff --git a/docs/prototype.md b/docs/prototype.md index 3aee6a28..14f9238c 100644 --- a/docs/prototype.md +++ b/docs/prototype.md @@ -39,7 +39,7 @@ https://b2stage.cineca.it/api/specs ### authentication process -Details on how to create a valid token upon the current release of the HTTP API is available [inside the user guide](user/authentication.md) +Details on how to create a valid token upon the current release of the HTTP API is available [inside the user guide](user/authentication.md). ### curl @@ -49,10 +49,26 @@ See also the [main user page](user/user.md) to understand which endpoints exists Since the HTTP API server follows the `openapi` standard, you can query its endpoints also using the official `swagger-ui` web server, by just passing the `JSON` file in input: -http://petstore.swagger.io/?url=https://b2stage.cineca.it/api/specs&docExpansion=none +[link to the swagger `petstore`](http://petstore.swagger.io/?url=https://b2stage.cineca.it/api/specs&docExpansion=none) -### python script +### python client -This is yet a work in progress: +You can find a [dedicated python module file](data/scripts/templates/client.py) to query the EUDAT B2STAGE HTTP-API. -> creating a dedicated python package to officially query the EUDAT B2STAGE HTTP-API +The script is already configured to work with a local deploy of the HTTP-API containers on your computer or to a remote host. Before using it open the file and [change](https://github.com/EUDAT-B2STAGE/http-api/blob/master/data/scripts/templates/client.py#L22-L23) the basic [parameters](https://github.com/EUDAT-B2STAGE/http-api/blob/master/data/scripts/templates/client.py#L27). + +#### local + +If you are [running a working copy]() of the `B2STAGE HTTP-API` you can test the client by simply calling from a `UNIX` terminal: + +```bash +data/scripts/templates/client.py +``` + +#### remote + +If you have credentials to a remote instance of the `HTTP-API` you just need to provide them inside the script and then execute with: + +```bash +data/scripts/templates/client.py --remote +``` \ No newline at end of file diff --git a/docs/quick_start.md b/docs/quick_start.md index 2497377e..143bb6af 100644 --- a/docs/quick_start.md +++ b/docs/quick_start.md @@ -1,305 +1,20 @@ # Quick start -This is a reference page to quick start your knowledge of the HTTP API project; by reading this page you may get a first insight on how this project works. - - -## Feedback on the first Release Candidate - -To gather into one place the feedback of any user testing a deployed HTTP API server or the online prototype, we created a dedicated free chat room on the `gitter` comunication platform: - -https://gitter.im/EUDAT-B2STAGE/http-api - -Please: feel free to report or comment to help us improve! +This is a reference page to quick start your knowledge of the project. ## Using the prototype online If you don't want to deploy or develop the current project state, you may test online our [prototype](https://b2stage.cineca.it/api/status). If that is the case see the [dedicated instructions](prototype.md). - -## Deployment pre-requisites - -In order to deploy this project you need to install the `RAPyDO controller` and use it to startup the services on a `Docker` environment with containers. - -The following instruction are based on the hyphotesis that you will work on a `UNIX`-based OS. Any `Linux` distributions (Ubuntu, CentOS, Fedora, etc.) or any version of `Mac OSX` will do. Command line examples were heavily tested on `bash` terminal (version `4.4.0`, but also version `3.x` should work). - -Please note that for installing tools into your machine the suggested option is through your preferred OS package manager (e.g. `apt`, `yum`, `brew`, etc.). - - -### Base tools - -- The `git` client. - -Most of UNIX distributions have it already installed. If that is not that case then refer to the [official documentation]() - -- The `python 3.4+` interpreter installed together with its main package manager `pip3`. - -Most of distributions comes bundled with `python 2.7+`, which is not suitable for our project. Once again use a package manager, for example in ubuntu you would run: - -```bash -$ apt-get update && apt-get install python3-pip -``` - - -### Containers environment - -#### docker engine - -To install docker on a unix terminal you may use the [get docker script](https://get.docker.com): - -``` -# Install docker -$ curl -fsSL get.docker.com -o get-docker.sh -$ sh get-docker.sh -``` - -For Mac and Windows users dedicated applications were written: - -- [Docker for Mac](https://www.docker.com/docker-mac) -- [Docker for Windows](https://www.docker.com/docker-windows) - -As alternative, the best way to get Docker ALL tools working -is by using their [toolbox](https://www.docker.com/toolbox). - -#### docker compose - -`Compose` is a tool for docker written in Python. See the [official instructions](https://docs.docker.com/compose/install/) to install it. - -NOTE: compose comes bundled with the toolbox. +Please feel free [to report or comment](https://gitter.im/EUDAT-B2STAGE/http-api) to help us improve! ## Start-up the project -Here's a step-by-step tutorial to work with the HTTP API project. - - -### 1. cloning - -To clone the working code: - -```bash -$ VERSION=0.6.1 \ - && git clone https://github.com/EUDAT-B2STAGE/http-api.git \ - && cd http-api \ - && git checkout $VERSION - -# now you will have the current latest release (RC1) -``` - - -### 2. configure - -Now that you have all necessary software installed, before launching services you should consider editing the main configuration: - -[`projects/eudat/project_configuration.yaml`](projects/eudat/project_configuration.yaml) - -Here you can change at least the basic passwords, or configure access to external service (e.g. your own instance of iRODS/B2SAFE) for production. - - -### 3. controller - -The controller is what let you manage the project without much effort. -Here's what you need to use it: - -```bash -# install and use the rapydo controller -$ data/scripts/prerequisites.sh -# you have now the executable 'rapydo' -$ rapydo --version -# If you use a shell different from bash (e.g. zsh) -# you can try also the short alias 'do' -$ do --help -``` - -NOTE: python install binaries in `/usr/local/bin`. If you are not the admin/`root` user then the virtual environment is created and you may find the binary in `$HOME/.local/bin`. Make sure that the right one of these paths is in your `$PATH` variable, otherwise you end up with `command not found`. - - -### 4. deploy initialization - -Your current project needs to be initialized. This step is needed only the first time you use the cloned repository. - -```bash -$ rapydo init -``` - -NOTE: with `RC1` there is no working `upgrade` process in place to make life easier if you already have this project cloned from a previous release. This is something important already in progress [here](https://github.com/EUDAT-B2STAGE/http-api/issues/87). - -If you wish to __**manually upgrade**__: - -```bash -VERSION="0.6.1" -git checkout $VERSION - -# supposely the rapydo framework has been updated, so you need to check: -rm -rf submodules/* -data/scripts/prerequisites.sh -rapydo init - -# update docker images with the new build templates in rapydo -# NOTE: select your mode based on the next paragraph -rapydo --mode YOURMODE build -r -f -``` - -### 5. MODES - -There are two main modes to work with the API server. The main one - called `debug` - is for developers: you are expected to test, debug and develop new code. The other options is mode `production`, suited for deploying your server in a real use case scenario on top of your already running `B2SAFE` instance. - -#### debug mode - -NOTE: follow this paragraph only if you plan to develop new features on the HTTP API. - -```bash -################ -# bring up the docker containers in `debug` mode -$ rapydo start -# NOTE: this above is equivalent to default value `do --mode debug start` - -# laungh the restful http-api server -$ rapydo shell backend --command 'restapi launch' -# or -$ rapydo shell backend -[container shell]$ restapi launch -``` - -NOTE: the block of commands above can be used at once with: - -```bash -$ rapydo start --mode development -# drawback: when the server fails at reloading, it crashes -``` - - -And now you may access a client for the API, from another shell and test the server: - -```bash -$ rapydo shell restclient -``` - -The client shell will give you further instructions on how to test the server. In case you want to log with the only existing admin user: - -- username: user@nomail.org -- password: test - -NOTE: on the client image you have multiple tools installed for testing: -- curl -- httpie -- http-prompt - - -#### production mode - -Some important points before going further: - -1. Please follow this paragraph only if you plan to deploy the HTTP API server in production -2. Usually in production you have a domain name associated to your host IP (e.g. `b2stage.cineca.it` to 240.bla.bla.bla). But you can just use 'localhost' if this is not the case. -3. You need a `B2ACCESS` account on the development server for the HTTP API application. Set the credentials [here](https://github.com/EUDAT-B2STAGE/http-api/blob/0.6.1/projects/eudat/project_configuration.yaml#L22-L26) otherwise the endpoint `/auth/askauth` would not work. - -Deploying is very simple: - -```bash -# define your domain -$ DOMAIN='b2stage.cineca.it' -# launch production mode -$ rapydo --hostname $DOMAIN --mode production start -``` - -Now may access your IP or your domain and the HTTP API endpoints are online, protected by a proxy server. You can test this with: - -```bash -open $DOMAIN/api/status -``` - -Up to now the current SSL certificate is self signed and is 'not secure' for all applications. Your browser is probably complaining for this. This is why we need to produce one with the free `letsencrypt` service. - -```bash -$ rapydo --hostname $DOMAIN --mode production ssl-certificate -#NOTE: this will work only if you have a real domain associated to your IP -``` - -If you check again the server should now be correctly certificated. At this point the service should be completely functional. - -In production it might be difficult to get informations if something goes wrong. If the server failed you have a couple of options to find out why: - -```bash -# check any service on any mode -$ rapydo --mode YOURMODE --service YOURSERVICE log -# e.g. check all the logs from production, following new updates -# $ rapydo --mode production log --follow - -## if this is not enough: - -# check the backend as admin of the container -$ rapydo shell --user root backend -# look at the production WSGI logs -less /var/log/uwsgi/*log -# check processes -ps aux --forest -# here uwsgi (as developer) and nginx (as www-data) should be running - -## if you only get 'no app loaded' from uWSGI, - -$ rapydo shell backend -# launch by hand a server instance -$ DEBUG_LEVEL=VERY_VERBOSE restapi launch -# check if you get any error in your output - -``` - -Also please take a look at how to launch interfaces in the upcoming paragraph, in case there is a problem with the database or swagger. - - -## Other operations - -Here we have more informations for further debugging/developing the project - - -### launch interfaces - -To explore data and query parameters there are few other services as options: - -```bash -SERVER=localhost # or your IP / Domain -PORT=8080 - -# access a swagger web ui -rapydo interfaces swagger -# access the webpage: -open http://$SERVER/swagger-ui/?url=http://$SERVER:$PORT/api/specs - -# SQL admin web ui -rapydo interfaces sqlalchemy -# access the webpage: -open http://$SERVER:81/adminer -``` - -### add valid b2handle credentials - -To resolve non-public `PID`s prefixes for the `EPIC HANDLE` project we may leverage the `B2HANDLE` library providing existing credentials. - -You can do so by copying such files into the dedicated directory: - -```bash -cp PATH/TO/YOUR/CREDENTIALS/FILES/* data/b2handle/ -``` - -### destroy all - -If you need to clean everything you have stored in docker from this project: - -```bash -# BE CAREFUL! -rapydo clean --rm # very DANGEROUS -# With this command you lose all your current data! -``` - -### hack the certificates volume - -This hack is necessary if you want to raw copy a Certification Authority credential. -It may be needed if your current B2SAFE host DN was produced with an internal certification authority which is not recognized from other clients. +Before starting up head to the [pre-requisites](docs/deploy/preq.md) page. +Here's step-by-step tutorial to work with the HTTP API project. -```bash -path=`docker inspect $(docker volume ls -q | grep sharedcerts) | jq -r ".[0].Mountpoint"` -sudo cp /PATH/TO/YOUR/CA/FILES/CA-CODE.* $path/simple_ca +`TO BE COMPLETED` -``` diff --git a/docs/user/endpoints.md b/docs/user/endpoints.md index 8713e622..f44cbe6f 100644 --- a/docs/user/endpoints.md +++ b/docs/user/endpoints.md @@ -1,8 +1,10 @@ # Available resources -* [registered](registered.md) -* [pids](pids.md) -* workspace -* digital entities -* ... +* [registered](registered.md) domain +* [pids](pids.md) resolution +* [publish](publish.md) your data publicly +* an `HTML` [landing page](landing.md) `[WORK IN PROGRESS]` +* [workspace](workspace.md) domain `[TO DO]` + + diff --git a/docs/user/landing.md b/docs/user/landing.md new file mode 100644 index 00000000..2278377e --- /dev/null +++ b/docs/user/landing.md @@ -0,0 +1,7 @@ + +# Landing page + +Published data can be displayed in a browser as an `HTML` web page. + +`[TO BE COMPLETED]` + diff --git a/docs/user/pids.md b/docs/user/pids.md index cde7f7cb..0b403131 100644 --- a/docs/user/pids.md +++ b/docs/user/pids.md @@ -14,15 +14,20 @@ The examples in this section use cURL commands. For information about cURL, see --- ## **GET** + ### Resolve PID + ##### Example + ```bash # Resolve (e.g. 11100/33ac01fc-6850-11e5-b66e-e41f13eb32b2) $ curl \ -H "Authorization: Bearer " \ /api/pids/ ``` + ##### Response + ```json { "Meta": { @@ -43,14 +48,18 @@ $ curl \ ### Resolve PID and download object + ##### Example + ```bash # Get 'filename.txt' metadata $ curl \ -H "Authorization: Bearer " \ /api/pids/?download=true ``` + ##### Response + ```json Content of the object to which the PID points to (if reachable by the HTTP-API server) ``` \ No newline at end of file diff --git a/docs/user/publish.md b/docs/user/publish.md new file mode 100644 index 00000000..eebccab2 --- /dev/null +++ b/docs/user/publish.md @@ -0,0 +1,6 @@ + +# Publish data + +Data available on `B2SAFE` can be shared to non-authenticated users. + +`[TO BE COMPLETED]` diff --git a/docs/user/registered.md b/docs/user/registered.md index cb027759..ddf19904 100644 --- a/docs/user/registered.md +++ b/docs/user/registered.md @@ -117,7 +117,7 @@ $ curl \ "location": "irods://rodserver.dockerized.io/path/to/directory", "metadata": { "PID": null, - "checksum": "9876543210", , + "checksum": "9876543210", "name": "test", "object_type": "dataobject" }, @@ -135,7 +135,7 @@ $ curl \ ## **PUT** ### Upload an entity **and trigger the registration in B2SAFE** -Both Form and Streaming upload are supported. Streaming is faster and does load the file in memory on clinet side. +Both Form and Streaming upload are supported. Streaming is more advisable uploding large data. > Notes: The entity registration depends on the policies adopted by the B2SAFE instance which the B2STAGE HTTP-API is connected to. This operation is idempotent. @@ -144,7 +144,7 @@ Both Form and Streaming upload are supported. Streaming is faster and does load |-----------|------|------------- | file (required) | string | Name of the local file to be uploaded | force | bool | Force overwrite -| pid | bool | Return PID (synchronous) +| pid_await | bool | Return PID in the response: the response is returned as the registration is completed (or after 10 seconds if the PID is not ready yet) ##### Example: Form upload ```bash @@ -159,6 +159,12 @@ $ curl -X PUT \ -H "Authorization: Bearer " \ -F file=@myfile2 \ /api/registered/path/to/directory/filename?force=true + +# Form upload 'myfile' and get the PID in the response +$ curl -X PUT \ + -H "Authorization: Bearer " \ + -F file=@myfile2 \ + /api/registered/path/to/directory/filename?pid_await=true ``` ##### Example: Streaming upload ```bash diff --git a/docs/user/user.md b/docs/user/user.md index fc9a1f62..b8ae17b8 100644 --- a/docs/user/user.md +++ b/docs/user/user.md @@ -12,6 +12,7 @@ The current release of the B2STAGE HTTP-API allows users to perform the followin The complete description of the operation you can perform is available in: - the [registered api](registered.md) page - the [pids api](pids.md) page +- the [publish api](publish.md) page ## Authentication flow diff --git a/docs/user/workspace.md b/docs/user/workspace.md new file mode 100644 index 00000000..9272625f --- /dev/null +++ b/docs/user/workspace.md @@ -0,0 +1,4 @@ + +# Workspace domain + +`[TO DO]` diff --git a/projects/eudat/builds/restclient/.gitkeep b/projects/b2host/.gitkeep similarity index 100% rename from projects/eudat/builds/restclient/.gitkeep rename to projects/b2host/.gitkeep diff --git a/projects/eudat/frontend/tests/.gitkeep b/projects/b2proxy/.gitkeep similarity index 100% rename from projects/eudat/frontend/tests/.gitkeep rename to projects/b2proxy/.gitkeep diff --git a/projects/b2safe/README.md b/projects/b2safe/README.md new file mode 100644 index 00000000..8c8f0338 --- /dev/null +++ b/projects/b2safe/README.md @@ -0,0 +1,79 @@ + +# B2SAFE dockerized + +This is an effort to maintain an updated iRODS image working, +with B2SAFE extensions installed at the latest version. + +## Prototype + +Currently the prototype is based on the HTTP-API development branch. + +The first draft is composed of three containers: + +1. a simple iRODS `iCAT` server `v4.2.1` with Globus/GSI installed +2. Postgresql for the iCAT database +3. A client to check connecting to iRODS from another container at the very least + +## Quick start + +```bash + +# NOTE: do not pull from existing repository +# remove any existing repo instead: +rm -rf myb2safe + +################## +# Clone the latest branch +git clone https://github.com/EUDAT-B2STAGE/http-api.git myb2safe +cd myb2safe && git checkout 0.6.2 + +################## +# Install the rapydo controller +sudo -H data/scripts/prerequisites.sh +# Initialise the working copy +rapydo init + +################## +# Launch containers + +# Build the normal iRODS iCat image +rapydo --project b2safe --mode simple_irods build + +# launch the stack +rapydo --project b2safe start +# check +rapydo --project b2safe status +# verify the irods port working +telnet localhost 1247 +## NOTE: you should be able to connect from outside too +## (only if you have no firewall blocking incoming connections on that port) + +################## + +################## +# Use it! + +# Check the current server +rapydo --project b2safe shell icat +## NOTE: this opens a shell inside the main server container +ls /opt/eudat/b2safe +# Become the irods administrator +berods +# Use the icommands +ils +iadmin lu +exit + +# Check from another client +rapydo --project b2safe shell iclient +## NOTE: this opens a shell inside a client container +# Become the irods administrator +berods +# Use the same icommands +ils +exit + +################## +# Destroy containers and data +rapydo --project b2safe clean --rm +``` \ No newline at end of file diff --git a/projects/eudat/backend/__init__.py b/projects/b2safe/backend/.gitkeep similarity index 100% rename from projects/eudat/backend/__init__.py rename to projects/b2safe/backend/.gitkeep diff --git a/projects/eudat/backend/apis/__init__.py b/projects/b2safe/backend/__main__.py similarity index 100% rename from projects/eudat/backend/apis/__init__.py rename to projects/b2safe/backend/__main__.py diff --git a/projects/eudat/backend/models/__init__.py b/projects/b2safe/backend/apis/.gitkeep similarity index 100% rename from projects/eudat/backend/models/__init__.py rename to projects/b2safe/backend/apis/.gitkeep diff --git a/projects/eudat/backend/project/__init__.py b/projects/b2safe/backend/models/.gitkeep similarity index 100% rename from projects/eudat/backend/project/__init__.py rename to projects/b2safe/backend/models/.gitkeep diff --git a/projects/eudat/backend/swagger/template/SKIP b/projects/b2safe/backend/swagger/.gitkeep similarity index 100% rename from projects/eudat/backend/swagger/template/SKIP rename to projects/b2safe/backend/swagger/.gitkeep diff --git a/projects/eudat/backend/tests/__init__.py b/projects/b2safe/backend/tests/.gitkeep similarity index 100% rename from projects/eudat/backend/tests/__init__.py rename to projects/b2safe/backend/tests/.gitkeep diff --git a/projects/b2safe/confs/b2safe.yml b/projects/b2safe/confs/b2safe.yml new file mode 100644 index 00000000..641e9e02 --- /dev/null +++ b/projects/b2safe/confs/b2safe.yml @@ -0,0 +1,19 @@ + +version: '3' +services: + + icat: + build: ../submodules/build-templates/b2safe + image: ${COMPOSE_PROJECT_NAME}/server:b2safe-mod + environment: + B2ACCESS_CAS: ${B2ACCESS_CAS} + + volumes: + - ../submodules/build-templates/b2safe/extra_b2access.sh:/docker-entrypoint.d/b2access.sh + - ../submodules/build-templates/b2safe/extra_b2safe.sh:/docker-entrypoint.d/b2safe.sh + # B2ACCESS dev and prod certificates + - ../submodules/build-templates/b2safe/b2access_certificates:${B2ACCESS_CAS} + + # Open irods port to Outside world + ports: + - 1247:1247 diff --git a/projects/b2safe/confs/commons.yml b/projects/b2safe/confs/commons.yml new file mode 100644 index 00000000..388d1acc --- /dev/null +++ b/projects/b2safe/confs/commons.yml @@ -0,0 +1,60 @@ +version: '3' + +volumes: + etcconf: + driver: local + irodscerts: + driver: local + irodshome: + driver: local + irodsvar: + driver: local + +services: + postgres: + environment: + POSTGRES_USER: "${ALCHEMY_USER}" + POSTGRES_PASSWORD: "${ALCHEMY_PASSWORD}" + POSTGRES_DBS: ${ALCHEMY_DBS} + icat: + environment: + ACTIVATE: 1 + POSTGRES_HOST: "${ALCHEMY_HOST}" + POSTGRES_USER: "${ALCHEMY_USER}" + POSTGRES_PASSWORD: "${ALCHEMY_PASSWORD}" + IRODS_HOST: "${IRODS_HOST}" + IRODS_PORT: ${IRODS_PORT} + IRODS_ZONE: ${IRODS_ZONE} + IRODS_DB: "${IRODS_DB}" + IRODS_USER: ${IRODS_USER} + IRODS_PASSWORD: ${IRODS_PASSWORD} + volumes: + - etcconf:/etc + - irodshome:/home/${IRODS_USER} + - irodsvar:/var/lib/${IRODS_USER} + - irodscerts:/opt/certificates + # # Open irods port to Outside world + # ports: + # - 1247:1247 + sqlalchemyui: + ports: + - 80:8888 + iclient: + build: ../submodules/build-templates/icat + image: ${COMPOSE_PROJECT_NAME}/icat:template + hostname: iclient + command: sleep infinity + environment: + ACTIVATE: 1 + IRODS_HOST: "${IRODS_HOST}" + IRODS_PORT: ${IRODS_PORT} + IRODS_ZONE: ${IRODS_ZONE} + IRODS_USER: ${IRODS_USER} + IRODS_PASSWORD: ${IRODS_PASSWORD} + volumes: + - irodshome:/home/${IRODS_USER} + - irodscerts:/opt/certificates + networks: + i_net: + depends_on: + - icat diff --git a/projects/b2safe/confs/debug.yml b/projects/b2safe/confs/debug.yml new file mode 120000 index 00000000..65caf82c --- /dev/null +++ b/projects/b2safe/confs/debug.yml @@ -0,0 +1 @@ +b2safe.yml \ No newline at end of file diff --git a/projects/b2safe/confs/simple_irods.yml b/projects/b2safe/confs/simple_irods.yml new file mode 100644 index 00000000..71632b8e --- /dev/null +++ b/projects/b2safe/confs/simple_irods.yml @@ -0,0 +1,10 @@ +version: '3' + +services: + + icat: + build: ../submodules/build-templates/icat + image: ${COMPOSE_PROJECT_NAME}/server:icat + # Open irods port to Outside world + ports: + - 1247:1247 diff --git a/projects/b2safe/project_configuration.yaml b/projects/b2safe/project_configuration.yaml new file mode 100644 index 00000000..7323abfa --- /dev/null +++ b/projects/b2safe/project_configuration.yaml @@ -0,0 +1,59 @@ +# ############################################### +# Copyright 2011-2017 EUDAT CDI - www.eudat.eu +# ############################################### + +project: + title: EUDAT-B2SAFE 'dockerized' server + description: Processing files in different EUDAT domains + version: v0.1.0 + +############################### +## Please CHECK VARIABLES here +## before starting your project +############################### +variables: + + env: + + IRODS_USER: irods + IRODS_GUEST_USER: guest # intended to work only with GSI + IRODS_DEFAULT_ADMIN_USER: rodsminer # intended to work only with GSI + IRODS_ZONE: tempZone + IRODS_HOME: home + IRODS_AUTHSCHEME: # to be auto-detected + # IRODS_AUTHSCHEME: credentials + # IRODS_AUTHSCHEME: GSI + # NOTE: this must match ALCHEMY_PASSWORD + IRODS_PASSWORD: chooseapasswordwisely + IRODS_DB: ICAT + IRODS_CHUNKSIZE: 1048576 + IRODS_ANONYMOUS: 1 + + # Postgres configuration: + ALCHEMY_USER: rods + ALCHEMY_PASSWORD: chooseapasswordwisely + ALCHEMY_DBS: ICAT + ALCHEMY_API_DB: + + # B2access certificates path + B2ACCESS_CAS: /tmp/certificates/b2access + +controller: + commands: + irods_restart: + description: Try to restart the current iCAT server instance + command: service irods restart + service: icat + user: root + +# Keep track of releases and dependencies +releases: + '0.6.2': + type: RC1 + rapydo: 0.5.7 + status: developing + # '1.0.0': + # type: stable + # # rapydo: 0.5.7 + # rapydo: null + # status: todo diff --git a/projects/eudat/frontend/templates/my_custom_template.html b/projects/b2stage/backend/__init__.py similarity index 100% rename from projects/eudat/frontend/templates/my_custom_template.html rename to projects/b2stage/backend/__init__.py diff --git a/projects/eudat/backend/__main__.py b/projects/b2stage/backend/__main__.py similarity index 100% rename from projects/eudat/backend/__main__.py rename to projects/b2stage/backend/__main__.py diff --git a/projects/b2stage/backend/apis/__init__.py b/projects/b2stage/backend/apis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/backend/apis/b2safeproxy.py b/projects/b2stage/backend/apis/b2safeproxy.py similarity index 95% rename from projects/eudat/backend/apis/b2safeproxy.py rename to projects/b2stage/backend/apis/b2safeproxy.py index 00f43433..b7de5663 100644 --- a/projects/eudat/backend/apis/b2safeproxy.py +++ b/projects/b2stage/backend/apis/b2safeproxy.py @@ -13,6 +13,8 @@ class B2safeProxy(EndpointResource): """ Login to B2SAFE: directly. """ + _anonymous_user = 'anonymous' + @decorate.catch_error() def get(self): @@ -48,6 +50,9 @@ def post(self): username = jargs.get('username') password = jargs.get('password') + if username == self._anonymous_user: + password = 'WHATEVERYOUWANT:)' + if username is None or password is None or \ username.strip() == '' or password.strip() == '': msg = "Missing username or password" diff --git a/projects/eudat/backend/apis/basic.py b/projects/b2stage/backend/apis/basic.py similarity index 97% rename from projects/eudat/backend/apis/basic.py rename to projects/b2stage/backend/apis/basic.py index 25fffdc2..bc0fc343 100644 --- a/projects/eudat/backend/apis/basic.py +++ b/projects/b2stage/backend/apis/basic.py @@ -16,8 +16,8 @@ from flask import request, current_app # from werkzeug import secure_filename -from eudat.apis.common import PRODUCTION, CURRENT_MAIN_ENDPOINT -from eudat.apis.common.b2stage import EudatEndpoint +from b2stage.apis.commons import PRODUCTION, CURRENT_MAIN_ENDPOINT +from b2stage.apis.commons.endpoint import EudatEndpoint from restapi.services.uploader import Uploader from restapi.flask_ext.flask_irods.client import IrodsException @@ -271,6 +271,9 @@ def put(self, irods_location=None): return self.send_errors('Location: missing filepath inside URI', code=hcodes.HTTP_BAD_REQUEST) irods_location = self.fix_location(irods_location) + # NOTE: irods_location will act strange due to Flask internals + # in case upload is served with streaming options, + # NOT finding the right path + filename if the path is a collection ################### # Basic init @@ -285,12 +288,8 @@ def put(self, irods_location=None): ipath = None request.get_data() - # FIXME: allow custom split of a custom response - # this piece of code does not work with a custom response - # if it changes the main blocks of the json root; - # the developer should be able to provide a 'custom_split' - # Manage both form and streaming upload + ################# # FORM UPLOAD if request.mimetype != 'application/octet-stream': @@ -339,6 +338,7 @@ def put(self, irods_location=None): # Transaction rollback: remove local cache in any case log.debug("Removing cache object") os.remove(abs_file) + ################# # STREAMING UPLOAD else: @@ -368,7 +368,7 @@ def put(self, irods_location=None): pid_found = True if not errors: out = {} - pid_parameter = self._args.get('pid') + pid_parameter = self._args.get('pid_await') if pid_parameter and 'true' in pid_parameter.lower(): # Shall we get the timeout from user? pid_found = False @@ -411,9 +411,8 @@ def put(self, irods_location=None): if pid_found: return self.force_response(content, errors=errors, code=status) else: - return self.send_warnings(content, - errors=errors, - code=hcodes.HTTP_OK_ACCEPTED) + return self.send_warnings( + content, errors=errors, code=hcodes.HTTP_OK_ACCEPTED) @decorate.catch_error(exception=IrodsException, exception_label='B2SAFE') def patch(self, irods_location=None): @@ -528,5 +527,4 @@ def delete(self, irods_location=None): icom.remove(irods_location, recursive=is_recursive, resource=resource) log.info("Removed %s", irods_location) - return {'removed': irods_location} diff --git a/projects/eudat/backend/apis/common/__init__.py b/projects/b2stage/backend/apis/commons/__init__.py similarity index 100% rename from projects/eudat/backend/apis/common/__init__.py rename to projects/b2stage/backend/apis/commons/__init__.py diff --git a/projects/eudat/backend/apis/common/b2access.py b/projects/b2stage/backend/apis/commons/b2access.py similarity index 99% rename from projects/eudat/backend/apis/common/b2access.py rename to projects/b2stage/backend/apis/commons/b2access.py index 55a1523c..9ea78d8c 100644 --- a/projects/eudat/backend/apis/common/b2access.py +++ b/projects/b2stage/backend/apis/commons/b2access.py @@ -16,7 +16,7 @@ from restapi.services.oauth2clients import decorate_http_request from utilities.certificates import Certificates from utilities import htmlcodes as hcodes -from eudat.apis.common import IRODS_EXTERNAL, InitObj +from b2stage.apis.commons import IRODS_EXTERNAL, InitObj from utilities.logs import get_logger log = get_logger(__name__) diff --git a/projects/b2stage/backend/apis/commons/b2handle.py b/projects/b2stage/backend/apis/commons/b2handle.py new file mode 100644 index 00000000..f723d0e9 --- /dev/null +++ b/projects/b2stage/backend/apis/commons/b2handle.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +""" +B2HANDLE utilities +""" + +import os +from b2stage.apis.commons.endpoint import EudatEndpoint +from b2handle.handleclient import EUDATHandleClient as b2handle +from b2handle.clientcredentials import PIDClientCredentials as credentials +from b2handle import handleexceptions +from utilities import htmlcodes as hcodes +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class B2HandleEndpoint(EudatEndpoint): + + """ + Handling PID requests. + It includes some methods to connect to B2HANDLE. + + FIXME: it should become a dedicated service in rapydo. + This way the client could be registered in memory with credentials + only if the provided credentials are working. + It should be read only access otherwise. + + """ + + eudat_pid_fields = [ + "URL", "EUDAT/CHECKSUM", "EUDAT/UNPUBLISHED", + "EUDAT/UNPUBLISHED_DATE", "EUDAT/UNPUBLISHED_REASON" + ] + + def connect_client(self, force_no_credentials=False): + + found = False + + # With credentials + if not force_no_credentials: + file = os.environ.get('HANDLE_CREDENTIALS', None) + if file is not None: + from utilities import path + credentials_path = path.build(file) + found = path.file_exists_and_nonzero(credentials_path) + if not found: + log.warning( + "B2HANDLE credentials file not found %s", file) + + if found: + client = b2handle.instantiate_with_credentials( + credentials.load_from_JSON(file) + ) + log.info("PID client connected: w/ credentials") + return client, True + + client = b2handle.instantiate_for_read_access() + log.warning("PID client connected: NO credentials") + return client, False + + def handle_pid_fields(self, client, pid): + """ Perform B2HANDLE request: retrieve URL from handle """ + + import requests + data = {} + try: + for field in self.eudat_pid_fields: + value = client.get_value_from_handle(pid, field) + log.info("B2HANDLE: %s=%s", field, value) + data[field] = value + except handleexceptions.HandleSyntaxError as e: + return data, e, hcodes.HTTP_BAD_REQUEST + except handleexceptions.HandleNotFoundException as e: + return data, e, hcodes.HTTP_BAD_NOTFOUND + except handleexceptions.GenericHandleError as e: + return data, e, hcodes.HTTP_SERVER_ERROR + except handleexceptions.HandleAuthenticationError as e: + return data, e, hcodes.HTTP_BAD_UNAUTHORIZED + except requests.exceptions.ConnectionError as e: + log.warning("No connection available...") + return data, e, hcodes.HTTP_SERVER_ERROR + except BaseException as e: + log.error("Generic:\n%s(%s)", e.__class__.__name__, e) + return data, e, hcodes.HTTP_SERVER_ERROR + + return data, None, hcodes.HTTP_FOUND + + def get_pid_metadata(self, pid): + + # First test: check if credentials exists and works + client, authenticated = self.connect_client() + data, error, code = self.handle_pid_fields(client, pid) + + # If credentials were found but they gave error + # TODO: this should be tested at server startup! + if error is not None and authenticated: + log.error("B2HANDLE credentials problem: %s", error) + client, _ = self.connect_client(force_no_credentials=True) + data, error, code = self.handle_pid_fields(client, pid) + + # Still getting error? Raise any B2HANDLE library problem + if error is not None: + log.error("B2HANDLE problem: %s", error) + return data, \ + self.send_errors(message='B2HANDLE: %s' % error, code=code) + else: + return data, None diff --git a/projects/eudat/backend/apis/common/b2stage.py b/projects/b2stage/backend/apis/commons/endpoint.py similarity index 97% rename from projects/eudat/backend/apis/common/b2stage.py rename to projects/b2stage/backend/apis/commons/endpoint.py index d6b9ee7b..a722eedc 100644 --- a/projects/eudat/backend/apis/common/b2stage.py +++ b/projects/b2stage/backend/apis/commons/endpoint.py @@ -7,8 +7,8 @@ import os # from restapi.rest.definition import EndpointResource from restapi.exceptions import RestApiException -from eudat.apis.common.b2access import B2accessUtilities -from eudat.apis.common import ( +from b2stage.apis.commons.b2access import B2accessUtilities +from b2stage.apis.commons import ( CURRENT_HTTPAPI_SERVER, CURRENT_B2SAFE_SERVER, IRODS_PROTOCOL, HTTP_PROTOCOL, PRODUCTION, IRODS_VARS, InitObj @@ -46,6 +46,10 @@ def init_endpoint(self): proxy = False external_user = None + + if internal_user is None: + raise AttributeError("Missing user association to token") + if internal_user.authmethod == 'credentials': icom = self.irodsuser_from_b2stage(internal_user) elif internal_user.authmethod == 'irods': @@ -63,8 +67,6 @@ def init_endpoint(self): # icd and ipwd do not give error with wrong certificates... # so the minimum command is ils inside the home dir icom.list() - # TODO: doublecheck if list is the best option with PRC - if proxy: log.debug("Current proxy certificate is valid") @@ -254,12 +256,8 @@ def get_file_parameters(self, icom, adding or removing replicas require explicit irods commands. """ - # iuser = icom.get_current_user() - - ############################ # Handle flask differences on GET/DELETE and PUT/POST myargs = self.get_input() - # log.pp(myargs) ############################ # main parameters diff --git a/projects/b2stage/backend/apis/extended.py b/projects/b2stage/backend/apis/extended.py new file mode 100644 index 00000000..ebc78fff --- /dev/null +++ b/projects/b2stage/backend/apis/extended.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +""" +B2SAFE HTTP REST API endpoints. +Code to implement the extended endpoints. + +Note: +Endpoints list and behaviour are available at: +https://github.com/EUDAT-B2STAGE/http-api/blob/master/docs/user/endpoints.md + +""" + +from restapi.flask_ext.flask_irods.client import IrodsException +from b2stage.apis.commons import CURRENT_HTTPAPI_SERVER +from b2stage.apis.commons.b2handle import B2HandleEndpoint + +from restapi.services.uploader import Uploader +from utilities import htmlcodes as hcodes +from restapi import decorators as decorate + +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class PIDEndpoint(Uploader, B2HandleEndpoint): + """ Handling PID on endpoint requests """ + + @decorate.catch_error(exception=IrodsException, exception_label='B2SAFE') + def get(self, pid=None): + """ Get metadata or file from pid """ + + if pid is None: + return self.send_errors( + message='Missing PID inside URI', code=hcodes.HTTP_BAD_REQUEST) + + # recover metadata from pid + metadata, bad_response = self.get_pid_metadata(pid) + if bad_response is not None: + return bad_response + url = metadata.get('URL') + if url is None: + return self.send_errors( + message='B2HANDLE: empty URL_value returned', + code=hcodes.HTTP_BAD_NOTFOUND) + + # parse query parameters + self.get_input() + download = False + if (hasattr(self._args, 'download')): + if self._args.download and 'true' in self._args.download.lower(): + download = True + + # If download is True, trigger file download + if download: + api_url = CURRENT_HTTPAPI_SERVER + + # If local HTTP-API perform a direct download + # FIXME: the following code can be improved + route = api_url + 'api/registered/' + # route = route.replace('http://', '') + + if (url.startswith(route)): + url = url.replace(route, '/') + r = self.init_endpoint() + if r.errors is not None: + return self.send_errors(errors=r.errors) + url = self.download_object(r, url) + else: + # Perform a request to an external service? + return self.send_warnings( + {'URL': url}, + errors=[ + "Data-object can't be downloaded by current " + + "HTTP-API server '%s'" % api_url + ] + ) + return url + + # When no download is requested + return metadata diff --git a/projects/eudat/backend/apis/internal.py b/projects/b2stage/backend/apis/internal.py similarity index 94% rename from projects/eudat/backend/apis/internal.py rename to projects/b2stage/backend/apis/internal.py index 2f365e30..7e768d9e 100644 --- a/projects/eudat/backend/apis/internal.py +++ b/projects/b2stage/backend/apis/internal.py @@ -7,8 +7,8 @@ """ -from eudat.apis.common.b2stage import EudatEndpoint -from eudat.apis.common import CURRENT_MAIN_ENDPOINT +from b2stage.apis.commons.endpoint import EudatEndpoint +from b2stage.apis.commons import CURRENT_MAIN_ENDPOINT from restapi import decorators as decorate from restapi.flask_ext.flask_irods.client import IrodsException from utilities import htmlcodes as hcodes diff --git a/projects/eudat/backend/apis/oauth.py b/projects/b2stage/backend/apis/oauth.py similarity index 94% rename from projects/eudat/backend/apis/oauth.py rename to projects/b2stage/backend/apis/oauth.py index 9013cea0..ee2e4a91 100644 --- a/projects/eudat/backend/apis/oauth.py +++ b/projects/b2stage/backend/apis/oauth.py @@ -8,9 +8,9 @@ from restapi.flask_ext.flask_irods.client import IrodsException from restapi import decorators as decorate # from utilities import htmlcodes as hcodes -from eudat.apis.common import PRODUCTION -# from eudat.apis.common.b2access import B2accessUtilities -from eudat.apis.common.b2stage import EudatEndpoint +from b2stage.apis.commons import PRODUCTION +# from b2stage.apis.commons.b2access import B2accessUtilities +from b2stage.apis.commons.endpoint import EudatEndpoint from utilities import htmlcodes as hcodes from utilities.logs import get_logger @@ -29,10 +29,8 @@ class OauthLogin(EudatEndpoint): exception_label='Server side B2ACCESS misconfiguration') def get(self): - from flask import request - # agent = request.headers.get('User-Agent') - # log.pp(request.user_agent.__dict__) - if request.user_agent.browser is None: + from restapi.rest.response import request_from_browser + if not request_from_browser(): return self.send_errors( "B2ACCESS authorization must be requested from a browser", code=hcodes.HTTP_BAD_METHOD_NOT_ALLOWED diff --git a/projects/b2stage/backend/apis/public.py b/projects/b2stage/backend/apis/public.py new file mode 100644 index 00000000..ab7ba424 --- /dev/null +++ b/projects/b2stage/backend/apis/public.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +""" +B2SAFE HTTP REST API endpoints. +Getting informations for public data. +""" + +from b2stage.apis.commons.b2handle import B2HandleEndpoint +from restapi import decorators as decorate +from utilities import htmlcodes as hcodes +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class Public(B2HandleEndpoint): + + @decorate.catch_error() + def get(self, irods_location): + + #################### + # self.get_input() + # log.pp(self._args, prefix_line='Parsed args') + + #################### + if irods_location is None: + return self.send_errors( + 'Location: missing filepath inside URI', + code=hcodes.HTTP_BAD_REQUEST) + irods_location = self.fix_location(irods_location) + + #################### + # check if public, with anonymous access in irods + icom = self.get_service_instance( + service_name='irods', user='anonymous', password='null') + + path, resource, filename, force = \ + self.get_file_parameters(icom, path=irods_location) + if icom.is_collection(path): + return self.send_errors( + message="Path provided is a collection", + code=hcodes.HTTP_BAD_REQUEST + ) + + # list content + tmp = icom.list(irods_location) + log.pp(tmp) + + #################### + # # look for pid metadata + # pid = '11100/33ac01fc-6850-11e5-b66e-e41f13eb32b2' + # metadata, bad_response = self.get_pid_metadata(pid) + + #################### + # check if browser + from restapi.rest.response import request_from_browser + if request_from_browser: + # # use logos in an html reply + # https://www.eudat.eu/sites/default/files/logo-b2stage.png + # https://www.eudat.eu/sites/default/files/EUDAT-logo.png + return """

This is HTML content

""" + + #################### + return 'Not a browser' diff --git a/projects/b2stage/backend/apis/publish.py b/projects/b2stage/backend/apis/publish.py new file mode 100644 index 00000000..53ea5d51 --- /dev/null +++ b/projects/b2stage/backend/apis/publish.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- + +""" +Publish a digital object as a public resource for anyone + +NOTE: this package will be loaded only if IRODS_ANONYMOUS is set +""" + +from utilities import htmlcodes as hcodes +from b2stage.apis.commons.endpoint import EudatEndpoint +from restapi import decorators as decorate +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class Publish(EudatEndpoint): + + def base(self, irods_location): + + if irods_location is None: + return self.send_errors( + 'Location: missing filepath inside URI', + code=hcodes.HTTP_BAD_REQUEST + ), None, None + else: + irods_location = self.fix_location(irods_location) + + r = self.init_endpoint() + if r.errors is not None: + return self.send_errors(errors=r.errors), None, None + + path, resource, filename, _ = \ + self.get_file_parameters(r.icommands, path=irods_location) + + # if r.icommands.is_collection(path): + # return self.send_errors( + # 'Provided path is a collection. ' + + # 'Publishing is not allowed as recursive.', + # code=hcodes.HTTP_NOT_IMPLEMENTED + # ), None, None + + # Does this path exist? + if not r.icommands.exists(path): + return self.send_errors( + errors=[{ + 'path': "'%s': not existing or no permissions" % path + }], code=hcodes.HTTP_BAD_NOTFOUND), \ + None, None + + return None, r, path + + def single_path_check(self, icom, zone, abs_path, check=True): + + permissions = icom.get_permissions(abs_path) + acls = permissions.get('ACL', []) + + published = False + for acl_user, acl_zone, acl_mode in acls: + if acl_zone == zone and acl_user == icom.anonymous_user: + if check: + if 'read' in acl_mode: + published = True + break + + return published + + def single_permission(self, icom, ipath, permission=None): + + icom.set_permissions( + ipath, + # NOTE: permission could be: read, write, null/None + permission=permission, + userOrGroup=icom.anonymous_user, + ) # , recursive=False) + # FIXME: should we publish recursively to subfiles and subfolders? + # NOTE: It looks dangerous to me + + def publish_helper(self, icom, ipath, check_only=True, unpublish=False): + + from utilities import path + current_zone = icom.get_current_zone() + ipath_steps = path.parts(ipath) + current = '' + + for ipath_step in ipath_steps: + + current = path.join(current, ipath_step, return_str=True) + # print("PUB STEP:", ipath_step, current, len(current)) + + # to skip: root dir, zone and home + if len(ipath_step) == 1 \ + or ipath_step == current_zone or ipath_step == 'home': + continue + + # find out if already published + check = self.single_path_check( + icom, current_zone, str(current)) + # if only checking + if check_only and not check: + return False + # otherwise you want to publish/unpublish this path + else: + if unpublish: + self.single_permission(icom, current, permission=None) + else: + self.single_permission(icom, current, permission='read') + + return True + + @decorate.catch_error() + def get(self, irods_location): + + error, handler, path = self.base(irods_location) + if error is not None: + return error + + icom = handler.icommands + user = icom.get_current_user() + log.info("user '%s' requested to check '%s'", user, path) + return {'published': self.publish_helper(icom, path)} + + @decorate.catch_error() + def put(self, irods_location=None): + + error, handler, path = self.base(irods_location) + if error is not None: + return error + + icom = handler.icommands + user = icom.get_current_user() + log.info("user '%s' requested to publish '%s'", user, path) + + # if already set as the same don't do anything + if not self.publish_helper(icom, path): + self.publish_helper(icom, path, check_only=False, unpublish=False) + + # # If you'd like to check again: + # return {'published': self.publish_helper(icom, path)} + return {'published': True} + + @decorate.catch_error() + def delete(self, irods_location): + + error, handler, path = self.base(irods_location) + if error is not None: + return error + + icom = handler.icommands + user = icom.get_current_user() + log.info("user '%s' requested to UNpublish '%s'", user, path) + + # if not already set as the same don't do anything + if self.publish_helper(icom, path): + self.publish_helper(icom, path, check_only=False, unpublish=True) + + # # If you'd like to check again: + # return {'published': self.publish_helper(icom, path)} + return {'published': False} diff --git a/projects/eudat/backend/apis/response.py b/projects/b2stage/backend/apis/response.py similarity index 100% rename from projects/eudat/backend/apis/response.py rename to projects/b2stage/backend/apis/response.py diff --git a/projects/eudat/backend/apis/services.py b/projects/b2stage/backend/apis/services.py similarity index 100% rename from projects/eudat/backend/apis/services.py rename to projects/b2stage/backend/apis/services.py diff --git a/projects/b2stage/backend/models/__init__.py b/projects/b2stage/backend/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/backend/models/mongo.py b/projects/b2stage/backend/models/mongo.py similarity index 100% rename from projects/eudat/backend/models/mongo.py rename to projects/b2stage/backend/models/mongo.py diff --git a/projects/eudat/backend/models/neo4j.py b/projects/b2stage/backend/models/neo4j.py similarity index 100% rename from projects/eudat/backend/models/neo4j.py rename to projects/b2stage/backend/models/neo4j.py diff --git a/projects/eudat/backend/models/sqlalchemy.py b/projects/b2stage/backend/models/sqlalchemy.py similarity index 100% rename from projects/eudat/backend/models/sqlalchemy.py rename to projects/b2stage/backend/models/sqlalchemy.py diff --git a/projects/b2stage/backend/project/__init__.py b/projects/b2stage/backend/project/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/backend/project/initialization.py b/projects/b2stage/backend/project/initialization.py similarity index 100% rename from projects/eudat/backend/project/initialization.py rename to projects/b2stage/backend/project/initialization.py diff --git a/projects/eudat/backend/swagger/b2access_authorize/get.yaml b/projects/b2stage/backend/swagger/b2access_authorize/get.yaml similarity index 100% rename from projects/eudat/backend/swagger/b2access_authorize/get.yaml rename to projects/b2stage/backend/swagger/b2access_authorize/get.yaml diff --git a/projects/eudat/backend/swagger/b2access_authorize/specs.yaml b/projects/b2stage/backend/swagger/b2access_authorize/specs.yaml similarity index 79% rename from projects/eudat/backend/swagger/b2access_authorize/specs.yaml rename to projects/b2stage/backend/swagger/b2access_authorize/specs.yaml index a55c5e76..ac74be83 100644 --- a/projects/eudat/backend/swagger/b2access_authorize/specs.yaml +++ b/projects/b2stage/backend/swagger/b2access_authorize/specs.yaml @@ -1,4 +1,7 @@ +depends_on: + - B2ACCESS_APPKEY + class: Authorize file: oauth baseuri: /auth diff --git a/projects/eudat/backend/swagger/b2access_proxy/post.yaml b/projects/b2stage/backend/swagger/b2access_proxy/post.yaml similarity index 100% rename from projects/eudat/backend/swagger/b2access_proxy/post.yaml rename to projects/b2stage/backend/swagger/b2access_proxy/post.yaml diff --git a/projects/eudat/backend/swagger/b2access_proxy/specs.yaml b/projects/b2stage/backend/swagger/b2access_proxy/specs.yaml similarity index 77% rename from projects/eudat/backend/swagger/b2access_proxy/specs.yaml rename to projects/b2stage/backend/swagger/b2access_proxy/specs.yaml index f3682c51..79f5f1e6 100644 --- a/projects/eudat/backend/swagger/b2access_proxy/specs.yaml +++ b/projects/b2stage/backend/swagger/b2access_proxy/specs.yaml @@ -1,4 +1,7 @@ +depends_on: + - B2ACCESS_APPKEY + class: B2accesProxyEndpoint file: oauth baseuri: /auth diff --git a/projects/eudat/backend/swagger/b2access_redirect/get.yaml b/projects/b2stage/backend/swagger/b2access_redirect/get.yaml similarity index 100% rename from projects/eudat/backend/swagger/b2access_redirect/get.yaml rename to projects/b2stage/backend/swagger/b2access_redirect/get.yaml diff --git a/projects/eudat/backend/swagger/b2access_redirect/specs.yaml b/projects/b2stage/backend/swagger/b2access_redirect/specs.yaml similarity index 81% rename from projects/eudat/backend/swagger/b2access_redirect/specs.yaml rename to projects/b2stage/backend/swagger/b2access_redirect/specs.yaml index 8f3e41b5..28181e54 100644 --- a/projects/eudat/backend/swagger/b2access_redirect/specs.yaml +++ b/projects/b2stage/backend/swagger/b2access_redirect/specs.yaml @@ -1,4 +1,7 @@ +depends_on: + - B2ACCESS_APPKEY + class: OauthLogin file: oauth baseuri: /auth diff --git a/projects/eudat/backend/swagger/b2safe_proxy/get.yaml b/projects/b2stage/backend/swagger/b2safe_proxy/get.yaml similarity index 100% rename from projects/eudat/backend/swagger/b2safe_proxy/get.yaml rename to projects/b2stage/backend/swagger/b2safe_proxy/get.yaml diff --git a/projects/eudat/backend/swagger/b2safe_proxy/post.yaml b/projects/b2stage/backend/swagger/b2safe_proxy/post.yaml similarity index 100% rename from projects/eudat/backend/swagger/b2safe_proxy/post.yaml rename to projects/b2stage/backend/swagger/b2safe_proxy/post.yaml diff --git a/projects/eudat/backend/swagger/b2safe_proxy/specs.yaml b/projects/b2stage/backend/swagger/b2safe_proxy/specs.yaml similarity index 82% rename from projects/eudat/backend/swagger/b2safe_proxy/specs.yaml rename to projects/b2stage/backend/swagger/b2safe_proxy/specs.yaml index 881fbf96..b57024b7 100644 --- a/projects/eudat/backend/swagger/b2safe_proxy/specs.yaml +++ b/projects/b2stage/backend/swagger/b2safe_proxy/specs.yaml @@ -1,4 +1,7 @@ +depends_on: + - not B2ACCESS_APPKEY + file: b2safeproxy class: B2safeProxy baseuri: /auth diff --git a/projects/eudat/backend/swagger/internal/patch.yaml b/projects/b2stage/backend/swagger/internal/patch.yaml similarity index 100% rename from projects/eudat/backend/swagger/internal/patch.yaml rename to projects/b2stage/backend/swagger/internal/patch.yaml diff --git a/projects/eudat/backend/swagger/internal/specs.yaml b/projects/b2stage/backend/swagger/internal/specs.yaml similarity index 100% rename from projects/eudat/backend/swagger/internal/specs.yaml rename to projects/b2stage/backend/swagger/internal/specs.yaml diff --git a/projects/b2stage/backend/swagger/models.yaml b/projects/b2stage/backend/swagger/models.yaml new file mode 100644 index 00000000..276f7192 --- /dev/null +++ b/projects/b2stage/backend/swagger/models.yaml @@ -0,0 +1,32 @@ + +definitions: + + FileUpdate: + required: + - newname + properties: + resource: + type: string + newname: + type: string + description: iRODS resource + + FileDelete: + properties: + # debugclean: + # type: string + # description: Only for debug mode + resource: + type: string + + # FileObject: + # properties: + # resource: + # type: string + # # description: yep + + # ForceObject: + # properties: + # force: + # type: boolean + # default: false diff --git a/projects/eudat/backend/swagger/pids/get.yaml b/projects/b2stage/backend/swagger/pids/get.yaml similarity index 100% rename from projects/eudat/backend/swagger/pids/get.yaml rename to projects/b2stage/backend/swagger/pids/get.yaml diff --git a/projects/eudat/backend/swagger/pids/specs.yaml b/projects/b2stage/backend/swagger/pids/specs.yaml similarity index 100% rename from projects/eudat/backend/swagger/pids/specs.yaml rename to projects/b2stage/backend/swagger/pids/specs.yaml diff --git a/projects/b2stage/backend/swagger/public/SKIP b/projects/b2stage/backend/swagger/public/SKIP new file mode 100644 index 00000000..e69de29b diff --git a/projects/b2stage/backend/swagger/public/get.yaml b/projects/b2stage/backend/swagger/public/get.yaml new file mode 100644 index 00000000..734edc57 --- /dev/null +++ b/projects/b2stage/backend/swagger/public/get.yaml @@ -0,0 +1,8 @@ + +public: + custom: + authentication: false + summary: Let non-authenticated users get data about a public data-object + responses: + '200': + description: Informations about the data-object \ No newline at end of file diff --git a/projects/b2stage/backend/swagger/public/specs.yaml b/projects/b2stage/backend/swagger/public/specs.yaml new file mode 100644 index 00000000..94bd02f5 --- /dev/null +++ b/projects/b2stage/backend/swagger/public/specs.yaml @@ -0,0 +1,11 @@ + +file: public +class: Public +schema: + expose: true +mapping: + public: "/public/" +labels: + - eudat + - pids + - public \ No newline at end of file diff --git a/projects/b2stage/backend/swagger/publish/delete.yaml b/projects/b2stage/backend/swagger/publish/delete.yaml new file mode 100644 index 00000000..210b58df --- /dev/null +++ b/projects/b2stage/backend/swagger/publish/delete.yaml @@ -0,0 +1,7 @@ +publish: + custom: + authentication: true + summary: Ensure that the irods path is not readable for anonymous user + responses: + '200': + description: return a bolean 'published' if readable for anonymous diff --git a/projects/b2stage/backend/swagger/publish/get.yaml b/projects/b2stage/backend/swagger/publish/get.yaml new file mode 100644 index 00000000..8ff6d473 --- /dev/null +++ b/projects/b2stage/backend/swagger/publish/get.yaml @@ -0,0 +1,8 @@ + +publish: + custom: + authentication: true + summary: Check if that irods path is currently readable for anonymous user + responses: + '200': + description: return a bolean 'published' if readable for anonymous \ No newline at end of file diff --git a/projects/b2stage/backend/swagger/publish/put.yaml b/projects/b2stage/backend/swagger/publish/put.yaml new file mode 100644 index 00000000..d1798319 --- /dev/null +++ b/projects/b2stage/backend/swagger/publish/put.yaml @@ -0,0 +1,7 @@ +publish: + custom: + authentication: true + summary: Ensure that the irods path becomes readable for anonymous user + responses: + '200': + description: return a bolean 'published' if readable for anonymous \ No newline at end of file diff --git a/projects/b2stage/backend/swagger/publish/specs.yaml b/projects/b2stage/backend/swagger/publish/specs.yaml new file mode 100644 index 00000000..5ffdda4d --- /dev/null +++ b/projects/b2stage/backend/swagger/publish/specs.yaml @@ -0,0 +1,15 @@ + +depends_on: + - IRODS_ANONYMOUS + +file: publish +class: Publish +schema: + expose: true +mapping: + publish: "/publish/" +ids: + irods_location: the filesystem path to your digital entities/objects +labels: + - eudat + - publish diff --git a/projects/eudat/backend/swagger/registered/delete.yaml b/projects/b2stage/backend/swagger/registered/delete.yaml similarity index 100% rename from projects/eudat/backend/swagger/registered/delete.yaml rename to projects/b2stage/backend/swagger/registered/delete.yaml diff --git a/projects/eudat/backend/swagger/registered/get.yaml b/projects/b2stage/backend/swagger/registered/get.yaml similarity index 75% rename from projects/eudat/backend/swagger/registered/get.yaml rename to projects/b2stage/backend/swagger/registered/get.yaml index 4cbf2182..07c5894f 100644 --- a/projects/eudat/backend/swagger/registered/get.yaml +++ b/projects/b2stage/backend/swagger/registered/get.yaml @@ -26,5 +26,4 @@ single: type: boolean responses: '200': - description: Returns the digital object information or file content if download is activated - or the list of objects related to the requested path (PID is returned if available) + description: "Returns the digital object information or file content if download is activated or the list of objects related to the requested path (PID is returned if available)" diff --git a/projects/eudat/backend/swagger/registered/patch.yaml b/projects/b2stage/backend/swagger/registered/patch.yaml similarity index 100% rename from projects/eudat/backend/swagger/registered/patch.yaml rename to projects/b2stage/backend/swagger/registered/patch.yaml diff --git a/projects/eudat/backend/swagger/registered/post.yaml b/projects/b2stage/backend/swagger/registered/post.yaml similarity index 100% rename from projects/eudat/backend/swagger/registered/post.yaml rename to projects/b2stage/backend/swagger/registered/post.yaml diff --git a/projects/eudat/backend/swagger/registered/put.yaml b/projects/b2stage/backend/swagger/registered/put.yaml similarity index 96% rename from projects/eudat/backend/swagger/registered/put.yaml rename to projects/b2stage/backend/swagger/registered/put.yaml index b2fd8f47..537f5833 100644 --- a/projects/eudat/backend/swagger/registered/put.yaml +++ b/projects/b2stage/backend/swagger/registered/put.yaml @@ -25,7 +25,7 @@ single: type: boolean # default: false description: force action even if getting warnings - - name: pid + - name: pid_await in: query type: boolean description: Returns PID in the JSON response diff --git a/projects/eudat/backend/swagger/registered/specs.yaml b/projects/b2stage/backend/swagger/registered/specs.yaml similarity index 100% rename from projects/eudat/backend/swagger/registered/specs.yaml rename to projects/b2stage/backend/swagger/registered/specs.yaml diff --git a/projects/b2stage/backend/swagger/template/SKIP b/projects/b2stage/backend/swagger/template/SKIP new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/backend/swagger/template/get.yaml b/projects/b2stage/backend/swagger/template/get.yaml similarity index 100% rename from projects/eudat/backend/swagger/template/get.yaml rename to projects/b2stage/backend/swagger/template/get.yaml diff --git a/projects/eudat/backend/swagger/template/specs.yaml b/projects/b2stage/backend/swagger/template/specs.yaml similarity index 100% rename from projects/eudat/backend/swagger/template/specs.yaml rename to projects/b2stage/backend/swagger/template/specs.yaml diff --git a/projects/b2stage/backend/tests/__init__.py b/projects/b2stage/backend/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/backend/tests/test_b2access.py b/projects/b2stage/backend/tests/test_b2access.py similarity index 100% rename from projects/eudat/backend/tests/test_b2access.py rename to projects/b2stage/backend/tests/test_b2access.py diff --git a/projects/eudat/backend/tests/test_b2safeproxy.py b/projects/b2stage/backend/tests/test_b2safeproxy.py similarity index 99% rename from projects/eudat/backend/tests/test_b2safeproxy.py rename to projects/b2stage/backend/tests/test_b2safeproxy.py index 4c5c7232..e2c33476 100644 --- a/projects/eudat/backend/tests/test_b2safeproxy.py +++ b/projects/b2stage/backend/tests/test_b2safeproxy.py @@ -4,7 +4,6 @@ Run unittests inside the RAPyDo framework """ -import json from tests import RestTestsAuthenticatedBase from utilities.logs import get_logger diff --git a/projects/eudat/backend/tests/test_example.py b/projects/b2stage/backend/tests/test_example.py similarity index 100% rename from projects/eudat/backend/tests/test_example.py rename to projects/b2stage/backend/tests/test_example.py diff --git a/projects/eudat/backend/tests/test_pids.py b/projects/b2stage/backend/tests/test_pids.py similarity index 57% rename from projects/eudat/backend/tests/test_pids.py rename to projects/b2stage/backend/tests/test_pids.py index 67b99cd1..f4675073 100644 --- a/projects/eudat/backend/tests/test_pids.py +++ b/projects/b2stage/backend/tests/test_pids.py @@ -8,7 +8,7 @@ """ # import io -import json +# import json from tests import RestTestsAuthenticatedBase from utilities.logs import get_logger @@ -21,12 +21,12 @@ class TestPids(RestTestsAuthenticatedBase): - _main_endpoint = '/pids' + _main_endpoint = '/pids/' - def tearDown(self): + # def tearDown(self): - log.debug('### Cleaning custom data ###\n') - super().tearDown() + # log.debug('### Cleaning custom data ###\n') + # super().tearDown() def test_01_GET_public_PID(self): """ Test directory creation: POST """ @@ -34,22 +34,21 @@ def test_01_GET_public_PID(self): log.info('*** Testing GET public PID') pid = '11100/33ac01fc-6850-11e5-b66e-e41f13eb32b2' - worng_pid = '11100/33ac01fc-6850-11e5-XXXX-e41f13eb3212' + pid_uri_path = 'data.repo.cineca.it:1247' + \ + '/CINECA01/home/cin_staff/rmucci00/DSI_Test/test.txt' + wrong_pid = pid.replace('6850', 'XXXX') # GET URL from PID - endpoint = (self._api_uri + self._main_endpoint + - '/' + pid) + endpoint = self._api_uri + self._main_endpoint + pid r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) - data = json.loads(r.get_data(as_text=True)) - - # To be fixed - self.assertEqual( - data['Response']['data']['URL'], - 'irods://data.repo.cineca.it:1247/CINECA01/home/cin_staff/rmucci00/DSI_Test/test.txt') + # data = json.loads(r.get_data(as_text=True)) + data = self.get_content(r) + self.assertEqual(data.get('URL'), 'irods://%s' % pid_uri_path) # GET URL from non existing PID - endpoint = (self._api_uri + self._main_endpoint + - '/' + worng_pid) + endpoint = self._api_uri + self._main_endpoint + wrong_pid r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + + # TODO: we may test right credentials using Travis secret variables diff --git a/projects/b2stage/backend/tests/test_public.py b/projects/b2stage/backend/tests/test_public.py new file mode 100644 index 00000000..5ef60f0f --- /dev/null +++ b/projects/b2stage/backend/tests/test_public.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +""" To be defined from designing this endoint """ + +from tests import RestTestsAuthenticatedBase +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class TestPublic(RestTestsAuthenticatedBase): + + _main_endpoint = '/public' + + # def setUp(self): + # """ override original setup to create custom data at each request """ + + # super().setUp() # Call father's method + # log.debug('### Create some custom data ###\n') + + def test_01_GET_public_data(self): + + assert True + + # endpoint = (self._api_uri + self._main_endpoint) + # log.info('*** Testing GET call on %s' % endpoint) + + # r = self.app.get(endpoint) + # self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + # output = self.get_content(r) + # # log.pp(output) + # self.assertEqual(output, 'Hello world!') + + # def tearDown(self): + # """ override teardown: remove custom data at each request """ + + # log.debug('### Cleaning custom data ###\n') + # super().tearDown() diff --git a/projects/b2stage/backend/tests/test_publish.py b/projects/b2stage/backend/tests/test_publish.py new file mode 100644 index 00000000..c637f575 --- /dev/null +++ b/projects/b2stage/backend/tests/test_publish.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- + +""" +Run unittests inside the RAPyDo framework +""" + +import io +import json +from tests import RestTestsAuthenticatedBase +from restapi.services.detect import detector +from utilities import path +from utilities.logs import get_logger + +log = get_logger(__name__) + + +class TestPublish(RestTestsAuthenticatedBase): + + _auth_endpoint = '/b2safeproxy' + _register_endpoint = '/registered' + _main_endpoint = '/publish' + _anonymous_user = 'anonymous' + _main_key = 'published' + + def setUp(self): + + # Call father's method + super().setUp() + + log.info("\n### Creating a test token (for ANONYMOUS IRODS user) ###") + credentials = json.dumps({'username': self._anonymous_user}) + endpoint = self._auth_uri + self._auth_endpoint + + log.debug('*** Testing anonymous authentication on %s' % endpoint) + r = self.app.post(endpoint, data=credentials) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + content = self.get_content(r) + self.save_token(content.get('token'), suffix=self._anonymous_user) + + self.irods_vars = detector.services_classes.get('irods').variables + self._filename = 'some_file.txt' + home_dirname = 'home' + self._ipath = str(path.join( + path.root(), self.irods_vars.get('zone'), + home_dirname, self.irods_vars.get('guest_user'), self._filename + )) + self._no_permission_path = str(path.join( + path.root(), self.irods_vars.get('zone'), + home_dirname, 'nonexisting' + )) + log.debug('*** Upload a test file: %s' % self._ipath) + + # Upload entity in test folder + endpoint = self._api_uri + self._register_endpoint + self._ipath + r = self.app.put( + endpoint, + data=dict(file=(io.BytesIO(b"just a test"), self._filename)), + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + + def test_01_GET_check_if_published(self): + + endpoint = self._api_uri + self._main_endpoint + log.info('*** Testing GET call on %s' % endpoint) + + # Current file is not published + r = self.app.get( + endpoint + self._ipath, headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is False + + # Random file: does not work + r = self.app.get( + endpoint + self._ipath + 'wrong', + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + errors = self.get_content(r, return_errors=True) + # log.pp(errors) + assert 'not existing' in errors.pop().get('path') + + # Some other user directory: does not work + r = self.app.get( + endpoint + self._no_permission_path, + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + errors = self.get_content(r, return_errors=True) + # log.pp(errors) + assert 'no permissions' in errors.pop().get('path') + + def test_02_PUT_publish_dataobject(self): + + endpoint = self._api_uri + self._main_endpoint + log.info('*** Testing PUT call on %s' % endpoint) + + # Publish the file which was already uploaded + r = self.app.put( + endpoint + self._ipath, + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is True + + # Current file is now published + r = self.app.get( + endpoint + self._ipath, headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is True + + # Current file can be accessed by anonymous with /api/registered + anonymous_endpoint = self._api_uri + self._register_endpoint + r = self.app.get( + anonymous_endpoint + self._ipath, + headers=self.__class__.auth_header_anonymous) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + data_object = data.pop().get(self._filename, {}) + key = 'metadata' + self.assertIn(key, data_object) + metadata = data_object.get(key) + self.assertEqual(metadata.get('name'), self._filename) + self.assertEqual(metadata.get('object_type'), 'dataobject') + + # Random file: cannot unpublish + r = self.app.put( + endpoint + self._ipath + 'wrong', + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + errors = self.get_content(r, return_errors=True) + assert 'not existing' in errors.pop().get('path') + + def test_03_POST_not_working(self): + + endpoint = self._api_uri + self._main_endpoint + log.info('*** Testing POST call on %s' % endpoint) + + # Post method should not exist and/or not working + r = self.app.post( + endpoint, data=dict(path=self._ipath), + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + r = self.app.post( + endpoint + '/some', data=dict(path=self._ipath), + headers=self.__class__.auth_header) + self.assertEqual( + r.status_code, self._hcodes.HTTP_BAD_METHOD_NOT_ALLOWED) + + def test_04_DELETE_unpublish_dataobject(self): + + endpoint = self._api_uri + self._main_endpoint + log.info('*** Testing DELETE call on %s' % endpoint) + + # Publish the file which was already uploaded + r = self.app.put( + endpoint + self._ipath, + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is True + + # Unpublish the file which was previously published + r = self.app.delete( + endpoint + self._ipath, + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is False + + # Current file is now unpublished + r = self.app.get( + endpoint + self._ipath, headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + data = self.get_content(r) + assert data.get(self._main_key) is False + + # Current file cannot be accessed by anonymous + anonymous_endpoint = self._api_uri + self._register_endpoint + r = self.app.get( + anonymous_endpoint + self._ipath, + headers=self.__class__.auth_header_anonymous) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + errors = self.get_content(r, return_errors=True) + # log.pp(errors) + assert "you don't have privileges" in errors.pop() + + # Random file: cannot unpublish + r = self.app.delete( + endpoint + self._ipath + 'wrong', + headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + errors = self.get_content(r, return_errors=True) + assert 'not existing' in errors.pop().get('path') + + def tearDown(self): + + log.debug('\n### Cleaning anonymous data ###') + + # Remove the test file + endpoint = self._api_uri + self._register_endpoint + self._ipath + r = self.app.delete(endpoint, headers=self.__class__.auth_header) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + + # Recover current token id + ep = self._auth_uri + '/tokens' + r = self.app.get(ep, headers=self.__class__.auth_header_anonymous) + self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) + content = self.get_content(r) + for element in content: + if element.get('token') == self.__class__.bearer_token_anonymous: + # delete only current token + ep += '/' + element.get('id') + rdel = self.app.delete( + ep, headers=self.__class__.auth_header_anonymous) + self.assertEqual( + rdel.status_code, self._hcodes.HTTP_OK_NORESPONSE) + + # The end + super().tearDown() diff --git a/projects/eudat/backend/tests/test_registered.py b/projects/b2stage/backend/tests/test_registered.py similarity index 86% rename from projects/eudat/backend/tests/test_registered.py rename to projects/b2stage/backend/tests/test_registered.py index ce7359da..66f9753f 100644 --- a/projects/eudat/backend/tests/test_registered.py +++ b/projects/b2stage/backend/tests/test_registered.py @@ -135,6 +135,22 @@ def test_02_PUT_upload_entity(self): headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) + def get_pid(self, response): + + # data = json.loads(r.get_data(as_text=True)) + data = self.get_content(response) + + # get the only file in the response + data_object = data.pop().get(self._test_filename, {}) + # check for metadata + key = 'metadata' + self.assertIn(key, data_object) + metadata = data_object.get(key) + # check for PID + key = 'PID' + self.assertIn(key, metadata) + return metadata.get(key) + def test_03_GET_entities(self): """ Test the entity listingend retrieval: GET """ @@ -144,8 +160,8 @@ def test_03_GET_entities(self): log.info('*** Testing GET') # GET non existing entity - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename + 'NOTEXISTS') + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename + 'NOTEXISTS' r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) @@ -171,50 +187,47 @@ def test_03_GET_entities(self): self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) # Obtain entity metadata - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) # Download an entity - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.get(endpoint, data=dict(download='True'), headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) self.assertEqual(r.data, STRANGE_BSTRING) # Obtain EUDAT entity metadata (not present) - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) - data = json.loads(r.get_data(as_text=True)) - self.assertEqual( - data['Response']['data'][0][self._test_filename]['metadata']['PID'], - None) + response_pid = self.get_pid(r) + self.assertEqual(response_pid, None) # Add EUDAT metadata params = json.dumps(dict({'PID': pid})) - endpoint = (self._api_uri + self._metadata_endpoint + self._irods_path + - '/' + self._test_filename) + endpoint = self._api_uri + self._metadata_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.patch(endpoint, data=params, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) # Obtain EUDAT entity metadata - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.get(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) - data = json.loads(r.get_data(as_text=True)) - self.assertEqual( - data['Response']['data'][0][self._test_filename]['metadata']['PID'], - pid) + response_pid = self.get_pid(r) + self.assertEqual(response_pid, pid) + # Uncomment when iRODS forces checksum calculation - #self.assertIsNotNone( + # self.assertIsNotNone( # data['Response']['data'][0][self._test_filename]['metadata']['checksum']) def test_04_PATCH_rename(self): @@ -243,16 +256,16 @@ def test_04_PATCH_rename(self): # Rename file params = json.dumps(dict(newname=new_file_name)) - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + self._test_filename r = self.app.patch(endpoint, data=params, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) # Rename again with the original name params = json.dumps(dict(newname=self._test_filename)) - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + new_file_name) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + new_file_name r = self.app.patch(endpoint, data=params, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) @@ -274,8 +287,8 @@ def test_04_PATCH_rename(self): # Rename non existing file params = json.dumps(dict(newname=self._test_filename)) - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + new_file_name) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + new_file_name r = self.app.patch(endpoint, data=params, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) @@ -288,8 +301,8 @@ def test_04_PATCH_rename(self): self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) # Rename w/o passing "newname" - endpoint = (self._api_uri + self._main_endpoint + - self._irods_path + '/' + new_file_name) + endpoint = self._api_uri + self._main_endpoint + \ + self._irods_path + '/' + new_file_name r = self.app.patch(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_REQUEST) @@ -319,8 +332,8 @@ def test_05_DELETE_entities(self): self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_REQUEST) # Delete entity - endpoint = (self._api_uri + self._main_endpoint + self._irods_path + - '/' + self._test_filename) + endpoint = self._api_uri + self._main_endpoint + self._irods_path + \ + '/' + self._test_filename r = self.app.delete(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) @@ -330,8 +343,8 @@ def test_05_DELETE_entities(self): self.assertEqual(r.status_code, self._hcodes.HTTP_OK_BASIC) # Delete non existing entity - endpoint = (self._api_uri + self._main_endpoint + self._irods_path + - '/' + self._test_filename + 'x') + endpoint = self._api_uri + self._main_endpoint + self._irods_path + \ + '/' + self._test_filename + 'x' r = self.app.delete(endpoint, headers=self.__class__.auth_header) self.assertEqual(r.status_code, self._hcodes.HTTP_BAD_NOTFOUND) # HTTP_BAD_NOTFOUND is more appropriate diff --git a/projects/eudat/builds/backend/fixip.sh b/projects/b2stage/builds/backend/fixip.sh similarity index 100% rename from projects/eudat/builds/backend/fixip.sh rename to projects/b2stage/builds/backend/fixip.sh diff --git a/projects/eudat/builds/proxy/badgateway.html b/projects/b2stage/builds/proxy/badgateway.html similarity index 100% rename from projects/eudat/builds/proxy/badgateway.html rename to projects/b2stage/builds/proxy/badgateway.html diff --git a/projects/b2stage/builds/restclient/.gitkeep b/projects/b2stage/builds/restclient/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/confs/commons.yml b/projects/b2stage/confs/commons.yml similarity index 98% rename from projects/eudat/confs/commons.yml rename to projects/b2stage/confs/commons.yml index f9829d7b..0ebbab5b 100644 --- a/projects/eudat/confs/commons.yml +++ b/projects/b2stage/confs/commons.yml @@ -62,6 +62,7 @@ services: ALCHEMY_USER: ${ALCHEMY_USER} ALCHEMY_PASSWORD: ${ALCHEMY_PASSWORD} ALCHEMY_DB: ${ALCHEMY_API_DB} + ALCHEMY_POOLSIZE: ${ALCHEMY_POOLSIZE} # irods configuration IRODS_ENABLE: 1 diff --git a/projects/eudat/confs/debug.yml b/projects/b2stage/confs/debug.yml similarity index 64% rename from projects/eudat/confs/debug.yml rename to projects/b2stage/confs/debug.yml index 2f2b9623..fec9ee00 100644 --- a/projects/eudat/confs/debug.yml +++ b/projects/b2stage/confs/debug.yml @@ -19,8 +19,11 @@ services: ################# # EUDAT RELATED B2ACCESS_ENV: development + # B2ACCESS_ENV: staging IRODS_GUEST_USER: ${IRODS_GUEST_USER} IRODS_DEFAULT_ADMIN_USER: ${IRODS_DEFAULT_ADMIN_USER} + IRODS_ANONYMOUS: ${IRODS_ANONYMOUS} + MAIN_LOGIN_ENABLE: 1 ################# ports: - 8080:${FLASK_DEFAULT_PORT} @@ -30,12 +33,17 @@ services: - postgres # - mongodb volumes: + # Unit tests + - ../submodules/http-api/tests:/code/tests + - ../projects/${COMPOSE_PROJECT_NAME}/backend/tests:/code/tests/custom # Rapydo repos - - ../submodules/backend/restapi:/usr/local/lib/python3.6/dist-packages/restapi - - ../submodules/utilities/utilities:/usr/local/lib/python3.6/dist-packages/utilities + # - ../projects/${COMPOSE_PROJECT_NAME}/confs/pytest.ini:/code/pytest.ini + - ../submodules/http-api/restapi:/usr/local/lib/python3.6/dist-packages/restapi + - ../submodules/utils/utilities:/usr/local/lib/python3.6/dist-packages/utilities - ../submodules/prc/irods:/usr/local/lib/python3.6/dist-packages/irods icat: + # build: ../submodules/build-templates/b2safe # if needed to be tested environment: ACTIVATE: 1 POSTGRES_HOST: "${ALCHEMY_HOST}" @@ -53,14 +61,14 @@ services: - etcconf:/etc - irodshome:/home/${IRODS_USER} - irodsvar:/var/lib/${IRODS_USER} + #################### + ## CERTIFICATES - sharedcerts:/opt/certificates - # NOTE: this is the script that adds GSI in our irods instance - - ../submodules/builds_base/icat/extra_b2access.sh:/docker-entrypoint.d/b2access.sh - # - ../submodules/builds_base/icat/extra_b2safe.sh:/docker-entrypoint.d/b2safe.sh - # # B2ACCESS dev and prod certificates - - ../submodules/builds_base/icat/b2access_certificates:${B2ACCESS_CAS} - # # B2ACCESS dev rest-of-the-chain certificates - # - ../data/certs:/usr/local/share/ca-certificates + # adds GSI users in our irods instance + - ../submodules/build-templates/icat/extra_gsi_users.sh:/docker-entrypoint.d/gsi.sh + # B2ACCESS certificates + - ../submodules/build-templates/b2safe/extra_b2access.sh:/docker-entrypoint.d/b2access.sh + - ../submodules/build-templates/b2safe/b2access_certificates:${B2ACCESS_CAS} # # Open irods port to Outside world # ## CAREFULL: don't uncomment the two lines below if you don't know why @@ -80,6 +88,9 @@ services: depends_on: - backend + # postgres: + # volumes: + # - ../submodules/build-templates/postgres/pgs_prod.sh:/docker-entrypoint-initdb.d/init-production.conf.sh:ro swaggerui: ports: - 80:80 @@ -87,7 +98,8 @@ services: # volumes: # - restlitedb:/dbs ports: - - 81:8888 + # - 81:8888 + - 81:8080 # graphdb: # ports: diff --git a/projects/eudat/confs/development.yml b/projects/b2stage/confs/development.yml similarity index 83% rename from projects/eudat/confs/development.yml rename to projects/b2stage/confs/development.yml index 6749c51f..80903b54 100644 --- a/projects/eudat/confs/development.yml +++ b/projects/b2stage/confs/development.yml @@ -2,7 +2,7 @@ version: '3' services: -# NOTE: we are not expecting dockerized irods here +# NOTE: we are not expecting a dockerized irods here backend: environment: @@ -11,8 +11,11 @@ services: DEBUG_LEVEL: DEBUG # DEBUG_LEVEL: VERBOSE APP_MODE: development + ################# # EUDAT RELATED B2ACCESS_ENV: development + IRODS_ANONYMOUS: ${IRODS_ANONYMOUS} + ################## depends_on: - icat - postgres @@ -35,11 +38,6 @@ services: depends_on: - proxy - # graphdb: - # ports: - # - 9090:7474 - # - 7687:7687 - # networks: # app_net: # ipam: diff --git a/projects/b2stage/confs/production.yml b/projects/b2stage/confs/production.yml new file mode 100644 index 00000000..05ccafc5 --- /dev/null +++ b/projects/b2stage/confs/production.yml @@ -0,0 +1,122 @@ +version: '3' + +volumes: + sslcerts: + driver: local + +services: + + backend: + restart: on-failure:5 + # TO TEST the failure: + # 1. go inside backend + # 2. ps aux --forest + # 3. kill process 1 and the tree branches + + # restart: always + # command: sleep infinity + environment: + ################## + # Base info + ACTIVATE: 1 + APP_MODE: production + FLASK_DEBUG: 0 + DEBUG_LEVEL: INFO + NGINX_ACTIVE: "True" + + ################## + # EUDAT RELATED + ## Enable publishing + IRODS_ANONYMOUS: ${IRODS_ANONYMOUS} + ## /api/login is not enabled in production + MAIN_LOGIN_ENABLE: 0 + ## If you don't provide credentials b2access will not be enabled + # B2ACCESS_ENV: staging + B2ACCESS_ENV: production + + ################## + depends_on: + # - icat + - postgres + # talk only to proxy + expose: + - 8080 + volumes: + # B2HANDLE credentials to solve PID metadata in write mode + - ../data/b2handle:${HANDLE_CREDENTIALS_INTERNAL_PATH} + # # FIX missing IP / DNS + # - ../projects/b2stage/builds/backend/fixip.sh:/docker-entrypoint.d/fixip.sh + # # # DEBUG modifications on submodules + # # - ../submodules/http-api/restapi:/usr/local/lib/python3.6/dist-packages/restapi + + postgres: + # restart: always + restart: on-failure:5 + # Activate a production ready configuration + volumes: + # - ../submodules/build-templates/postgres/postgresql.conf:/var/lib/postgresql/data/postgresql.conf + - ../submodules/build-templates/postgres/pgs_prod.sh:/docker-entrypoint-initdb.d/init-production.conf.sh:ro + + proxy: + restart: always + environment: + ACTIVATE: 1 + DOMAIN: ${PROJECT_DOMAIN} + MODE: ${LETSENCRYPT_MODE} + volumes: + # SSL / HTTPS + - ../confs/nginx/production.conf:/etc/nginx/sites-enabled/production + - sslcerts:/etc/letsencrypt + # 502 Bad Gateway + - ../projects/${COMPOSE_PROJECT_NAME}/builds/proxy/badgateway.html:/usr/share/nginx/html/custom_502.html + ports: + - ${PROXY_DEV_PORT}:${PROXY_DEV_PORT} # 80 redirect + - ${PROXY_PROD_PORT}:${PROXY_PROD_PORT} # 443 SSL + + # if using self signed certificates + # and trying to test locally: + restclient: + environment: + + # ACTIVATE: 1 + ACTIVATE: 0 # change for debugging + + APP_HOST: --verify /tmp/certs/real/fullchain1.pem https://${PROJECT_DOMAIN} + APP_PORT: + DOMAIN: ${PROJECT_DOMAIN} + PROXY_HOST: ${PROXY_HOST} + IRODS_GUEST_USER: ${IRODS_USER} + networks: + proxy_net: + depends_on: + - proxy + volumes: + - sslcerts:/tmp/certs + + ################ + icat: + command: /docker-entrypoint.d/b2access.sh + # command: echo enabled b2access certificates + volumes: + # B2ACCESS certificates + - sharedcerts:/opt/certificates + - ../submodules/build-templates/b2safe/extra_b2access.sh:/docker-entrypoint.d/b2access.sh + - ../submodules/build-templates/b2safe/b2access_certificates:${B2ACCESS_CAS} + environment: + ## Activate this container only if B2ACCESS will be used + ## In python if B2ACCESS_SECRET is empty string -> False + ## if string with chars -> True + ACTIVATE: ${B2ACCESS_SECRET} + B2ACCESS_CAS: ${B2ACCESS_CAS} + ## Warning: fake variables; + ## icat is needed only to produce the b2access certificates + POSTGRES_HOST: null + POSTGRES_USER: null + POSTGRES_PASSWORD: null + IRODS_HOST: null + IRODS_PORT: null + IRODS_USER: null + IRODS_ZONE: null + IRODS_DB: null + IRODS_PASSWORD: null + ################ diff --git a/projects/eudat/dev-requirements.txt b/projects/b2stage/dev-requirements.txt similarity index 100% rename from projects/eudat/dev-requirements.txt rename to projects/b2stage/dev-requirements.txt diff --git a/projects/eudat/frontend/js/routing.extra.js b/projects/b2stage/frontend/js/routing.extra.js similarity index 100% rename from projects/eudat/frontend/js/routing.extra.js rename to projects/b2stage/frontend/js/routing.extra.js diff --git a/projects/eudat/frontend/libs/bower.json b/projects/b2stage/frontend/libs/bower.json similarity index 100% rename from projects/eudat/frontend/libs/bower.json rename to projects/b2stage/frontend/libs/bower.json diff --git a/projects/b2stage/frontend/templates/my_custom_template.html b/projects/b2stage/frontend/templates/my_custom_template.html new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/frontend/templates/topbar.html b/projects/b2stage/frontend/templates/topbar.html similarity index 100% rename from projects/eudat/frontend/templates/topbar.html rename to projects/b2stage/frontend/templates/topbar.html diff --git a/projects/b2stage/frontend/tests/.gitkeep b/projects/b2stage/frontend/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/projects/eudat/project_configuration.yaml b/projects/b2stage/project_configuration.yaml similarity index 83% rename from projects/eudat/project_configuration.yaml rename to projects/b2stage/project_configuration.yaml index d3766a33..4d298323 100644 --- a/projects/eudat/project_configuration.yaml +++ b/projects/b2stage/project_configuration.yaml @@ -1,8 +1,11 @@ +# ############################################### +# Copyright 2011-2017 EUDAT CDI - www.eudat.eu +# ############################################### project: title: EUDAT-B2STAGE HTTP-API server - version: v0.6.1-rc2 description: Processing files in different EUDAT domains + version: 1.0.0 ############################### ## Please CHECK VARIABLES here @@ -21,8 +24,10 @@ variables: ############################### ## B2ACCESS - B2ACCESS_ACCOUNT: httpapi - B2ACCESS_SECRET: SETYOURSECRETKEY + # Provide some existing credentials here + # if you want to provide endpoints to authenticate with B2ACCESS + B2ACCESS_ACCOUNT: + B2ACCESS_SECRET: ############################### ## LOCAL iRODS server VERSION @@ -43,8 +48,8 @@ variables: ############################### ## PRODUCTION example: iRODS server VERSION - # IRODS_HOST: eudat-dmp1 - # IRODS_ZONE: cinecaDMPZone1 + # IRODS_HOST: yourhostname + # IRODS_ZONE: productionZone1 # IRODS_HOME: home # # You MUST provide user/password credentials to iquery users informations # IRODS_AUTHSCHEME: credentials @@ -59,7 +64,7 @@ variables: ## if you don't know what you are doing ############################### - # COMPOSE_PROJECT_NAME: eudat + # COMPOSE_PROJECT_NAME: b2stage # PROJECT_DOMAIN: myserver.mydomain.dev AUTH_SERVICE: sqlalchemy # AUTH_SERVICE: mongo @@ -67,8 +72,10 @@ variables: # LETSENCRYPT_MODE: --staging LETSENCRYPT_MODE: + IRODS_ANONYMOUS: 1 IRODS_DB: ICAT IRODS_CHUNKSIZE: 1048576 + ALCHEMY_POOLSIZE: 30 # or 20 # (default is 5) B2ACCESS_CAS: /tmp/certificates/b2access # sqlalchemy fixed: @@ -104,15 +111,6 @@ variables: # default: normal_user repos: - utils: - branch: 0.5.4 - backend: - branch: 0.5.4 - build-templates: - branch: 0.5.4 - # develop: - # branch: 0.5.4 - # if: true irods-client: branch: master # python-irodsclient==0.7.0a0 online_url: https://github.com/irods/python-irodsclient.git @@ -130,8 +128,11 @@ controller: tags: eudat: all endpoints associated to EUDAT services b2access: request/refresh authorization from B2ACCESS + b2safe: proxy access to irods b2safe credentials registered: upload, download, list and delete objects pids: resolve and download data from EUDAT Persistent IDentifier + publish: allow public access to data objects + public: landing page b2safe: proxy access to irods b2safe credentials # internal: for internal testing purpose only @@ -153,15 +154,15 @@ releases: type: RC2 rapydo: 0.5.4 status: released - '0.6.2': - type: RC3 - rapydo: 0.5.5 - status: developing - '0.6.3': - type: RC4 - rapydo: null + '1.0.0': + type: stable + rapydo: 0.5.7 + status: released + '1.0.1': + type: patch + rapydo: 0.5.8 + status: development + '1.0.2': + type: development + rapydo: 0.5.8 status: todo - # '0.7.0': - # type: RC5 - # rapydo: null - # status: todo diff --git a/projects/b2stage/requirements.txt b/projects/b2stage/requirements.txt new file mode 100644 index 00000000..f40e5bdf --- /dev/null +++ b/projects/b2stage/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/rapydo/utils.git@0.5.7 +requests==2.11.1 +git+https://github.com/rapydo/do.git@0.5.7 diff --git a/projects/eudat/scripts/demo.sh b/projects/b2stage/scripts/demo.sh similarity index 100% rename from projects/eudat/scripts/demo.sh rename to projects/b2stage/scripts/demo.sh diff --git a/projects/eudat/backend/apis/extended.py b/projects/eudat/backend/apis/extended.py deleted file mode 100644 index 6a69eb1e..00000000 --- a/projects/eudat/backend/apis/extended.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -B2SAFE HTTP REST API endpoints. - -Code to implement the extended endpoints - -Note: -Endpoints list and behaviour are available at: -https://github.com/EUDAT-B2STAGE/http-api/blob/metadata_parser/docs/user/endpoints.md - -""" - -import os -from restapi.flask_ext.flask_irods.client import IrodsException -from eudat.apis.common import CURRENT_HTTPAPI_SERVER # , PRODUCTION -from eudat.apis.common.b2stage import EudatEndpoint - -from restapi.services.uploader import Uploader -from utilities import htmlcodes as hcodes -from restapi import decorators as decorate -from b2handle.handleclient import EUDATHandleClient -from b2handle.clientcredentials import PIDClientCredentials -from b2handle import handleexceptions - -from utilities.logs import get_logger - -log = get_logger(__name__) - - -############################### -# Classes - -class PIDEndpoint(Uploader, EudatEndpoint): - - @decorate.catch_error(exception=IrodsException, exception_label='B2SAFE') - def get(self, pid=None): - """ Get metadata or file from pid """ - - if pid is None: - return self.send_errors( - message='Missing PID inside URI', - code=hcodes.HTTP_BAD_REQUEST) - - # parse query parameters - self.get_input() - - ################### - # Perform B2HANDLE request: retrieve URL from handle - # TOFIX: move the PID part inside a http-api-base class - ################### - URL_value = None - CHECKSUM_value = None - - credentials_file = os.environ.get('HANDLE_CREDENTIALS') - credentials_found = False - GenericHandleError_received = False - - if credentials_file: - if os.path.isfile(credentials_file): - credentials_found = True - else: - log.critical("B2HANDLE credentials file not found %s", - credentials_file) - - if credentials_found: - cred = PIDClientCredentials.load_from_JSON(credentials_file) - client = EUDATHandleClient.instantiate_with_credentials(cred) - try: - URL_value = client.get_value_from_handle(pid, "URL") - CHECKSUM_value = client.get_value_from_handle( - pid, "EUDAT/CHECKSUM") - log.info("B2HANDLE response. URL: %s, EUDAT/CHECKSUM: %s", - URL_value, CHECKSUM_value) - except handleexceptions.HandleSyntaxError as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.warning(errorMessage) - return self.send_errors( - message=errorMessage, code=hcodes.HTTP_BAD_REQUEST) - except handleexceptions.HandleNotFoundException as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.critical(errorMessage) - return self.send_errors( - message=errorMessage, code=hcodes.HTTP_BAD_NOTFOUND) - except handleexceptions.GenericHandleError as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.warning(errorMessage) - GenericHandleError_received = True - - if ((GenericHandleError_received and credentials_found) or - not credentials_found): - log.info("Trying to resolve PID without credentials...") - client = EUDATHandleClient.instantiate_for_read_access() - try: - URL_value = client.get_value_from_handle(pid, "URL") - CHECKSUM_value = client.get_value_from_handle( - pid, "EUDAT/CHECKSUM") - log.info("B2HANDLE response. URL: %s, EUDAT/CHECKSUM: %s", - URL_value, CHECKSUM_value) - except handleexceptions.HandleSyntaxError as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.warning(errorMessage) - return self.send_errors( - message=errorMessage, code=hcodes.HTTP_BAD_REQUEST) - except handleexceptions.HandleNotFoundException as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.critical(errorMessage) - return self.send_errors( - message=errorMessage, code=hcodes.HTTP_BAD_NOTFOUND) - except handleexceptions.HandleAuthenticationError as e: - errorMessage = "B2HANDLE: %s" % str(e) - log.critical(errorMessage) - return self.send_errors( - message=errorMessage, code=hcodes.HTTP_BAD_UNAUTHORIZED) - - if URL_value is None: - return self.send_errors( - message='B2HANDLE empty URL_value returned', - code=hcodes.HTTP_BAD_NOTFOUND) - - # If downlaod is True, trigger file download - if (hasattr(self._args, 'download')): - if self._args.download and 'true' in self._args.download.lower(): - - api_url = CURRENT_HTTPAPI_SERVER - - # TODO: check download in debugging mode - # if not PRODUCTION: - # # For testing pourpose, then to be removed - # URL_value = CURRENT_HTTPAPI_SERVER + \ - # '/api/namespace/tempZone/home/guest/gettoken' - - # If local HTTP-API perform a direct download - # TOFIX: the following code can be improved - route = api_url + 'api/registered/' - # route = route.replace('http://', '') - - if (URL_value.startswith(route)): - URL_value = URL_value.replace(route, '/') - r = self.init_endpoint() - if r.errors is not None: - return self.send_errors(errors=r.errors) - URL_value = self.download_object(r, URL_value) - else: - # Perform a request to an external service? - return self.send_warnings( - {'URL': URL_value}, - errors=[ - "Data-object can't be downloaded by current " + - "HTTP-API server '%s'" % api_url - ] - ) - return URL_value - - return {'URL': URL_value, 'EUDAT/CHECKSUM': CHECKSUM_value} diff --git a/projects/eudat/backend/swagger/params_models.yaml b/projects/eudat/backend/swagger/params_models.yaml deleted file mode 100644 index ac609ae5..00000000 --- a/projects/eudat/backend/swagger/params_models.yaml +++ /dev/null @@ -1,30 +0,0 @@ - -FileUpdate: - required: - - newname - properties: - resource: - type: string - newname: - type: string - description: iRODS resource - -FileDelete: - properties: - # debugclean: - # type: string - # description: Only for debug mode - resource: - type: string - -# FileObject: -# properties: -# resource: -# type: string -# # description: yep - -# ForceObject: -# properties: -# force: -# type: boolean -# default: false diff --git a/projects/eudat/confs/b2safe.yml b/projects/eudat/confs/b2safe.yml deleted file mode 100644 index e364a642..00000000 --- a/projects/eudat/confs/b2safe.yml +++ /dev/null @@ -1,33 +0,0 @@ - -services: - - icat: - environment: - ACTIVATE: 1 - POSTGRES_HOST: "${ALCHEMY_HOST}" - POSTGRES_USER: "${ALCHEMY_USER}" - POSTGRES_PASSWORD: "${ALCHEMY_PASSWORD}" - IRODS_HOST: "${IRODS_HOST}" - IRODS_PORT: ${IRODS_PORT} - IRODS_ZONE: ${IRODS_ZONE} - IRODS_DB: "${IRODS_DB}" - IRODS_USER: ${IRODS_USER} - IRODS_PASSWORD: ${IRODS_PASSWORD} - B2ACCESS_CAS: ${B2ACCESS_CAS} - - volumes: - - etcconf:/etc - - irodshome:/home/${IRODS_USER} - - irodsvar:/var/lib/${IRODS_USER} - - sharedcerts:/opt/certificates - # NOTE: this is the script that adds GSI in our irods instance - - ../submodules/builds_base/icat/extra_b2access.sh:/docker-entrypoint.d/b2access.sh - - ../submodules/builds_base/icat/extra_b2safe.sh:/docker-entrypoint.d/b2safe.sh - # # B2ACCESS dev and prod certificates - - ../submodules/builds_base/icat/b2access_certificates:${B2ACCESS_CAS} - # # B2ACCESS dev rest-of-the-chain certificates - # - ../data/certs:/usr/local/share/ca-certificates - - # Open irods port to Outside world - ports: - - 1247:1247 diff --git a/projects/eudat/confs/irods.yml b/projects/eudat/confs/irods.yml deleted file mode 100644 index 16374c1c..00000000 --- a/projects/eudat/confs/irods.yml +++ /dev/null @@ -1,25 +0,0 @@ - -services: - - icat: - environment: - ACTIVATE: 1 - POSTGRES_HOST: "${ALCHEMY_HOST}" - POSTGRES_USER: "${ALCHEMY_USER}" - POSTGRES_PASSWORD: "${ALCHEMY_PASSWORD}" - IRODS_HOST: "${IRODS_HOST}" - IRODS_PORT: ${IRODS_PORT} - IRODS_ZONE: ${IRODS_ZONE} - IRODS_DB: "${IRODS_DB}" - IRODS_USER: ${IRODS_USER} - IRODS_PASSWORD: ${IRODS_PASSWORD} - - volumes: - - etcconf:/etc - - irodshome:/home/${IRODS_USER} - - irodsvar:/var/lib/${IRODS_USER} - - irodscerts:/opt/certificates - - # # Open irods port to Outside world - ports: - - 1247:1247 diff --git a/projects/eudat/confs/production.yml b/projects/eudat/confs/production.yml deleted file mode 100644 index d9a2dd92..00000000 --- a/projects/eudat/confs/production.yml +++ /dev/null @@ -1,67 +0,0 @@ -version: '3' - -volumes: - sslcerts: - driver: local - -services: - - backend: - # command: sleep infinity - environment: - ACTIVATE: 1 - APP_MODE: production - FLASK_DEBUG: 0 - DEBUG_LEVEL: INFO - NGINX_ACTIVE: "True" - ################## - # EUDAT RELATED - B2ACCESS_ENV: development - # B2ACCESS_ENV: production - ################## - depends_on: - # - icat - - postgres - expose: - - 8080 - volumes: - - ../data/b2handle:${HANDLE_CREDENTIALS_INTERNAL_PATH} - # # FIX missing IP / DNS - # - ../projects/eudat/builds/backend/fixip.sh:/docker-entrypoint.d/fixip.sh - # # DEBUG modifications on submodules - # - ../submodules/backend/restapi:/usr/local/lib/python3.6/dist-packages/restapi - - proxy: - environment: - ACTIVATE: 1 - DOMAIN: ${PROJECT_DOMAIN} - MODE: ${LETSENCRYPT_MODE} - volumes: - # SSL / HTTPS - - ../confs/nginx/production.conf:/etc/nginx/sites-enabled/production - - sslcerts:/etc/letsencrypt - # 502 Bad Gateway - - ../projects/${COMPOSE_PROJECT_NAME}/builds/proxy/badgateway.html:/usr/share/nginx/html/custom_502.html - ports: - - ${PROXY_DEV_PORT}:${PROXY_DEV_PORT} # 80 redirect - - ${PROXY_PROD_PORT}:${PROXY_PROD_PORT} # 443 SSL - - # if using self signed certificates - # and trying to test locally: - restclient: - environment: - - # ACTIVATE: 1 - ACTIVATE: 0 # change for debugging - - APP_HOST: --verify /tmp/certs/real/fullchain1.pem https://${PROJECT_DOMAIN} - APP_PORT: - DOMAIN: ${PROJECT_DOMAIN} - PROXY_HOST: ${PROXY_HOST} - IRODS_GUEST_USER: ${IRODS_USER} - networks: - proxy_net: - depends_on: - - proxy - volumes: - - sslcerts:/tmp/certs diff --git a/projects/eudat/requirements.txt b/projects/eudat/requirements.txt deleted file mode 100644 index e54239e5..00000000 --- a/projects/eudat/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -git+https://github.com/rapydo/utils.git@0.5.4 -requests==2.11.1 -git+https://github.com/rapydo/do.git@0.5.4