From 6e43625836cb599cc74b28af54df0c0b22ded9d3 Mon Sep 17 00:00:00 2001 From: Leonardo Rossi Date: Tue, 19 Jul 2016 15:59:49 +0200 Subject: [PATCH] docs: improve documentation * Improves documentation. (addresses #99) * Fixes suggester configuration in example app. Signed-off-by: Leonardo Rossi --- examples/app.py | 23 ++- invenio_records_rest/config.py | 189 +++++++++++++++++-- invenio_records_rest/errors.py | 2 +- invenio_records_rest/facets.py | 23 ++- invenio_records_rest/links.py | 6 +- invenio_records_rest/serializers/__init__.py | 7 + invenio_records_rest/serializers/response.py | 4 +- invenio_records_rest/sorter.py | 6 +- invenio_records_rest/utils.py | 34 +++- invenio_records_rest/views.py | 77 +++++++- 10 files changed, 328 insertions(+), 43 deletions(-) diff --git a/examples/app.py b/examples/app.py index c46285df..71fc79c2 100644 --- a/examples/app.py +++ b/examples/app.py @@ -32,11 +32,12 @@ .. code-block:: console $ cd examples - $ flask -a app.py db init - $ flask -a app.py db create - $ flask -a app.py index init - $ flask -a app.py fixtures records - $ flask -a app.py --debug run + $ export FLASK_APP=app.py + $ flask db init + $ flask db create + $ flask index init + $ flask fixtures records + $ flask run --debugger -h 0.0.0.0 -p 5000 Try to get some records: @@ -73,7 +74,7 @@ See suggestions: - $ curl -v -XGET 'http://localhost:5000/records/_suggestions?text=Reg' + $ curl -v -XGET 'http://localhost:5000/records/_suggest?title-complete=Reg' """ from __future__ import absolute_import, print_function @@ -126,6 +127,16 @@ app.config['RECORDS_REST_ENDPOINTS'] = RECORDS_REST_ENDPOINTS app.config['RECORDS_REST_ENDPOINTS']['recid']['search_index'] = index_name app.config['RECORDS_REST_ENDPOINTS']['recid']['record_class'] = MementoRecord +# Configure suggesters +app.config['RECORDS_REST_ENDPOINTS']['recid']['suggesters'] = { + "title-complete": { + "completion": { + # see testrecord-v1.0.0.json for index configuration + "field": "suggest_title", + "size": 10 + } + } +} # Sort options app.config['RECORDS_REST_SORT_OPTIONS'] = { index_name: { diff --git a/invenio_records_rest/config.py b/invenio_records_rest/config.py index e5dd41ff..f8895664 100644 --- a/invenio_records_rest/config.py +++ b/invenio_records_rest/config.py @@ -27,15 +27,13 @@ from __future__ import absolute_import, print_function from flask import request +from flask_babelex import gettext as _ from invenio_search import RecordsSearch from .facets import terms_filter from .utils import check_elasticsearch, deny_all -def _(x): - return x - RECORDS_REST_ENDPOINTS = dict( recid=dict( pid_type='recid', @@ -58,6 +56,144 @@ def _(x): max_result_window=10000, ), ) +"""Default REST endpoints loaded. + +This option can be overwritten to describe the endpoints of the different +record types. Each endpoint is in charge of managing all its CRUD operations +(GET, POST, PUT, DELETE, ...). + +The structure of the dictionary is as follows: + +.. code-block:: python + + from invenio_records_rest.query import es_search_factory + + + def search_factory(*args, **kwargs): + if not current_user.is_authenticated: + abort(401) + return es_search_factory(*args, **kwargs) + + + def permission_check_factory(): + def check_title(record, *args, **kwargs): + def can(self): + if record['title'] == 'Hello World': + return True + return type('Check', (), {'can': can})() + + + RECORDS_REST_ENDPOINTS = { + "record-pid-type": { + "create_permission_factory_imp": permission_check_factory(), + "default_media_type": "application/json", + "delete_permission_factory_imp": permission_check_factory(), + "item_route": ""/recods/"", + "links_factory_imp": ("invenio_records_rest.links:" + "default_links_factory"), + "list_route": "/records/", + "max_result_window": 10000, + "pid_fetcher": "", + "pid_minter": "", + "pid_type": "", + "read_permission_factory_imp": permission_check_factory(), + "record_class": "mypackage.api:MyRecord", + "record_loaders": { + "application/json": "mypackage.loaders:json_loader" + }, + "record_serializers": { + "application/json": "mypackage.utils:my_json_serializer" + }, + "search_class": "mypackage.utils:mysearchclass", + "search_factory_imp": search_factory(), + "search_index": "elasticsearch-index-name", + "search_serializers": { + "application/json": "mypackage.utils:my_json_search_serializer" + }, + "search_type": "elasticsearch-doc-type", + "suggesters": { + "my_url_param_to_complete": { + "completion": { + "field": "suggest_byyear_elasticsearch_field", + "size": 10, + "context": "year" + } + }, + }, + "update_permission_factory_imp": permission_check_factory(), + "use_options_view": True, + }, + } + +:param create_permission_factory_imp: Import path to factory that crcreate + permission object for a given record. + +:param default_media_type: Default media type for both records and search. + +:param delete_permission_factory_imp: Import path to factory that creates a + delete permission object for a given record. + +:param item_route: URL template for a single record. + +:param links_factory_imp: Factory for record links generation. + +:param list_route: Base URL for the records endpoint. + +:param max_result_window: Maximum total number of records retrieved from a + query. + +:param pid_type: It specifies the record pid type. It's used also to build the + endpoint name. Required. + E.g. ``url_for('record-pid-type_list')`` to point to the records list. + +:param pid_fetcher: It identifies the registered fetcher name + (see class:`invenio_pidstore:_PIDStoreState`). Required. + +:param pid_minter: It identifies the registered minter name + (see class:`invenio_pidstore:_PIDStoreState`). Required. + +:param read_permission_factory_imp: Import path to factory that creates a + read permission object for a given record. + +:param record_class: Name of the record API class. + +:param record_loaders: It contains the list of record deserializers for + supperted formats. + +:param record_serializers: It contains the list of record serializers for + supported formats. + +:param search_class: Import path or class object for the object in charge of + execute the search queries. The default search class is + class:`invenio_search.api.RecordsSearch`. + For more information about resource loading, see + class:`elasticsearch_dsl.Search`. + +:param search_factory_imp: Factory that parse query that returns a tuple with + search instance and URL arguments. + +:param search_index: Name of the search index used when searching records. + +:param search_serializers: It contains the list of records serializers for all + supported format. This configuration differ from the previous because in + this case it handle a list of records resulted by a search query instead of + a single record. + +:param search_type: Name of the search type used when searching records. + +:param suggesters: Suggester fields configuration. Any element of the + dictionary represents a suggestion field. The key is used as url query + parameter. + To have more information about suggestion configuration, you can read + suggesters section on ElasticSearch documentation. + Note: only completion suggessters are supported. + +:param update_permission_factory_imp: Import path to factory that creates a + update permission object for a given record. + +:param use_options_view: Determines if a special option view should be + installed. +""" RECORDS_REST_DEFAULT_LOADERS = { 'application/json': lambda: request.get_json(), @@ -65,9 +201,11 @@ def _(x): } """Default data loaders per request mime type. -This option can be overritten in each REST endpoint as follows:: +This option can be overritten in each REST endpoint as follows: + +.. code-block:: python - { + RECORDS_REST_ENDPOINTS = { "recid": { ... "record_loaders": { @@ -97,13 +235,17 @@ def _(x): ) """Sort options for default sorter factory. -The structure of the dictionary is as follows:: +The structure of the dictionary is as follows: - { +.. code-block:: python + + RECORDS_REST_SORT_OPTIONS = { "": { - "fields": ["", "", ...], - "title": "", - "default_order": "<default sort order in search-ui>", + "<sort-field-name>": { + "fields": ["<search_field>", "<search_field>", ...], + "title": "<title displayed to end user in search-ui>", + "default_order": "<default sort order in search-ui>", + } } } @@ -124,7 +266,19 @@ def _(x): noquery='mostrecent', ) ) -"""Default sort option per index with/without query string.""" +"""Default sort option per index with/without query string. + +The structure of the dictionary is as follows: + +.. code-block:: python + + RECORDS_REST_DEFAULT_SORT = { + "<index or index alias>": { + "query": "<default-sort-if-a-query-is-passed-from-url>", + "noquery": "<default-sort-if-no-query-in-passed-from-url>" + } + } +""" RECORDS_REST_FACETS = dict( records=dict( @@ -138,9 +292,11 @@ def _(x): ) """Facets per index for the default facets factory. -The structure of the dictionary is as follows:: +The structure of the dictionary is as follows: + +.. code-block:: python - { + RECORDS_REST_FACETS = { "<index or index alias>": { "aggs": { "<key>": <aggregation definition>, @@ -159,6 +315,13 @@ def _(x): """ RECORDS_REST_DEFAULT_CREATE_PERMISSION_FACTORY = deny_all +"""Default crete permission factory: reject any request.""" + RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY = check_elasticsearch +"""Default read permission factory: check if the record exists.""" + RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY = deny_all +"""Default update permission factory: reject any request.""" + RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY = deny_all +"""Default delete permission factory: reject any request.""" diff --git a/invenio_records_rest/errors.py b/invenio_records_rest/errors.py index 765a2283..b1db53b1 100644 --- a/invenio_records_rest/errors.py +++ b/invenio_records_rest/errors.py @@ -34,7 +34,7 @@ # Search # class MaxResultWindowRESTError(RESTException): - """Maximum number of results passed.""" + """Maximum number of passed results have been reached.""" code = 400 description = 'Maximum number of results have been reached.' diff --git a/invenio_records_rest/facets.py b/invenio_records_rest/facets.py index f2c1433c..c21ca85b 100644 --- a/invenio_records_rest/facets.py +++ b/invenio_records_rest/facets.py @@ -34,14 +34,25 @@ def terms_filter(field): - """Create a term filter.""" + """Create a term filter. + + :param field: Field name. + :returns: Function that returns the Terms query. + """ def inner(values): return Q('terms', **{field: values}) return inner def range_filter(field, start_date_math=None, end_date_math=None, **kwargs): - """Create a range filter.""" + """Create a range filter. + + :param field: Field name. + :param start_date_math: Starting date. + :param end_date_math: Ending date. + :param kwargs: Addition arguments passed to the Range query. + :returns: Function that returns the Range query. + """ def inner(values): if len(values) != 1 or values[0].count('--') != 1 or values[0] == '--': raise RESTValidationError( @@ -121,7 +132,13 @@ def _aggregations(search, definitions): def default_facets_factory(search, index): - """Add facets to query.""" + """Add a default facets to query. + + :param search: Basic search object. + :param index: Index name. + :returns: A tuple containing the new search object and a dictionary with + all fields and values used. + """ urlkwargs = MultiDict() facets = current_app.config['RECORDS_REST_FACETS'].get(index) diff --git a/invenio_records_rest/links.py b/invenio_records_rest/links.py index 94bd51de..62cffe82 100644 --- a/invenio_records_rest/links.py +++ b/invenio_records_rest/links.py @@ -28,7 +28,11 @@ def default_links_factory(pid): - """Factory for record links generation.""" + """Factory for record links generation. + + :param pid: A class:`invenio_pidstore.models.PersistentIdentifier` object. + :returns: Dictionary containing a list of useful links for the record. + """ endpoint = '.{0}_item'.format(pid.pid_type) links = dict(self=url_for(endpoint, pid_value=pid.pid_value, _external=True)) diff --git a/invenio_records_rest/serializers/__init__.py b/invenio_records_rest/serializers/__init__.py index d3d36d90..e970bc33 100644 --- a/invenio_records_rest/serializers/__init__.py +++ b/invenio_records_rest/serializers/__init__.py @@ -31,5 +31,12 @@ from .schemas.json import RecordSchemaJSONV1 json_v1 = JSONSerializer(RecordSchemaJSONV1) +"""JSON v1 serializer. + +It follows the schema class:`.schemas.json.RecordSchemaJSONV1`.""" + json_v1_response = record_responsify(json_v1, 'application/json') +"""JSON response builder that uses the JSON v1 serializer.""" + json_v1_search = search_responsify(json_v1, 'application/json') +"""JSON search response builder that uses the JSON v1 serializer.""" diff --git a/invenio_records_rest/serializers/response.py b/invenio_records_rest/serializers/response.py index e7522838..d728d9ad 100644 --- a/invenio_records_rest/serializers/response.py +++ b/invenio_records_rest/serializers/response.py @@ -37,6 +37,7 @@ def record_responsify(serializer, mimetype): :param serializer: Serializer instance. :param mimetype: MIME type of response. + :returns: Function that generates a record HTTP response. """ def view(pid, record, code=200, headers=None, links_factory=None): response = current_app.response_class( @@ -61,6 +62,7 @@ def search_responsify(serializer, mimetype): :param serializer: Serializer instance. :param mimetype: MIME type of response. + :returns: Function that generates a record HTTP response. """ def view(pid_fetcher, search_result, code=200, headers=None, links=None, item_links_factory=None): @@ -90,5 +92,5 @@ def add_link_header(response, links): if links is not None: response.headers.extend({ 'Link': ', '.join([ - '<{0}>; rel="{1}"'.format(l, r) for r, l in links.items()]) + '<{0}>; rel="{1}"'.format(l, r) for r, l in links.items()]) }) diff --git a/invenio_records_rest/sorter.py b/invenio_records_rest/sorter.py index 4de84c0b..3d0e14b9 100644 --- a/invenio_records_rest/sorter.py +++ b/invenio_records_rest/sorter.py @@ -51,6 +51,7 @@ def geolocation_sort(field_name, argument, unit, mode=None, :param unit: Distance unit (e.g. km). :param mode: Sort mode (avg, min, max). :param distance_type: Distance calculation mode. + :returns: Function that returns geolocation sort field. """ def inner(asc): locations = request.values.getlist(argument, type=str) @@ -73,7 +74,7 @@ def parse_sort_field(field_value): """Parse a URL field. :param field_value: Field value (e.g. 'key' or '-key'). - :returns: Tuple of (field, ascending). + :returns: Tuple of (field, ascending) as string and boolean. """ if field_value.startswith("-"): return (field_value[1:], False) @@ -98,6 +99,7 @@ def eval_field(field, asc): :param field: Field definition (string, dict or callable). :param asc: ``True`` if order is ascending, ``False`` if descending. + :returns: Dictionary with the sort field query. """ if isinstance(field, dict): if asc: @@ -118,7 +120,7 @@ def eval_field(field, asc): def default_sorter_factory(search, index): - """Sort a query. + """Default sort query factory. :param query: Search query. :param index: Index to search in. diff --git a/invenio_records_rest/utils.py b/invenio_records_rest/utils.py index 3f7fe3f8..d1733fd5 100644 --- a/invenio_records_rest/utils.py +++ b/invenio_records_rest/utils.py @@ -41,7 +41,12 @@ def obj_or_import_string(value, default=None): - """Import string or return object.""" + """Import string or return object. + + :params value: Import path or class object to instantiate. + :params default: Default object to return if the import fails. + :returns: The imported object. + """ if isinstance(value, six.string_types): return import_string(value) elif value: @@ -50,14 +55,20 @@ def obj_or_import_string(value, default=None): def load_or_import_from_config(key, app=None, default=None): - """Load or import value from config.""" + """Load or import value from config. + + :returns: A class:`invenio_access.permissions.DynamicPermission` object + """ app = app or current_app imp = app.config.get(key) return obj_or_import_string(imp, default=default) def allow_all(*args, **kwargs): - """Return permission that always allow an access.""" + """Return permission that always allow an access. + + :returns: A class:`invenio_access.permissions.DynamicPermission` object + """ return type('Allow', (), {'can': lambda self: True})() @@ -67,7 +78,11 @@ def deny_all(*args, **kwargs): def check_elasticsearch(record, *args, **kwargs): - """Return permission that check if the record exists in ES index.""" + """Return permission that check if the record exists in ES index. + + :params record: A record object. + :returns: A class:`invenio_access.permissions.DynamicPermission` object + """ def can(self): """Try to search for given record.""" search = request._methodview.search_class() @@ -81,13 +96,20 @@ class LazyPIDValue(object): """Lazy resolver for PID value.""" def __init__(self, resolver, value): - """Initialize with resolver and URL value.""" + """Initialize with resolver and URL value. + + :params resolver: Used to resolve PID value and return a record. + :params value: pid object + """ self.resolver = resolver self.value = value @cached_property def data(self): - """Resolve PID value and return tuple with PID and record.""" + """Resolve PID value and return tuple with PID and record. + + :returns: A tuple with the PID and the record resolved. + """ try: return self.resolver.resolve(self.value) except PIDDoesNotExistError: diff --git a/invenio_records_rest/views.py b/invenio_records_rest/views.py index 39993c49..dbf407ce 100644 --- a/invenio_records_rest/views.py +++ b/invenio_records_rest/views.py @@ -57,8 +57,10 @@ from .query import default_search_factory from .utils import obj_or_import_string + current_records_rest = LocalProxy( lambda: current_app.extensions['invenio-records-rest']) +"""Points to the current configured invenio-records-rest application.""" def elasticsearch_query_parsing_exception_handler(error): @@ -74,7 +76,11 @@ def elasticsearch_query_parsing_exception_handler(error): def create_blueprint(endpoints): - """Create Invenio-Records-REST blueprint.""" + """Create Invenio-Records-REST blueprint. + + :params: Dictionary representing the endpoints configuration. + :returns: Configured blueprint. + """ blueprint = Blueprint( 'invenio_records_rest', __name__, @@ -128,12 +134,14 @@ def create_url_rules(endpoint, list_route=None, item_route=None, """Create Werkzeug URL rules. :param endpoint: Name of endpoint. - :param list_route: record listing URL route . Required. - :param item_route: record URL route (must include ``<pid_value>`` pattern). + :param list_route: Record listing URL route. Required. + :param item_route: Record URL route (must include ``<pid_value>`` pattern). Required. :param pid_type: Persistent identifier type for endpoint. Required. - :param template: Template to render. Defaults to - ``invenio_records_ui/detail.html``. + :param pid_minter: It identifies the registered minter name. + (see class:`invenio_pidstore:_PIDStoreState`). Required. + :param pid_fetcher: It identifies the registered fetcher name + (see class:`invenio_pidstore:_PIDStoreState`). Required. :param read_permission_factory_imp: Import path to factory that creates a read permission object for a given record. :param create_permission_factory_imp: Import path to factory that creates a @@ -142,18 +150,29 @@ def create_url_rules(endpoint, list_route=None, item_route=None, update permission object for a given record. :param delete_permission_factory_imp: Import path to factory that creates a delete permission object for a given record. + :param record_class: Name of the record API class. + :param record_serializers: Serializers used for records. + :param record_loaders: It contains the list of record deserializers for + supperted formats. + :param search_class: Import path or class object for the object in charge + of execute the search queries. The default search class is + class:`invenio_search.api.RecordsSearch`. + For more information about resource loading, see + class:`elasticsearch_dsl.Search`. + :param search_serializers: Serializers used for search results. :param search_index: Name of the search index used when searching records. :param search_type: Name of the search type used when searching records. - :param record_class: Name of the record API class. - :param record_serializers: serializers used for records. - :param search_serializers: serializers used for search results. - :param default_media_type: default media type for both records and search. - :param max_result_window: maximum number of results that Elasticsearch can + :param default_media_type: Default media type for both records and search. + :param max_result_window: Maximum number of results that Elasticsearch can provide for the given search index without use of scroll. This value should correspond to Elasticsearch ``index.max_result_window`` value for the index. :param use_options_view: Determines if a special option view should be installed. + :param search_factory_imp: Factory that parse query that returns a tuple + with search instance and URL arguments. + :param links_factory_imp: Factory for record links generation. + :param suggesters: Suggester fields configuration. :returns: a list of dictionaries with can each be passed as keywords arguments to ``Blueprint.add_url_rule``. @@ -291,6 +310,10 @@ def inner(self, pid_value, *args, **kwargs): def verify_record_permission(permission_factory, record): """Check that the current user has the required permissions on record. + In case the permission check fails, an Flask abort is launched. + If the user was previously logged-in, a HTTP error 403 is returned. + Otherwise, is returned a HTTP error 401. + :param permission_factory: permission factory used to check permissions. :param record: record whose access is limited. """ @@ -451,6 +474,17 @@ def get(self, **kwargs): def post(self, **kwargs): """Create a record. + Procedure description: + #. First of all, the `create_permission_factory` permissions are + checked. + #. Then, the record is deserialized by the proper loader. + #. A second call to the `create_permission_factory` factory is done: + it differs from the previous call because this time the record is + passed as parameter. + #. A `uuid` is generated for the record and the minter is called. + #. The record class is called to create the record. + #. The HTTP response is built with the help of the item link factory. + :returns: The created record. """ if request.content_type not in self.loaders: @@ -524,6 +558,12 @@ def __init__(self, resolver=None, read_permission_factory=None, def delete(self, pid, record, **kwargs): """Delete a record. + Procedure description: + #. The record is resolved reading the pid value from the url. + #. The ETag is checked. + #. The record is deleted. + #. All PIDs are marked as DELETED. + :param pid: Persistent identifier for record. :param record: Record object. """ @@ -547,6 +587,11 @@ def delete(self, pid, record, **kwargs): def get(self, pid, record, **kwargs): """Get a record. + Procedure description: + #. The record is resolved reading the pid value from the url. + #. The ETag and If-Modifed-Since is checked. + #. The HTTP response is built with the help of the link factory. + :param pid: Persistent identifier for record. :param record: Record object. :returns: The requested record. @@ -567,6 +612,12 @@ def patch(self, pid, record, **kwargs): The data should be a JSON-patch, which will be applied to the record. + Procedure description: + #. The record is deserialized using the proper loader. + #. The ETag is checked. + #. The record is patched. + #. The HTTP response is built with the help of the link factory. + :param pid: Persistent identifier for record. :param record: Record object. :returns: The modified record. @@ -595,6 +646,12 @@ def put(self, pid, record, **kwargs): The body should be a JSON object, which will fully replace the current record metadata. + Procedure description: + #. The ETag is checked. + #. The record is updated by calling the record API `clear()`, + `update()` and then `commit()`. + #. The HTTP response is built with the help of the link factory. + :param pid: Persistent identifier for record. :param record: Record object. :returns: The modified record.