Support Collection as a resource in agent and bring on par with lates…

…t changes in core library (#153)

* provide Serverurl and apiname instead of default

* more compact way of identifying class endpoints and collection endpoints in redis layer

* minor fixes

* change the collection _id:

* remove the collection_id

* build correct url and encode values in strings

* Fix redis error

* better identify embedded resources

* get working

* put and post working

* most of the tests passing

* get proceesing extract embedded resources

* link between drone and state instance

* edges_correctly forming

* delete, put and get working

* all tests passing

* clean code

* Remove hardcoded nature.

* update readme and requirements.txt

.travis.yml

@@ -6,11 +6,10 @@ before_script:
   - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph:2.0-edge
 
 python:
-  - "3.5"
-  - "3.5-dev"  # 3.5 development branch
   - "3.6"
   - "3.6-dev"  # 3.6 development branch
-  - "3.7-dev"
+  - "3.7"
+  - "3.7-dev" # 3.7 development branch
 install:
   - pip install -r requirements.txt --no-cache
 

README.md

@@ -64,27 +64,85 @@ The agent supports GET, PUT, POST or DELETE:
 
 **To GET** a existing resource you should:
 ```
-agent.get("http://localhost:8080/serverapi/<CollectionType>/<Resource-ID>")
+agent.get("http://localhost:8080/serverapi/<ResourceType>/<Resource-ID>")
 agent.get("http://localhost:8080/serverapi/<CollectionType>/")
+agent.get("http://localhost:8080/serverapi/<CollectionType>/<Collection-ID>")
 ```
 
-**To PUT** a new resource you should:
+**To PUT** a new resource say on a Drone endpoint, you should:
 ```
-new_resource = {"@type": "Drone", "name": "Drone 1", "model": "Model S", ...}
-agent.put("http://localhost:8080/serverapi/<CollectionType>/", new_resource)
+new_resource = {
+    "@type": "Drone",
+    "DroneState": {
+        "@type": "State",
+        "Battery": "50%",
+        "Direction": "N",
+        "Position": "50.34",
+        "SensorStatus": "Active",
+        "Speed": "100"
+    },
+    "MaxSpeed": "500",
+    "Sensor": "Active",
+    "model": "Drone_1",
+    "name": "Drone1"
+}
+agent.put("http://localhost:8080/serverapi/Drone/", new_resource)
 ```
 
 **To UPDATE** a resource you should:
 ```
 existing_resource["name"] = "Updated Name"
-agent.post("http://localhost:8080/serverapi/<CollectionType>/<Resource-ID>", existing_resource)
+agent.post("http://localhost:8080/serverapi/<ResourceType>/<Resource-ID>", existing_resource)
 ```
 
 **To DELETE** a resource you should:
 ```
-agent.delete("http://localhost:8080/serverapi/<CollectionType>/<Resource-ID>")
+agent.delete("http://localhost:8080/serverapi/<ResourceType>/<Resource-ID>")
 ```
+**To ADD** members in collection:
+```
+request_body = {
+    "@type": "<CollectionType>",
+    "members": [
+        {
+            "@id": "<ResourceID>",
+            "@type": "<ResourceType>"
+        },
+        {
+            "@id": "<ResourceID>",
+            "@type": "<ResourceType>"
+        },
+    ]
+}
+agent.put("http://localhost:8080/serverapi/<CollectionType>", request_body)
+```
+NOTE: \<ResourceType\> can be different in given request body. 
 
+**TO GET** members of specific Collection:
+```
+agent.get("http://localhost:8080/serverapi/<CollectionType>/<CollectionID>")
+```
+**TO UPDATE** members of specific collection:
+```
+updated_collection = {
+    "@type": "<CollectionType>",
+    "members": [
+        {
+            "@id": "<ResourceID>",
+            "@type": "<ResourceType>"
+        },
+        {
+            "@id": "<ResourceID>",
+            "@type": "<ResourceType>"
+        },
+    ]
+}
+agent.post("http://localhost:8080/serverapi/<CollectionType>/<CollectionID>",updated_collection )
+```
+**TO DELETE** members of specific Collection:
+```
+agent.delete("http://localhost:8080/serverapi/<CollectionType>/<CollectionID>")
+```
 More than that, Agent extends Session from https://2.python-requests.org/en/master/api/#request-sessions, so all methods like auth, cookies, headers and so on can also be used.
 
 ### Natural-language-like Command Line Tool

hydra_agent/agent.py

@@ -1,16 +1,17 @@
 import logging
 import sys
 import socketio
+from urllib.parse import urlparse
 from hydra_agent.redis_core.redis_proxy import RedisProxy
 from hydra_agent.redis_core.graphutils_operations import GraphOperations
 from hydra_agent.redis_core.graph_init import InitialGraph
 from hydra_python_core import doc_maker
+from hydra_python_core.doc_writer import HydraDoc
 from typing import Union, Tuple
 from requests import Session
 import json
 from hydra_agent.helpers import expand_template
 from hydra_agent.collection_paginator import Paginator
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__file__)
 
@@ -20,33 +21,33 @@ class Agent(Session, socketio.ClientNamespace, socketio.Client):
     CRUD interface - to query hydrus
     """
 
-    def __init__(self, entrypoint_url: str, sync: bool = True) -> None:
+    def __init__(self, entrypoint_url: str, namespace: str = '/sync') -> None:
         """Initialize the Agent
         :param entrypoint_url: Entrypoint URL for the hydrus server
         :param namespace: Namespace endpoint to listen for updates
         :return: None
         """
         self.entrypoint_url = entrypoint_url.strip().rstrip('/')
-        self.sync = sync
+        url_parse = urlparse(entrypoint_url)
+        self.entrypoint = url_parse.scheme + "://" + url_parse.netloc
+        self.api_name = url_parse.path.rstrip('/')
+        self.redis_proxy = RedisProxy()
+        self.redis_connection = self.redis_proxy.get_connection()
         Session.__init__(self)
         self.fetch_apidoc()
-        if sync:
-            self.redis_proxy = RedisProxy()
-            self.redis_connection = self.redis_proxy.get_connection()
-            namespace = '/sync'
-            self.initialize_graph()
-            self.graph_operations = GraphOperations(self.entrypoint_url,
-                                                    self.api_doc,
-                                                    self.redis_proxy)
-            # Declaring Socket Rules and instantiating Synchronization Socket
-            socketio.ClientNamespace.__init__(self, namespace)
-            socketio.Client.__init__(self, logger=True)
-            socketio.Client.register_namespace(self, self)
-            socketio.Client.connect(self, self.entrypoint_url,
-                                    namespaces=namespace)
-            self.last_job_id = ""
+        self.initialize_graph()
+        self.graph_operations = GraphOperations(self.entrypoint_url,
+                                                self.api_doc,
+                                                self.redis_proxy)
+        # Declaring Socket Rules and instantiation Synchronization Socket
+        socketio.ClientNamespace.__init__(self, namespace)
+        socketio.Client.__init__(self, logger=True)
+        socketio.Client.register_namespace(self, self)
+        socketio.Client.connect(self, self.entrypoint_url,
+                                namespaces=namespace)
+        self.last_job_id = ""
 
-    def fetch_apidoc(self) -> dict:
+    def fetch_apidoc(self) -> HydraDoc:
         """Fetches API DOC from Link Header by checking the hydra apiDoc
         relation and passes the obtained JSON-LD to doc_maker module of
         hydra_python_core to return HydraDoc which is used by the agent.
@@ -56,12 +57,11 @@ def fetch_apidoc(self) -> dict:
             res = super().get(self.entrypoint_url)
             api_doc_url = res.links['http://www.w3.org/ns/hydra/core#apiDocumentation']['url']
             jsonld_api_doc = super().get(api_doc_url).json()
-            self.api_doc = doc_maker.create_doc(jsonld_api_doc)
+            self.api_doc = doc_maker.create_doc(jsonld_api_doc, self.entrypoint, self.api_name )
             return self.api_doc
         except:
-            print("Error parsing your API Documentation. Please make sure Link header \
-                contains the URL of APIDOC with rel http://www.w3.org/ns/hydra/core#apiDocumentation")
-            raise
+            print("Error parsing your API Documentation")
+            raise SyntaxError
 
     def initialize_graph(self) -> None:
         """Initialize the Graph on Redis based on ApiDoc
@@ -83,7 +83,7 @@ def get(self, url: str = None, resource_type: str = None,
         :param filters: filters to apply when searching, resources properties
         :param cached_limit : Minimum amount of resources to be fetched
         :param follow_partial_links: If set to True, Paginator can go through pages.
-        :return: Dict when one object or a list when multiple targerted objects
+        :return: Dict when one object or a list when multiple targeted objects
         :return: Iterator when param follow_partial_links is set to true
                     Iterator will be returned.
                     Usage:
@@ -102,36 +102,32 @@ def get(self, url: str = None, resource_type: str = None,
                     To Jump:
                     paginator.jump_to_page(2) 
         """
-        if self.sync:
-            redis_response = self.graph_operations.get_resource(url, resource_type, filters)
-            if redis_response:
-                if type(redis_response) is dict:
-                    return redis_response
-                elif len(redis_response) >= cached_limit:
-                    return redis_response
+        redis_response = self.graph_operations.get_resource(url, self.graph, resource_type,
+                                                            filters)
+        if redis_response:
+            if type(redis_response) is dict:
+                return redis_response
+            elif len(redis_response) >= cached_limit:
+                return redis_response
+        if url:
 
-        # If querying with resource type build url
-        # This can be more stable when adding Manages Block
-        # More on: https://www.hydra-cg.com/spec/latest/core/#manages-block
-        if resource_type:
-            url = self.entrypoint_url + "/" + resource_type + "Collection"
-            response = super().get(url, params=filters)
-        else:
             if not bool(filters):
                 response = super().get(url)
             else:
-                response_body = super().get(url)
+                response_body = super().get(url, filters)
                 # filters can be simple dict or a json-ld
-                templated_url = expand_template(
+                try:
+                    templated_url = expand_template(
                     url, response_body.json(), filters)
-                response = super().get(templated_url)
+                    response = super().get(templated_url)
+                except KeyError:
+                    response = response_body
 
         if response.status_code == 200:
             # Graph_operations returns the embedded resources if finding any
-            if self.sync:
-                embedded_resources = \
-                    self.graph_operations.get_processing(url, response.json())
-                self.process_embedded(embedded_resources)
+            embedded_resources = \
+                self.graph_operations.get_processing(url, response.json())
+            self.process_embedded(embedded_resources)
             if response.json()['@type'] in self.api_doc.parsed_classes:
                 return response.json()
             else:
@@ -153,9 +149,9 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]:
         if response.status_code == 201:
             url = response.headers['Location']
             # Graph_operations returns the embedded resources if finding any
-            if self.sync:
-                embedded_resources = self.graph_operations.put_processing(url, new_object)
-                self.process_embedded(embedded_resources)
+            full_resource = super().get(url)
+            embedded_resources = self.graph_operations.put_processing(url, full_resource.json())
+            self.process_embedded(embedded_resources)
             return response.json(), url
         else:
             return response.text, ""
@@ -170,10 +166,9 @@ def post(self, url: str, updated_object: dict) -> dict:
 
         if response.status_code == 200:
             # Graph_operations returns the embedded resources if finding any
-            if self.sync:
-                embedded_resources = \
-                    self.graph_operations.post_processing(url, updated_object)
-                self.process_embedded(embedded_resources)
+            embedded_resources = \
+                self.graph_operations.post_processing(url, updated_object)
+            self.process_embedded(embedded_resources)
             return response.json()
         else:
             return response.text
@@ -184,10 +179,8 @@ def delete(self, url: str) -> dict:
         :return: Dict with server's response
         """
         response = super().delete(url)
-
         if response.status_code == 200:
-            if self.sync:
-                self.graph_operations.delete_processing(url)
+            self.graph_operations.delete_processing(url)
             return response.json()
         else:
             return response.text
@@ -203,7 +196,9 @@ def process_embedded(self, embedded_resources: list) -> None:
             self.graph_operations.link_resources(
                 embedded_resource['parent_id'],
                 embedded_resource['parent_type'],
-                embedded_resource['embedded_url'])
+                embedded_resource['embedded_url'],
+                embedded_resource['embedded_type'],
+                self.graph)
 
     # Below are the functions that are responsible to process Socket Events
     def on_connect(self, data: dict = None) -> None:

hydra_agent/redis_core/classes_objects.py

@@ -11,6 +11,7 @@ class RequestError(Exception):
     """A class for client-side exceptions"""
     pass
 
+
 class ClassEndpoints:
     """Contains all the classes endpoint and the objects"""
 

hydra_agent/redis_core/graph_init.py

@@ -9,12 +9,12 @@
 
 
 class InitialGraph:
-
-
-    def get_apistructure(self,entrypoint_node, api_doc):
-        """ It breaks the endpoint into two parts collection and classes"""
+    def __init__(self):
         self.collection_endpoints = {}
         self.class_endpoints = {}
+
+    def get_apistructure(self, entrypoint_node, api_doc):
+        """ It breaks the endpoint into two parts collection and classes"""
         print("split entrypoint into 2 types of endpoints collection and classes")
         for support_property in api_doc.entrypoint.entrypoint.supportedProperty:
             if isinstance(
@@ -27,19 +27,18 @@ def get_apistructure(self,entrypoint_node, api_doc):
                     doc_writer.EntryPointCollection):
                 self.collection_endpoints[support_property.name] = support_property.id_
 
-        if len(self.class_endpoints.keys())>0:
+        if len(self.class_endpoints.keys()) > 0:
             clas = ClassEndpoints(self.redis_graph, self.class_endpoints)
             clas.endpointclasses(entrypoint_node, api_doc, self.url)
 
-        if len(self.collection_endpoints.keys())>0:
+        if len(self.collection_endpoints.keys()) > 0:
             coll = CollectionEndpoints(self.redis_graph, self.class_endpoints)
             coll.endpointCollection(
                 self.collection_endpoints,
                 entrypoint_node,
                 api_doc,
                 self.url)
 
-
     def get_endpoints(self,api_doc, redis_connection):
         """Create node for entrypoint"""
         print("creating entrypoint node")
@@ -55,8 +54,6 @@ def get_endpoints(self,api_doc, redis_connection):
         self.redis_graph.add_node(entrypoint_node)
         return self.get_apistructure(entrypoint_node, api_doc)
 
-
-
     def main(self,new_url,api_doc,check_commit):
         redis_connection = RedisProxy()
         redis_con = redis_connection.get_connection()

hydra_agent/redis_core/graphutils.py

@@ -34,15 +34,13 @@ def read(self, match: str, ret: str,
         if where:
             query += " WHERE(p.{})".format(where)
         query += " RETURN p{}".format(ret)
-
         query_result = self.redis_graph.query(query)
 
         # Processing Redis-set response format
         query_result = self.process_result(query_result)
 
-        if not query_result:
-            query_result = None
-
+        # if not query_result:
+        #     query_result = None
         return query_result
 
     def update(self, match: str, set: str, where: Optional[str]=None) -> list:
@@ -53,11 +51,10 @@ def update(self, match: str, set: str, where: Optional[str]=None) -> list:
         :param where: Used to filter results, not mandatory.
         :return: Query results
         """
-        query = "MATCH(p:{})".format(match)
+        query = "MATCH(p{})".format(match)
         if where is not None:
             query += " WHERE(p.{})".format(where)
         query += " SET p.{}".format(set)
-
         return self.redis_connection.execute_command("GRAPH.QUERY",
                                                      self.graph_name,
                                                      query)