changes for v0.6

.github/workflows/pytest.yml

@@ -29,6 +29,9 @@ jobs:
         echo "$CA_CERT" > ca-cert.pem
         echo "$CA_KEY" > ca-key.pem
         echo "$CA_SIGNING_KEY" > ca-signing-key.pem
+    - name: Start the proxy
+      run: |
+        poetry run python ontologytimemachine/custom_proxy.py &    
     - name: Test with pytest
       run: |
-        poetry run pytest
+        poetry run pytest -v

Dockerfile

@@ -21,6 +21,4 @@ RUN pip install poetry==$POETRY_VERSION
 RUN poetry config virtualenvs.create false
 RUN poetry install --no-dev && rm pyproject.toml
 
-
-CMD python3 -m proxy --ca-key-file ca-key.pem --ca-cert-file ca-cert.pem --ca-signing-key-file ca-signing-key.pem --hostname 0.0.0.0 --port $PORT --plugins ontologytimemachine.custom_proxy.OntologyTimeMachinePlugin
-
+ENTRYPOINT ["python3", "ontologytimemachine/custom_proxy.py"]

README.md

@@ -35,3 +35,6 @@ cp ca-signing-key.pem ~/ontology-time-machine/ca-signing-key.pem
 ### Not working: 
 - curl -x http://0.0.0.0:8899 -H "Accept: text/turtle" --cacert ca-cert.pem http://ontologi.es/days#
 
+
+
+python3 -m proxy --ca-key-file ca-key.pem --ca-cert-file ca-cert.pem --ca-signing-key-file ca-signing-key.pem --hostname IP --port 8899 --plugins ontologytimemachine.custom_proxy.OntologyTimeMachinePlugin --ontoFormat ntriples --ontoVersion originalFailoverLive --ontoPrecedence enforcedPriority

ontologytimemachine/custom_proxy.py

@@ -1,11 +1,12 @@
 from proxy.http.proxy import HttpProxyBasePlugin
-from proxy.http.parser import HttpParser, httpParserTypes
+from proxy.http.parser import HttpParser
 from proxy.common.utils import build_http_response
-from proxy.http.methods import HttpMethods
-from ontologytimemachine.utils.utils import proxy_logic, parse_arguments
-from ontologytimemachine.utils.utils import check_if_archivo_ontology_requested
+from ontologytimemachine.utils.utils import parse_arguments
 from ontologytimemachine.utils.mock_responses import mock_response_403
-from requests.exceptions import SSLError, Timeout, ConnectionError, RequestException
+from ontologytimemachine.proxy_wrapper import HttpRequestWrapper
+from ontologytimemachine.utils.proxy_logic import proxy_logic, is_ontology_request_only_ontology
+from ontologytimemachine.utils.proxy_logic import is_archivo_ontology_request
+from ontologytimemachine.utils.proxy_logic import if_intercept_host
 from http.client import responses
 import proxy
 import sys
@@ -15,69 +16,69 @@
 IP = '0.0.0.0'
 PORT = '8899'
 
+config = ({'format': 'turtle', 'precedence': 'enforcedPriority', 'patchAcceptUpstream': False}, 'originalFailoverLiveLatest', False, 'all', False, True, None, None)
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
 class OntologyTimeMachinePlugin(HttpProxyBasePlugin):
     def __init__(self, *args, **kwargs):
+        logger.info('Init')
         super().__init__(*args, **kwargs)
-        (self.ontoFormat, self.ontoVersion, self.only_ontologies,
-         self.https_intercept, self.inspect_redirects, self.forward_headers,
-         self.subject_binary_search_threshold) = parse_arguments()
-
+        (self.ontoFormat, self.ontoVersion, self.restrictedAccess,
+         self.httpsInterception, self.disableRemovingRedirects, 
+         self.forward_headers, self.timestamp, self.manifest) = config
 
     def before_upstream_connection(self, request: HttpParser):
         logger.info('Before upstream connection hook')
         logger.info(f'Request method: {request.method} - Request host: {request.host} - Request path: {request.path} - Request headers: {request.headers}')
+        wrapped_request = HttpRequestWrapper(request)
 
-        if request.method == b'CONNECT':
-            logger.info(f'HTTPS interception mode: {self.https_intercept}')
+        if wrapped_request.is_connect_request():
+            logger.info(f'HTTPS interception mode: {self.httpsInterception}')
             # Only intercept if interception is enabled
-            if self.https_intercept in ['all', 'archivo']:
+            # Move this to the utils
+            if if_intercept_host(self.httpsInterception):
+                logger.info('HTTPS interception is on, forwardig the request')
                 return request
             else:
+                logger.info('HTTPS interception is turned off')
                 return None
-
 
-        ontology_request = check_if_archivo_ontology_requested(request)
         # If only ontology mode, return None in all other cases
-        if self.only_ontologies and not ontology_request:
+        if is_ontology_request_only_ontology(wrapped_request, self.restrictedAccess):
             logger.warning('Request denied: not an ontology request and only ontologies mode is enabled')
             self.queue_response(mock_response_403)
             return None
 
-        if ontology_request:
+        if is_archivo_ontology_request(wrapped_request):
             logger.debug('The request is for an ontology')
-            response = proxy_logic(request, self.ontoFormat, self.ontoVersion)
+            response = proxy_logic(wrapped_request, self.ontoFormat, self.ontoVersion, self.disableRemovingRedirects, self.timestamp, self.manifest)
             self.queue_response(response)
             return None
         return request
 
-
     def handle_client_request(self, request: HttpParser):
         logger.info('Handle client request hook')
         logger.info(f'Request method: {request.method} - Request host: {request.host} - Request path: {request.path} - Request headers: {request.headers}')
 
-        logger.debug(request.method)
-        if request.method == b'CONNECT':
+        wrapped_request = HttpRequestWrapper(request)
+        if wrapped_request.is_connect_request():
             return request
 
-        ontology_request = check_if_archivo_ontology_requested(request)
-        if not ontology_request:
+        is_ontology_request = is_archivo_ontology_request(wrapped_request)
+        if not is_ontology_request:
             logger.info('The requested IRI is not part of DBpedia Archivo')
             return request   
 
-        response = proxy_logic(request, self.ontoFormat, self.ontoVersion)
+        response = proxy_logic(wrapped_request, self.ontoFormat, self.ontoVersion, self.disableRemovingRedirects, self.timestamp, self.manifest)
         self.queue_response(response)
 
         return None
 
-
     def handle_upstream_chunk(self, chunk: memoryview):
         return chunk
 
-
     def queue_response(self, response):
         self.client.queue(
             build_http_response(
@@ -93,15 +94,23 @@ def queue_response(self, response):
 
 if __name__ == '__main__':
 
-    sys.argv += [
-        '--ca-key-file', 'ca-key.pem',
-        '--ca-cert-file', 'ca-cert.pem',
-        '--ca-signing-key-file', 'ca-signing-key.pem',
-    ]
+    config = parse_arguments()
+
+    sys.argv = [sys.argv[0]]
+
+    # check it https interception is enabled
+    if config[3] != 'none':
+        sys.argv += [
+            '--ca-key-file', 'ca-key.pem',
+            '--ca-cert-file', 'ca-cert.pem',
+            '--ca-signing-key-file', 'ca-signing-key.pem',
+        ]
+
     sys.argv += [
         '--hostname', IP,
         '--port', PORT,
         '--plugins', __name__ + '.OntologyTimeMachinePlugin'
     ]
+
     logger.info("Starting OntologyTimeMachineProxy server...")
     proxy.main()

ontologytimemachine/proxy_wrapper.py

@@ -0,0 +1,101 @@
+from abc import ABC, abstractmethod
+from proxy.http.parser import HttpParser
+import logging
+
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+class AbstractRequestWrapper(ABC):
+    def __init__(self, request):
+        self.request = request
+
+    @abstractmethod
+    def is_get_request(self) -> bool:
+        pass
+
+    @abstractmethod
+    def is_connect_request(self) -> bool:
+        pass
+
+    @abstractmethod
+    def is_head_request(self) -> bool:
+        pass
+
+    @abstractmethod
+    def is_https_request(self) -> bool:
+        pass
+
+    @abstractmethod
+    def get_request(self):
+        pass
+
+    @abstractmethod
+    def get_request_headers(self):
+        pass
+
+    @abstractmethod
+    def get_request_accept_header(self):
+        pass
+
+    @abstractmethod
+    def set_request_accept_header(self, mime_type):
+        pass
+
+    @abstractmethod
+    def get_ontology_from_request(self):
+        pass
+
+
+class HttpRequestWrapper(AbstractRequestWrapper):
+    def __init__(self, request: HttpParser):
+        super().__init__(request)
+
+    def is_get_request(self) -> bool:
+        return self.request.method == b'GET'
+
+    def is_connect_request(self):
+        return self.request.method == b'CONNECT'
+
+    def is_head_request(self):
+        return self.request.method == b'HEAD'
+
+    def is_https_request(self):
+        return self.request.method == b'CONNECT' or self.request.headers.get(b'Host', b'').startswith(b'https')
+
+    def get_request(self):
+        return self.request
+
+    def get_request_headers(self):
+        headers = {}
+        for k, v in self.request.headers.items():
+            headers[v[0].decode('utf-8')] = v[1].decode('utf-8')
+        return headers
+
+    def get_request_accept_header(self):
+        logger.info('Wrapper - get_request_accept_header')
+        return self.request.headers[b'accept'][1].decode('utf-8')
+
+    def set_request_accept_header(self, mime_type):
+        self.request.headers[b'accept'] = (b'Accept', mime_type.encode('utf-8'))
+        logger.info(f'Accept header set to: {self.request.headers[b"accept"][1]}')
+
+    def get_ontology_from_request(self):
+        logger.info('Get ontology from request')
+        print(f'Request protocol: {self.request.protocol}')
+        print(f'Request host: {self.request.host}')
+        print(f'Request _url: {self.request._url}')
+        print(f'Request path: {self.request.path}')
+        if (self.request.method == b'GET' or self.request.method == b'HEAD') and not self.request.host:
+            for k, v in self.request.headers.items():
+                if v[0].decode('utf-8') == 'Host':
+                    host = v[1].decode('utf-8')
+                    path = self.request.path.decode('utf-8')
+            ontology = 'https://' + host + path
+        else:
+            host = self.request.host.decode('utf-8')
+            path = self.request.path.decode('utf-8')
+            ontology = str(self.request._url)
+        logger.info(f'Ontology: {ontology}')
+        return ontology, host, path

ontologytimemachine/utils/dependency.ttl

@@ -0,0 +1,25 @@
+@prefix ex-version: <https://example.org/versioning/>
+<https://example.org/ontology/> owl:imports <http://xmlns.com/foaf/spec/>, <http://purl.org/dc/terms/> ;
+    ex-version:current <https://example.org/ontology/2024-01-24> ;
+    ex-version:version
+    <https://example.org/ontology/2024-01-24> ,
+    [
+         ex-version:snapshot <https://databus.dbpedia.org/ontologies/w3.org/2020--example/2023.02.11-215415> ;
+         ex-version:file <https://archivo.dbpedia.org/download?o=https%3A//example.org/ontology/&f=ttl&v=2023.02.01-215415> ;
+         ex-version:dependency <http://xmlns.com/foaf/spec/20100101.html>, <https://databus.dbpedia.org/ontologies/w3.org/2020--dct/2020.05.23-215415> ;
+    ] .
+
+<https://example.org/ontology/2024-01-24>
+    ex-version:snapshot <https://databus.dbpedia.org/ontologies/w3.org/2020--example/2024.01.24-215415> ;
+    ex-version:file <https://archivo.dbpedia.org/download?o=https%3A//example.org/ontology/&f=ttl&v=2024.01.24-215415> ;
+    ex-version:dependency <http://xmlns.com/foaf/spec/20140114.html>, <https://databus.dbpedia.org/ontologies/w3.org/2020--dct/2020.05.23-215415> ;
+]
+
+<http://xmlns.com/foaf/spec/20100101.html> ex-version:snapshot <http://xmlns.com/foaf/spec/20100101.html> ;
+    ex-version:file <http://xmlns.com/foaf/spec/20100101.rdf> .
+
+<http://xmlns.com/foaf/spec/20140114.html> ex-version:snapshot <http://xmlns.com/foaf/spec/20140114.html> ;
+    ex-version:file <http://xmlns.com/foaf/spec/20140114.rdf> .
+
+<https://databus.dbpedia.org/ontologies/w3.org/2020--dct/2020.05.23-215415> ex-version:snapshot <https://databus.dbpedia.org/ontologies/w3.org/2020--dct/2020.05.23-215415> ;
+    ex-version:file <https://archivo.dbpedia.org/download?o=http%3A//purl.org/dc/terms/&f=ttl&v=2020.05.23-215415> .