From 54b249d520a3623d2dee9726659b45d047f35cdc Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 14:28:49 -0400
Subject: [PATCH 01/10] feat: Add URL-based file loading and associated tests

---
 dbt_loom/config.py             | 20 +++++++++--
 dbt_loom/manifests.py          | 65 ++++++++++++++++++++++++++++++----
 tests/test_manifest_loaders.py | 41 +++++++++++++++++++++
 3 files changed, 118 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_manifest_loaders.py

diff --git a/dbt_loom/config.py b/dbt_loom/config.py
index 5591447..d15e8a6 100644
--- a/dbt_loom/config.py
+++ b/dbt_loom/config.py
@@ -1,8 +1,9 @@
 from enum import Enum
 from pathlib import Path
+import re
 from typing import List, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, AnyUrl, validator
 
 from dbt_loom.clients.az_blob import AzureReferenceConfig
 from dbt_loom.clients.dbt_cloud import DbtCloudReferenceConfig
@@ -23,7 +24,22 @@ class ManifestReferenceType(str, Enum):
 class FileReferenceConfig(BaseModel):
     """Configuration for a file reference"""
 
-    path: Path
+    path: AnyUrl
+
+    @validator("path", pre=True, always=True)
+    def default_path(cls, v, values):
+        """
+        Check if the provided path is a valid URL. If not, convert it into an
+        absolute file path.
+        """
+
+        if isinstance(v, AnyUrl):
+            return v
+
+        if bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", v)):
+            return v
+
+        return "file://" + str(Path(v).absolute())
 
 
 class ManifestReference(BaseModel):
diff --git a/dbt_loom/manifests.py b/dbt_loom/manifests.py
index e7a3b0a..a3c5f01 100644
--- a/dbt_loom/manifests.py
+++ b/dbt_loom/manifests.py
@@ -1,9 +1,12 @@
 import datetime
+from io import BytesIO
 import json
 import gzip
+from pathlib import Path
 from typing import Dict, List, Optional
 
 from pydantic import BaseModel, Field, validator
+import requests
 
 try:
     from dbt.artifacts.resources.types import NodeType
@@ -84,6 +87,14 @@ def dump(self) -> Dict:
         return self.dict(exclude=exclude_set)
 
 
+class UnknownManifestPathType(Exception):
+    """Raised when the ManifestLoader receives a FileReferenceConfig with a path that does not have a known URL scheme."""
+
+
+class InvalidManifestPath(Exception):
+    """Raised when the ManifestLoader receives a FileReferenceConfig with an invalid path."""
+
+
 class ManifestLoader:
     def __init__(self):
         self.loading_functions = {
@@ -94,17 +105,59 @@ def __init__(self):
             ManifestReferenceType.azure: self.load_from_azure,
         }
 
+    @staticmethod
+    def load_from_path(config: FileReferenceConfig) -> Dict:
+        """
+        Load a manifest dictionary based on a FileReferenceConfig. This config's
+        path can point to either a local file or a URL to a remote location.
+        """
+
+        if config.path.scheme in ("http", "https"):
+            return ManifestLoader.load_from_http(config)
+
+        if config.path.scheme in ("file"):
+            return ManifestLoader.load_from_local_filesystem(config)
+
+        raise UnknownManifestPathType()
+
     @staticmethod
     def load_from_local_filesystem(config: FileReferenceConfig) -> Dict:
         """Load a manifest dictionary from a local file"""
-        if not config.path.exists():
-            raise LoomConfigurationError(f"The path `{config.path}` does not exist.")
 
-        if config.path.suffix == ".gz":
-            with gzip.open(config.path, "rt") as file:
+        if not config.path.path:
+            raise InvalidManifestPath()
+
+        file_path = Path(config.path.path)
+
+        if not file_path.exists():
+            raise LoomConfigurationError(f"The path `{file_path}` does not exist.")
+
+        if file_path.suffix == ".gz":
+            with gzip.open(file_path, "rt") as file:
                 return json.load(file)
-        else:
-            return json.load(open(config.path))
+
+        return json.load(open(file_path))
+
+    @staticmethod
+    def load_from_http(config: FileReferenceConfig) -> Dict:
+        """Load a manifest dictionary from a local file"""
+
+        if not config.path.path:
+            raise InvalidManifestPath()
+
+        response = requests.get(config.path.path, stream=True)
+        response.raise_for_status()  # Check for request errors
+
+        # Check for compression on the file. If compressed, store it in a buffer
+        # and decompress it.
+        if (
+            config.path.path.endswith(".gz")
+            or response.headers.get("Content-Encoding") == "gzip"
+        ):
+            with gzip.GzipFile(fileobj=BytesIO(response.content)) as gz_file:
+                return json.load(gz_file)
+
+        return response.json()
 
     @staticmethod
     def load_from_dbt_cloud(config: DbtCloudReferenceConfig) -> Dict:
diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
new file mode 100644
index 0000000..13f3515
--- /dev/null
+++ b/tests/test_manifest_loaders.py
@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+from pydantic import AnyUrl
+from dbt_loom.config import FileReferenceConfig
+from dbt_loom.manifests import ManifestLoader
+
+
+def test_load_from_local_filesystem_pass():
+    """Test that ManifestLoader can load a local JSON file."""
+
+    example_content = {"foo": "bar"}
+    path = Path("example.json")
+
+    with open(path, "w") as file:
+        json.dump(example_content, file)
+
+    file_config = FileReferenceConfig(
+        path=AnyUrl("file://" + str(Path(path).absolute()))
+    )
+
+    output = ManifestLoader.load_from_local_filesystem(file_config)
+    path.unlink()
+
+    assert output == example_content
+
+
+def test_load_from_local_filesystem_local_path():
+    """Test that ManifestLoader can load a local JSON file."""
+
+    example_content = {"foo": "bar"}
+    path = Path("example.json")
+
+    with open(path, "w") as file:
+        json.dump(example_content, file)
+
+    file_config = FileReferenceConfig(path=str(path))  # type: ignore
+
+    output = ManifestLoader.load_from_local_filesystem(file_config)
+    path.unlink()
+
+    assert output == example_content

From beb81d488e10a73361f87aa36bdc724b4fd548b4 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 14:36:59 -0400
Subject: [PATCH 02/10] test: Add a basic test for loading files from a URL.
 Fix a small defect in path pasing

---
 dbt_loom/manifests.py          |  2 +-
 tests/test_manifest_loaders.py | 38 +++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/dbt_loom/manifests.py b/dbt_loom/manifests.py
index a3c5f01..22d93aa 100644
--- a/dbt_loom/manifests.py
+++ b/dbt_loom/manifests.py
@@ -145,7 +145,7 @@ def load_from_http(config: FileReferenceConfig) -> Dict:
         if not config.path.path:
             raise InvalidManifestPath()
 
-        response = requests.get(config.path.path, stream=True)
+        response = requests.get(str(config.path), stream=True)
         response.raise_for_status()  # Check for request errors
 
         # Check for compression on the file. If compressed, store it in a buffer
diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 13f3515..69e5d1d 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -1,8 +1,10 @@
 import json
 from pathlib import Path
+import subprocess
 from pydantic import AnyUrl
+import pytest
 from dbt_loom.config import FileReferenceConfig
-from dbt_loom.manifests import ManifestLoader
+from dbt_loom.manifests import ManifestLoader, UnknownManifestPathType
 
 
 def test_load_from_local_filesystem_pass():
@@ -39,3 +41,37 @@ def test_load_from_local_filesystem_local_path():
     path.unlink()
 
     assert output == example_content
+
+
+def test_load_from_path_fails_invalid_scheme():
+    """
+    est that ManifestLoader will raise the appropriate exception if an invalid
+    scheme is applied.
+    """
+
+    file_config = FileReferenceConfig(path=AnyUrl("ftp://example.com/example.json"))  # type: ignore
+
+    with pytest.raises(UnknownManifestPathType):
+        ManifestLoader.load_from_path(file_config)
+
+
+def test_load_from_remote_pass():
+    """Test that ManifestLoader can load a remote JSON file via HTTP(S)."""
+
+    example_content = {"foo": "bar"}
+    path = Path("example.json")
+
+    with open(path, "w") as file:
+        json.dump(example_content, file)
+
+    file_config = FileReferenceConfig(path=AnyUrl("http://127.0.0.1:8000/example.json"))
+
+    # Invoke a server for hosting the test file.
+    process = subprocess.Popen(["python3", "-m", "http.server", "8000"])
+
+    output = ManifestLoader.load_from_http(file_config)
+
+    process.terminate()
+    path.unlink()
+
+    assert output == example_content

From 8e44c032e38cedf17f465cee23ac173edc90edc9 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 14:55:18 -0400
Subject: [PATCH 03/10] feat: Add tests for remote

---
 tests/test_dbt_core_execution.py | 6 ++++++
 tests/test_manifest_loaders.py   | 5 +++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tests/test_dbt_core_execution.py b/tests/test_dbt_core_execution.py
index 3266fc5..e020afc 100644
--- a/tests/test_dbt_core_execution.py
+++ b/tests/test_dbt_core_execution.py
@@ -46,6 +46,8 @@ def test_dbt_core_runs_loom_plugin():
         "revenue.orders.v2",
     }
 
+    os.chdir(starting_path)
+
     assert set(output.result).issuperset(
         subset
     ), "The child project is missing expected nodes. Check that injection still works."
@@ -88,6 +90,8 @@ def test_dbt_loom_injects_dependencies():
 
     path.unlink()
 
+    os.chdir(starting_path)
+
     # Make sure nothing failed
     assert isinstance(output.exception, dbt.exceptions.DbtReferenceError)
 
@@ -129,5 +133,7 @@ def test_dbt_loom_injects_groups():
 
     path.unlink()
 
+    os.chdir(starting_path)
+
     # Make sure nothing failed
     assert isinstance(output.exception, dbt.exceptions.DbtReferenceError)
diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 69e5d1d..ef6b07e 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -59,12 +59,13 @@ def test_load_from_remote_pass():
     """Test that ManifestLoader can load a remote JSON file via HTTP(S)."""
 
     example_content = {"foo": "bar"}
-    path = Path("example.json")
+    path = Path("example3.json")
+    base_url = "http://127.0.0.1:8000"
 
     with open(path, "w") as file:
         json.dump(example_content, file)
 
-    file_config = FileReferenceConfig(path=AnyUrl("http://127.0.0.1:8000/example.json"))
+    file_config = FileReferenceConfig(path=AnyUrl(f"{base_url}/example3.json"))
 
     # Invoke a server for hosting the test file.
     process = subprocess.Popen(["python3", "-m", "http.server", "8000"])

From b8316f6c820e8799d68c2575cacfacb9784ad9ac Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 15:01:33 -0400
Subject: [PATCH 04/10] tests: Create fixture for common scafolding code

---
 tests/test_manifest_loaders.py | 39 ++++++++++++++++------------------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index ef6b07e..3c7f5bb 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -1,49 +1,50 @@
 import json
 from pathlib import Path
 import subprocess
+from typing import Dict, Generator, Tuple
 from pydantic import AnyUrl
 import pytest
 from dbt_loom.config import FileReferenceConfig
 from dbt_loom.manifests import ManifestLoader, UnknownManifestPathType
 
 
-def test_load_from_local_filesystem_pass():
-    """Test that ManifestLoader can load a local JSON file."""
-
+@pytest.fixture
+def example_file() -> Generator[Tuple[Path, Dict], None, None]:
     example_content = {"foo": "bar"}
     path = Path("example.json")
-
     with open(path, "w") as file:
         json.dump(example_content, file)
+    yield path, example_content
+    path.unlink()
+
+
+def test_load_from_local_filesystem_pass(example_file):
+    """Test that ManifestLoader can load a local JSON file."""
+
+    path, example_content = example_file
 
     file_config = FileReferenceConfig(
         path=AnyUrl("file://" + str(Path(path).absolute()))
     )
 
     output = ManifestLoader.load_from_local_filesystem(file_config)
-    path.unlink()
 
     assert output == example_content
 
 
-def test_load_from_local_filesystem_local_path():
+def test_load_from_local_filesystem_local_path(example_file):
     """Test that ManifestLoader can load a local JSON file."""
 
-    example_content = {"foo": "bar"}
-    path = Path("example.json")
-
-    with open(path, "w") as file:
-        json.dump(example_content, file)
+    path, example_content = example_file
 
     file_config = FileReferenceConfig(path=str(path))  # type: ignore
 
     output = ManifestLoader.load_from_local_filesystem(file_config)
-    path.unlink()
 
     assert output == example_content
 
 
-def test_load_from_path_fails_invalid_scheme():
+def test_load_from_path_fails_invalid_scheme(example_file):
     """
     est that ManifestLoader will raise the appropriate exception if an invalid
     scheme is applied.
@@ -55,17 +56,14 @@ def test_load_from_path_fails_invalid_scheme():
         ManifestLoader.load_from_path(file_config)
 
 
-def test_load_from_remote_pass():
+def test_load_from_remote_pass(example_file):
     """Test that ManifestLoader can load a remote JSON file via HTTP(S)."""
 
-    example_content = {"foo": "bar"}
-    path = Path("example3.json")
-    base_url = "http://127.0.0.1:8000"
+    path, example_content = example_file
 
-    with open(path, "w") as file:
-        json.dump(example_content, file)
+    base_url = "http://127.0.0.1:8000"
 
-    file_config = FileReferenceConfig(path=AnyUrl(f"{base_url}/example3.json"))
+    file_config = FileReferenceConfig(path=AnyUrl(f"{base_url}/{path}"))
 
     # Invoke a server for hosting the test file.
     process = subprocess.Popen(["python3", "-m", "http.server", "8000"])
@@ -73,6 +71,5 @@ def test_load_from_remote_pass():
     output = ManifestLoader.load_from_http(file_config)
 
     process.terminate()
-    path.unlink()
 
     assert output == example_content

From a9f36a956efbafb3e60d7aff470dbe947bb378c8 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 18:23:34 -0400
Subject: [PATCH 05/10] fix: Store the reference json for the test in S3. I'll
 last forever, or until this project no longer matters

---
 tests/test_manifest_loaders.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 3c7f5bb..0941e6f 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -59,17 +59,14 @@ def test_load_from_path_fails_invalid_scheme(example_file):
 def test_load_from_remote_pass(example_file):
     """Test that ManifestLoader can load a remote JSON file via HTTP(S)."""
 
-    path, example_content = example_file
-
-    base_url = "http://127.0.0.1:8000"
-
-    file_config = FileReferenceConfig(path=AnyUrl(f"{base_url}/{path}"))
+    _, example_content = example_file
 
-    # Invoke a server for hosting the test file.
-    process = subprocess.Popen(["python3", "-m", "http.server", "8000"])
+    file_config = FileReferenceConfig(
+        path=AnyUrl(
+            "https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
+        )
+    )
 
     output = ManifestLoader.load_from_http(file_config)
 
-    process.terminate()
-
     assert output == example_content

From 10fe848440f826c4034f0807dd26c2cf4ffd9de7 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 18:30:51 -0400
Subject: [PATCH 06/10] tmp: Using kwargs instead because maybe it'll appease
 the old pydantic gods

---
 tests/test_manifest_loaders.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 0941e6f..3ef3d59 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -24,7 +24,7 @@ def test_load_from_local_filesystem_pass(example_file):
     path, example_content = example_file
 
     file_config = FileReferenceConfig(
-        path=AnyUrl("file://" + str(Path(path).absolute()))
+        path=AnyUrl(url="file://" + str(Path(path).absolute()))
     )
 
     output = ManifestLoader.load_from_local_filesystem(file_config)
@@ -63,7 +63,7 @@ def test_load_from_remote_pass(example_file):
 
     file_config = FileReferenceConfig(
         path=AnyUrl(
-            "https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
+            url="https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
         )
     )
 

From 5d4568b3a4d79c227984baef8eb81cd8fb7462a3 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 20:59:23 -0400
Subject: [PATCH 07/10] fix: Migrate from AnyUrl to ParsedResult for
 compatibility with dbt 1.6

---
 dbt_loom/config.py             |  9 +++++----
 dbt_loom/manifests.py          |  3 ++-
 tests/test_manifest_loaders.py | 17 ++++++++++-------
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/dbt_loom/config.py b/dbt_loom/config.py
index d15e8a6..c8c1efd 100644
--- a/dbt_loom/config.py
+++ b/dbt_loom/config.py
@@ -2,8 +2,9 @@
 from pathlib import Path
 import re
 from typing import List, Union
+from urllib.parse import ParseResult, urlparse
 
-from pydantic import BaseModel, AnyUrl, validator
+from pydantic import BaseModel, validator
 
 from dbt_loom.clients.az_blob import AzureReferenceConfig
 from dbt_loom.clients.dbt_cloud import DbtCloudReferenceConfig
@@ -24,7 +25,7 @@ class ManifestReferenceType(str, Enum):
 class FileReferenceConfig(BaseModel):
     """Configuration for a file reference"""
 
-    path: AnyUrl
+    path: ParseResult
 
     @validator("path", pre=True, always=True)
     def default_path(cls, v, values):
@@ -33,13 +34,13 @@ def default_path(cls, v, values):
         absolute file path.
         """
 
-        if isinstance(v, AnyUrl):
+        if isinstance(v, ParseResult):
             return v
 
         if bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", v)):
             return v
 
-        return "file://" + str(Path(v).absolute())
+        return urlparse("file://" + str(Path(v).absolute()))
 
 
 class ManifestReference(BaseModel):
diff --git a/dbt_loom/manifests.py b/dbt_loom/manifests.py
index 22d93aa..b89edbb 100644
--- a/dbt_loom/manifests.py
+++ b/dbt_loom/manifests.py
@@ -4,6 +4,7 @@
 import gzip
 from pathlib import Path
 from typing import Dict, List, Optional
+from urllib.parse import urlunparse
 
 from pydantic import BaseModel, Field, validator
 import requests
@@ -145,7 +146,7 @@ def load_from_http(config: FileReferenceConfig) -> Dict:
         if not config.path.path:
             raise InvalidManifestPath()
 
-        response = requests.get(str(config.path), stream=True)
+        response = requests.get(urlunparse(config.path), stream=True)
         response.raise_for_status()  # Check for request errors
 
         # Check for compression on the file. If compressed, store it in a buffer
diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 3ef3d59..780c4af 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -1,8 +1,9 @@
 import json
 from pathlib import Path
-import subprocess
+
 from typing import Dict, Generator, Tuple
-from pydantic import AnyUrl
+from urllib.parse import urlparse
+
 import pytest
 from dbt_loom.config import FileReferenceConfig
 from dbt_loom.manifests import ManifestLoader, UnknownManifestPathType
@@ -24,7 +25,7 @@ def test_load_from_local_filesystem_pass(example_file):
     path, example_content = example_file
 
     file_config = FileReferenceConfig(
-        path=AnyUrl(url="file://" + str(Path(path).absolute()))
+        path=urlparse("file://" + str(Path(path).absolute()))
     )
 
     output = ManifestLoader.load_from_local_filesystem(file_config)
@@ -50,7 +51,9 @@ def test_load_from_path_fails_invalid_scheme(example_file):
     scheme is applied.
     """
 
-    file_config = FileReferenceConfig(path=AnyUrl("ftp://example.com/example.json"))  # type: ignore
+    file_config = FileReferenceConfig(
+        path=urlparse("ftp://example.com/example.json"),
+    )  # type: ignore
 
     with pytest.raises(UnknownManifestPathType):
         ManifestLoader.load_from_path(file_config)
@@ -62,9 +65,9 @@ def test_load_from_remote_pass(example_file):
     _, example_content = example_file
 
     file_config = FileReferenceConfig(
-        path=AnyUrl(
-            url="https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
-        )
+        path=urlparse(
+            "https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
+        ),
     )
 
     output = ManifestLoader.load_from_http(file_config)

From 14b589413f8fd3a59b76be3c4696775b7788c905 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 21:09:19 -0400
Subject: [PATCH 08/10] fix: Correct a file loader sub-path based on scheme
 type

---
 dbt_loom/manifests.py          |  2 +-
 tests/test_manifest_loaders.py | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/dbt_loom/manifests.py b/dbt_loom/manifests.py
index b89edbb..ef6bb05 100644
--- a/dbt_loom/manifests.py
+++ b/dbt_loom/manifests.py
@@ -99,7 +99,7 @@ class InvalidManifestPath(Exception):
 class ManifestLoader:
     def __init__(self):
         self.loading_functions = {
-            ManifestReferenceType.file: self.load_from_local_filesystem,
+            ManifestReferenceType.file: self.load_from_path,
             ManifestReferenceType.dbt_cloud: self.load_from_dbt_cloud,
             ManifestReferenceType.gcs: self.load_from_gcs,
             ManifestReferenceType.s3: self.load_from_s3,
diff --git a/tests/test_manifest_loaders.py b/tests/test_manifest_loaders.py
index 780c4af..88e8547 100644
--- a/tests/test_manifest_loaders.py
+++ b/tests/test_manifest_loaders.py
@@ -5,7 +5,11 @@
 from urllib.parse import urlparse
 
 import pytest
-from dbt_loom.config import FileReferenceConfig
+from dbt_loom.config import (
+    FileReferenceConfig,
+    ManifestReference,
+    ManifestReferenceType,
+)
 from dbt_loom.manifests import ManifestLoader, UnknownManifestPathType
 
 
@@ -73,3 +77,23 @@ def test_load_from_remote_pass(example_file):
     output = ManifestLoader.load_from_http(file_config)
 
     assert output == example_content
+
+
+def test_manifest_loader_selection(example_file):
+    """Confirm scheme parsing works for picking the manifest loader."""
+    _, example_content = example_file
+    manifest_loader = ManifestLoader()
+
+    file_config = FileReferenceConfig(
+        path=urlparse(
+            "https://s3.us-east-2.amazonaws.com/com.nicholasyager.dbt-loom/example.json"
+        ),
+    )
+
+    manifest_reference = ManifestReference(
+        name="example", type=ManifestReferenceType.file, config=file_config
+    )
+
+    manifest = manifest_loader.load(manifest_reference)
+
+    assert manifest == example_content

From 522c2033d7bd5a31dfecc9afa513de9daf522ee3 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Sat, 5 Oct 2024 21:11:47 -0400
Subject: [PATCH 09/10] docs: Add documentation for using remote paths to
 manifests

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 7be5522..153b67c 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,7 @@ flowchart LR
 dbt-loom currently supports obtaining model definitions from:
 
 - Local manifest files
+- Remote manifest files via http(s)
 - dbt Cloud
 - GCS
 - S3-compatible object storage services
@@ -57,6 +58,8 @@ manifests:
   - name: project_name # This should match the project's real name
     type: file
     config:
+      # A path to your manifest. This can be either a local path, or a remote
+      # path accessible via http(s).
       path: path/to/manifest.json
 ```
 

From 4324bc1216caf714c6ac917d44a6571464788e65 Mon Sep 17 00:00:00 2001
From: Nicholas Yager <yager@nicholasyager.com>
Date: Mon, 7 Oct 2024 20:07:43 -0400
Subject: [PATCH 10/10] fix: Return ParseResult if FileReferenceConfig.path is
 a URL with a schema

---
 dbt_loom/config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbt_loom/config.py b/dbt_loom/config.py
index c8c1efd..19d2402 100644
--- a/dbt_loom/config.py
+++ b/dbt_loom/config.py
@@ -28,7 +28,7 @@ class FileReferenceConfig(BaseModel):
     path: ParseResult
 
     @validator("path", pre=True, always=True)
-    def default_path(cls, v, values):
+    def default_path(cls, v, values) -> ParseResult:
         """
         Check if the provided path is a valid URL. If not, convert it into an
         absolute file path.
@@ -38,7 +38,7 @@ def default_path(cls, v, values):
             return v
 
         if bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", v)):
-            return v
+            return urlparse(v)
 
         return urlparse("file://" + str(Path(v).absolute()))