Skip to content

Commit

Permalink
Convert file spec to FSS entry
Browse files Browse the repository at this point in the history
  • Loading branch information
philss committed Aug 22, 2023
1 parent 920dceb commit d4f0dee
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 2 deletions.
34 changes: 32 additions & 2 deletions lib/kino/fs.ex
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,14 @@ defmodule Kino.FS do
key: String.t()
}

@typep fss_entry :: FSS.Local.Entry.t() | FSS.S3.Entry.t() | FSS.HTTP.Entry.t()

@doc false
@spec file_spec(String.t()) :: file_spec()
@spec file_spec(String.t()) :: fss_entry()
def file_spec(name) do
case Kino.Bridge.get_file_entry_spec(name) do
{:ok, spec} ->
spec
file_spec_to_fss(spec)

{:error, :forbidden} ->
raise ForbiddenError, name: name
Expand All @@ -83,4 +85,32 @@ defmodule Kino.FS do
raise "failed to access file spec, reason: #{inspect(reason)}"
end
end

@doc false
@spec file_spec_to_fss(file_spec()) ::
fss_entry()
def file_spec_to_fss(%{type: :local} = file_spec) do
%FSS.Local.Entry{path: file_spec.path}
end

def file_spec_to_fss(%{type: :url} = file_spec) do
case FSS.HTTP.parse(file_spec.url) do
{:ok, entry} -> entry
{:error, error} -> raise error
end
end

def file_spec_to_fss(%{type: :s3} = file_spec) do
case FSS.S3.parse("s3:///" <> file_spec.key,
config: [
region: file_spec.region,
endpoint: file_spec.bucket_url,
access_key_id: file_spec.access_key_id,
secret_access_key: file_spec.secret_access_key
]
) do
{:ok, entry} -> entry
{:error, error} -> raise error
end
end
end
1 change: 1 addition & 0 deletions mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ defmodule Kino.MixProject do
defp deps do
[
{:table, "~> 0.1.2"},
{:fss, github: "elixir-explorer/fss", branch: "main"},
{:nx, "~> 0.1", optional: true},
{:ex_doc, "~> 0.28", only: :dev, runtime: false}
]
Expand Down
1 change: 1 addition & 0 deletions mix.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"complex": {:hex, :complex, "0.4.2", "923e5db0be13dbb3ea00cf8459d9f75f3afdd9ff5a82742ded21064330d28273", [:mix], [], "hexpm", "069a085ef820ce675a2619fd125b963ff4514af2102c7f7d7965128e5ec0a429"},
"earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"},
"ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"},
"fss": {:git, "https://github.com/elixir-explorer/fss.git", "bf54b55612d12aa172aa7f75d0555359a20bfab1", [branch: "main"]},
"makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
"makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
"makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
Expand Down
68 changes: 68 additions & 0 deletions test/kino/fs_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
defmodule Kino.FSTest do
use ExUnit.Case, async: true

describe "file_spec_to_fss/1" do
test "returns a file spec" do
path = "/home/bob/file.txt"
assert %FSS.Local.Entry{path: ^path} = Kino.FS.file_spec_to_fss(%{type: :local, path: path})
end

test "returns an HTTP FSS entry" do
url = "https://example.com/file.txt"

assert %FSS.HTTP.Entry{url: ^url, config: %FSS.HTTP.Config{headers: []}} =
Kino.FS.file_spec_to_fss(%{type: :url, url: url})
end

test "returns a S3 FSS entry" do
bucket_url = "https://s3.us-west-1.amazonaws.com/my-bucket"

s3_spec = %{
type: :s3,
bucket_url: bucket_url,
region: "us-west-1",
access_key_id: "access-key-1",
secret_access_key: "secret-access-key-1",
key: "file"
}

assert %FSS.S3.Entry{} = s3 = Kino.FS.file_spec_to_fss(s3_spec)

assert s3.key == s3_spec.key

assert s3.config.region == s3_spec.region
assert s3.config.endpoint == bucket_url
assert s3.config.access_key_id == s3_spec.access_key_id
assert s3.config.secret_access_key == s3_spec.secret_access_key
assert s3.config.bucket == nil
end

test "raises an error in case s3 file_spec has something nil" do
s3_spec = %{
type: :s3,
bucket_url: nil,
region: "us-west-1",
access_key_id: "access-key-1",
secret_access_key: "secret-access-key-1",
key: "file"
}

assert_raise ArgumentError, "endpoint is required when bucket is nil", fn ->
Kino.FS.file_spec_to_fss(s3_spec)
end

bucket_url = "https://s3.us-west-1.amazonaws.com/my-bucket"

s3_spec =
s3_spec
|> Map.replace!(:bucket_url, bucket_url)
|> Map.replace!(:access_key_id, nil)

assert_raise ArgumentError,
"missing :access_key_id for FSS.S3 (set the key or the AWS_ACCESS_KEY_ID env var)",
fn ->
Kino.FS.file_spec_to_fss(s3_spec)
end
end
end
end

0 comments on commit d4f0dee

Please sign in to comment.