From 266d8915bcc54a232fc423591d34e1ea2e16ee89 Mon Sep 17 00:00:00 2001 From: Giovanni Visciano Date: Fri, 3 Jun 2022 17:41:18 +0200 Subject: [PATCH] date custom type loader --- README.md | 113 +++++++++++++++------------------- coveralls.json | 5 ++ lib/dataspecs.ex | 2 + lib/dataspecs/loader.ex | 10 ++- lib/dataspecs/loader/extra.ex | 35 +++++++++++ test/dataspec_test.exs | 64 +------------------ test/extra_test.exs | 70 +++++++++++++++++++++ test/support/sample_type.ex | 1 + 8 files changed, 175 insertions(+), 125 deletions(-) create mode 100644 coveralls.json create mode 100644 test/extra_test.exs diff --git a/README.md b/README.md index 02b3bf2..55849dd 100644 --- a/README.md +++ b/README.md @@ -7,30 +7,39 @@ Typespec based data loader and validator (inspired by [forma](https://github.com DataSpecs **validate and load** elixir data into a more structured form by trying to map it to conform to a [typespec](https://hexdocs.pm/elixir/typespecs.html). -It support most typespec specification: **basic** types, **literal** types, -**built-in** types, **union** type, **parametrized** types, **maps**, **remote** types -and **user defined** types. -It can be used to validate some elixir data against a typespec or it -can be useful when interfacing with external data sources that provide -you data as JSON or MessagePack, but that you wish to validate transform -into either proper structs or richer data types without a native -JSON representation (such as dates or sets) in your application. +It support the following typespec type specifications: +- basic +- literal +- built-in +- union +- parametrized +- maps (and elixir struct) +- remote +- user defined + +The main use cases are about elixir data validatation against a typespec or +interfacing with external data sources that provide you data as JSON or MessagePack, +but that you wish to validate and transform into either proper structs or +richer data types without a native JSON representation (such as dates or sets). ## Usage +Given the following `Person` struct specification + ```elixir defmodule Person do use DataSpecs @enforce_keys [:name, :surname] - defstruct @enforce_keys ++ [:gender, :address] + defstruct @enforce_keys ++ [:gender, :address, :birth_date] @type t :: %__MODULE__{ name: String.t(), surname: String.t(), gender: option(:male | :female | :other), - address: option(nonempty_list(Address.t())) + address: option([Address.t(), ...]), + birth_date: option(Date.t()) } @type option(x) :: nil | x @@ -47,23 +56,29 @@ defmodule Address do town: String.t() } end +``` -raw = %{ - "name" => "Joe", - "surname" => "Smith", - "gender" => "male", - "address" => [%{ - "streetname" => "High Street", - "streenumber" => "3a", - "postcode" => "SO31 4NG", - "town" => "Hedge End, Southampton" - }] -} +we can load a JSON object encoding an instance of a `Person` with: -DataSpecs.load(raw, {Person, :t}) +```elixir +~s/{ + "name": "Joe", + "surname": "Smith", + "gender": "male", + "birth_date": "1980-12-31", + "address": [{ + "streetname": "High Street", + "streenumber": "3a", + "postcode": "SO31 4NG", + "town": "Hedge End, Southampton" + }] +}/ +|> Jason.decode!() +|> Person.load(DataSpecs.Loader.Extra.type_loaders()) -# OR (if "use DataSpecs" is included in the struct module) -Person.load(raw) +# NOTE: +# DataSpecs.Loader.Extra.type_loaders() is included here to support +# the loading of isodates strings into Date.t() types. # => %Person{ # address: [ @@ -74,20 +89,21 @@ Person.load(raw) # town: "Hedge End, Southampton" # } # ], +# birth_date: ~D[1980-12-31], # gender: :male, # name: "Joe", # surname: "Smith" # } ``` -DataSpecs tries to figure out how to translate its input to a typespec. +DataSpecs tries to figure out how to translate its input to an elixir datatype using the typespec as "type schema". Scalar types (such as booleans, integers, etc.) and some composite types (such as lists, plain maps), can be simply mapped one to one after validation without any additional transformation. However, not all Elixir types have natural representations in JSON-like data, -for example dates, or don't want to expose their internals (opaque types). +for example atoms, dates, or don't want to expose their internals (opaque types). Refer to the library test suite for more examples. @@ -131,45 +147,20 @@ def project do end ``` -## Custom type loaders +## Type loaders -In these cases you can pass a set of custom type loaders along as an optional argument -to the `DataSpecs.load` function +### Builtin -```elixir -defmodule LogRow do - use DataSpecs +For reference check the loaders available under `DataSpecs.Loader.{Builtin, Extra}`. - @enforce_keys [:log, :timestamp] - defstruct @enforce_keys +The modules `DataSpecs.Loader.Extra` provides pre defined custom type loader for: +- `DateTime.t`: load iso datetime strings (ie: `2001-12-31 06:54:02Z` -> `~U[2001-12-31 06:54:02Z]`) +- `DateTime.t`: load iso datetime strings (ie: `2001-12-31` -> `~D[2022-06-03]`) +- `MapSet.t`: load lists of T into a `MapSet.t(T)` (ie: `[1, 2]` -> `#MapSet<[1, 2]>`) - type t :: %__MODULE__{ - log: String.t(), - timestamp: DateTime.t() - } -end +### Custom -def custom_isodatetime_loader(value, _custom_type_loaders, []) do - with {:is_binary, true} <- {:is_binary, is_binary(value)}, - {:from_iso8601, {:ok, datetime, _}} <- {:from_iso8601, DateTime.from_iso8601(value)} do - {:ok, datetime} - else - {:is_binary, false} -> - {:error, ["can't convert #{inspect(value)} to a DateTime.t/0"]} - - {:from_iso8601, {:error, reason}} -> - {:error, ["can't convert #{inspect(value)} to a DateTime.t/0 (#{inspect(reason)})"]} - end -end - -custom_type_loaders = %{{DateTime, :t, 0} => &custom_isodatetime_loader/3} -LogRow.load(%{"log" => "An error occurred", "timestamp" => "2021-07-14 20:22:49.653077Z"}, custom_type_loaders) - -# => %LogRow{ -# log: "An error occurred", -# timestamp: ~U[2021-07-14 20:22:49.653077Z] -# } -``` +You can pass a set of custom type loaders along as an optional argument to the `DataSpecs.load` function The type of the custom loader function is @@ -223,8 +214,6 @@ then the custom type loader function will be called with custom_mapset_loader(1..10, custom_type_loaders, [&DataSpecs.Loader.Builtin.integer/3]) ``` -For reference check the loaders available under `DataSpecs.Loader.{Builtin, Extra}` - ## Validators Custom validation rules can be defined with a custom type loader. diff --git a/coveralls.json b/coveralls.json new file mode 100644 index 0000000..6ad3cfb --- /dev/null +++ b/coveralls.json @@ -0,0 +1,5 @@ +{ + "coverage_options": { + "minimum_coverage": 100 + } +} \ No newline at end of file diff --git a/lib/dataspecs.ex b/lib/dataspecs.ex index 2681ccf..210ab61 100644 --- a/lib/dataspecs.ex +++ b/lib/dataspecs.ex @@ -29,6 +29,7 @@ defmodule DataSpecs do "name" => "Joe", "surname" => "Smith", "gender" => "male", + "birth_date" => "1980-12-31", "address" => [%{ "streetname" => "High Street", "streenumber" => "3a", @@ -46,6 +47,7 @@ defmodule DataSpecs do town: "Hedge End, Southampton" } ], + birth_date: ~D[1980-12-31], gender: :male, name: "Joe", surname: "Smith" diff --git a/lib/dataspecs/loader.ex b/lib/dataspecs/loader.ex index 7f3d40b..5422b09 100644 --- a/lib/dataspecs/loader.ex +++ b/lib/dataspecs/loader.ex @@ -40,7 +40,15 @@ defmodule DataSpecs.Loader do eaf_types :error -> - raise "Can't fetch type specifications for module #{inspect(module)}" + raise """ + Can't fetch type specifications for module #{inspect(module)}. + + The dataspec library works leveraging the typespec of #{inspect(module)}. + To correctly introspec and retrieve the typespecs the module #{inspect(module)} + should be compiled with the option strip_beams=false. + Even more, the typespec cannot be retrieved if you define the module #{inspect(module)} + in an interactive iex shell session. + """ end end diff --git a/lib/dataspecs/loader/extra.ex b/lib/dataspecs/loader/extra.ex index c5d4a83..9b74119 100644 --- a/lib/dataspecs/loader/extra.ex +++ b/lib/dataspecs/loader/extra.ex @@ -5,6 +5,21 @@ defmodule DataSpecs.Loader.Extra do alias DataSpecs.{Loader, Types} + @spec type_loaders() :: Types.custom_type_loaders() + + @doc """ + All extra type loaders. + + MyData.load(value, #{inspect(__MODULE__)}.type_loaders()) + """ + def type_loaders do + %{ + {MapSet, :t, 1} => &mapset/3, + {DateTime, :t, 0} => &isodatetime/3, + {Date, :t, 0} => &isodate/3 + } + end + @spec mapset(Types.value(), Types.custom_type_loaders(), [Types.type_loader_fun()]) :: {:error, Types.reason()} | {:ok, MapSet.t()} @@ -50,4 +65,24 @@ defmodule DataSpecs.Loader.Extra do {:error, ["can't convert #{inspect(value)} to a DateTime.t/0 (#{inspect(reason)})"]} end end + + @spec isodate(Types.value(), Types.custom_type_loaders(), [Types.type_loader_fun()]) :: + {:error, Types.reason()} | {:ok, DateTime.t()} + + @doc """ + Type loader for Elixir Date.t(). + Expect an iso8601 date string value, returns a Date.t(). + """ + def isodate(value, _custom_type_loaders, []) do + with {:is_binary, true} <- {:is_binary, is_binary(value)}, + {:from_iso8601, {:ok, date}} <- {:from_iso8601, Date.from_iso8601(value)} do + {:ok, date} + else + {:is_binary, false} -> + {:error, ["can't convert #{inspect(value)} to a Date.t/0"]} + + {:from_iso8601, {:error, reason}} -> + {:error, ["can't convert #{inspect(value)} to a Date.t/0 (#{inspect(reason)})"]} + end + end end diff --git a/test/dataspec_test.exs b/test/dataspec_test.exs index e89468c..24bd71d 100644 --- a/test/dataspec_test.exs +++ b/test/dataspec_test.exs @@ -1,5 +1,5 @@ defmodule Test.DataSpecs do - use ExUnit.Case + use ExUnit.Case, async: true alias DataSpecs.Loader alias Test.DataSpecs.CustomLoader @@ -9,7 +9,7 @@ defmodule Test.DataSpecs do describe "Unknown" do test "module" do - assert_raise RuntimeError, "Can't fetch type specifications for module :unknown_module", fn -> + assert_raise RuntimeError, ~r/Can't fetch type specifications for module :unknown_module/, fn -> DataSpecs.load(:a, {:unknown_module, :t}) end end @@ -456,66 +456,6 @@ defmodule Test.DataSpecs do end end - describe "extra loaders" do - test "isodatetime" do - custom_type_loaders = %{ - {DateTime, :t, 0} => &Loader.Extra.isodatetime/3 - } - - datetime = ~U[2021-07-14 20:22:49.653077Z] - iso_datetime_string = DateTime.to_iso8601(datetime) - - assert {:ok, datetime} == DataSpecs.load(iso_datetime_string, {@types_module, :t_datetime}, custom_type_loaders) - end - - test "isodatetime error" do - custom_type_loaders = %{ - {DateTime, :t, 0} => &Loader.Extra.isodatetime/3 - } - - value = "not a datetime" - - assert {:error, ["can't convert \"#{value}\" to a DateTime.t/0 (:invalid_format)"]} == - DataSpecs.load(value, {@types_module, :t_datetime}, custom_type_loaders) - - value = 123 - - assert {:error, ["can't convert #{value} to a DateTime.t/0"]} == - DataSpecs.load(value, {@types_module, :t_datetime}, custom_type_loaders) - end - - test "mapset" do - custom_type_loaders = %{ - {MapSet, :t, 1} => &Loader.Extra.mapset/3 - } - - assert {:ok, MapSet.new(1..3)} == DataSpecs.load(1..3, {@types_module, :t_mapset}, custom_type_loaders) - - assert {:ok, MapSet.new(["1", :a, 1])} == - DataSpecs.load(["1", :a, 1], {@types_module, :t_mapset_1}, custom_type_loaders) - end - - test "mapset error" do - custom_type_loaders = %{ - {MapSet, :t, 1} => &Loader.Extra.mapset/3 - } - - value = 123 - - assert {:error, ["can't convert #{value} to a MapSet.t/1, value not enumerable"]} == - DataSpecs.load(value, {@types_module, :t_mapset}, custom_type_loaders) - - value = [%{}] - - assert {:error, - [ - "can't convert #{inspect(value)} to a MapSet.t/1", - ["can't convert #{inspect(value)} to a list, bad item at index=0", ["can't convert %{} to an integer"]] - ]} == - DataSpecs.load(value, {@types_module, :t_mapset}, custom_type_loaders) - end - end - test "typep" do assert {:ok, :a} == DataSpecs.load(:a, {@types_module, :t_reference_to_private_type}) end diff --git a/test/extra_test.exs b/test/extra_test.exs new file mode 100644 index 0000000..f79247a --- /dev/null +++ b/test/extra_test.exs @@ -0,0 +1,70 @@ +defmodule Test.DataSpecs.Loader.Extra do + use ExUnit.Case, async: true + + import DataSpecs.Loader.Extra, only: [type_loaders: 0] + + @types_module Test.DataSpecs.SampleType + + describe "extra loaders" do + test "isodatetime" do + datetime = ~U[2021-07-14 20:22:49.653077Z] + iso_datetime_string = DateTime.to_iso8601(datetime) + + assert {:ok, datetime} == DataSpecs.load(iso_datetime_string, {@types_module, :t_datetime}, type_loaders()) + end + + test "isodatetime error" do + value = "not a datetime" + + assert {:error, ["can't convert \"#{value}\" to a DateTime.t/0 (:invalid_format)"]} == + DataSpecs.load(value, {@types_module, :t_datetime}, type_loaders()) + + value = 123 + + assert {:error, ["can't convert #{value} to a DateTime.t/0"]} == + DataSpecs.load(value, {@types_module, :t_datetime}, type_loaders()) + end + + test "isodate" do + date = ~D[2021-07-14] + iso_date_string = Date.to_iso8601(date) + + assert {:ok, date} == DataSpecs.load(iso_date_string, {@types_module, :t_date}, type_loaders()) + end + + test "isodate error" do + value = "not a date" + + assert {:error, ["can't convert \"#{value}\" to a Date.t/0 (:invalid_format)"]} == + DataSpecs.load(value, {@types_module, :t_date}, type_loaders()) + + value = 123 + + assert {:error, ["can't convert #{value} to a Date.t/0"]} == + DataSpecs.load(value, {@types_module, :t_date}, type_loaders()) + end + + test "mapset" do + assert {:ok, MapSet.new(1..3)} == DataSpecs.load(1..3, {@types_module, :t_mapset}, type_loaders()) + + assert {:ok, MapSet.new(["1", :a, 1])} == + DataSpecs.load(["1", :a, 1], {@types_module, :t_mapset_1}, type_loaders()) + end + + test "mapset error" do + value = 123 + + assert {:error, ["can't convert #{value} to a MapSet.t/1, value not enumerable"]} == + DataSpecs.load(value, {@types_module, :t_mapset}, type_loaders()) + + value = [%{}] + + assert {:error, + [ + "can't convert #{inspect(value)} to a MapSet.t/1", + ["can't convert #{inspect(value)} to a list, bad item at index=0", ["can't convert %{} to an integer"]] + ]} == + DataSpecs.load(value, {@types_module, :t_mapset}, type_loaders()) + end + end +end diff --git a/test/support/sample_type.ex b/test/support/sample_type.ex index 0a53ff0..91fc8e8 100644 --- a/test/support/sample_type.ex +++ b/test/support/sample_type.ex @@ -66,6 +66,7 @@ defmodule Test.DataSpecs.SampleType do @type t_mapset :: MapSet.t(integer()) @type t_mapset_1 :: MapSet.t(t_union_0(binary())) @type t_datetime :: DateTime.t() + @type t_date :: Date.t() @type t_recursive :: atom() | %{recursive: t_recursive()}