Remove cmark-gfm cli in favor of MDEx (comrak nif)

.github/workflows/elixir.yml

@@ -15,6 +15,7 @@ jobs:
 
     env:
       MIX_ENV: test
+      MDEX_BUILD: 1
 
     strategy:
       matrix:
@@ -66,16 +67,14 @@ jobs:
 
       - run: mix tailwind.install
 
-      - name: install cmark-gfm
-        run: sudo apt-get install -y cmark-gfm
-
       - run: mix test
 
   quality:
     name: quality (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}})
 
     env:
       MIX_ENV: dev
+      MDEX_BUILD: 1
 
     strategy:
       matrix:

README.md

@@ -29,9 +29,7 @@ Main components:
 
 The file `dev.exs` is a self-contained Phoenix application running Beacon with sample data and code reloading enabled. Follow these steps to get a site up and running:
 
-1. Install [cmark-gfm](https://github.com/github/cmark-gfm)
-
-2. Install dependencies, build assets, and run database setup:
+1. Install dependencies, build assets, and run database setup:
 
 ```sh
 mix setup
@@ -40,7 +38,7 @@ mix setup
 If deps compilation fails, make sure your environment has the compilers installed.
 On Ubuntu look for the `build_essential` package, on macOS install utilities with `xcode-select --install`
 
-3. Execute the dev script:
+2. Execute the dev script:
 
 ```sh
 iex --sname core -S mix dev

guides/introduction/installation.md

@@ -14,31 +14,29 @@ We recomment following the guide thoroughly, but if you want a short version or
   mix archive.install hex phx_new
   ```
 
-3. Install [cmark-gfm](https://github.com/github/cmark-gfm)
+3. Setup a database
 
-4. Setup a database
-
-5. Generate a new Phoenix application
+4. Generate a new Phoenix application
 
   ```sh
   mix phx.new --install my_app
   ```
 
-6. Add `:beacon` dependency to `mix.exs`
+5. Add `:beacon` dependency to `mix.exs`
 
   ```elixir
   {:beacon, github: "beaconCMS/beacon"}
   ```
 
-7. Run `mix deps.get`
+6. Run `mix deps.get`
 
-8. Add `:beacon` dependency to `.formatter.exs` in `:
+7. Add `:beacon` dependency to `.formatter.exs` in `:
 
-9. Run `mix beacon.install --site my_site`
+8. Run `mix beacon.install --site my_site`
 
-10. Run `mix setup`
+9. Run `mix setup`
 
-11. Run `mix phx.server`
+10. Run `mix phx.server`
 
 Visit http://localhost:4000/my_site/home to see the page created from seeds.
 
@@ -56,7 +54,7 @@ The minimum required version to run Beacon is Elixir v1.14. Make sure you have a
 elixir --version
 ```
 
-2. Install or updated Hex
+2. Install or update Hex
 
 ```sh
 mix local.hex
@@ -72,10 +70,6 @@ Beacon also requires a minimum Phoenix version to work properly, make sure you h
 mix archive.install hex phx_new
 ```
 
-### cmark-gfm
-
-Is the tool used to convert Markdown to HTML. Install it from https://github.com/github/cmark-gfm and make sure the binary `cmark-gfm` is in your env `$PATH`
-
 ### Database
 
 [PostgresSQL](https://www.postgresql.org) is the default database used by Phoenix and Beacon but it also supports MySQL and SQLServer through [ecto](https://hex.pm/packages/ecto) official adapters. Make sure one of them is up and running in your environment.

lib/beacon/loader/page_module_loader.ex

@@ -16,7 +16,7 @@ defmodule Beacon.Loader.PageModuleLoader do
     - `:request` - it will load the template, useful during a request
 
   """
-  if Code.ensure_loaded?(Mix.Project) and Mix.env() in [:test] do
+  if Code.ensure_loaded?(Mix.Project) and Mix.env() in [:test, :dev] do
     def load_page!(%Content.Page{} = page, stage \\ :request) do
       do_load_page!(page, stage)
     end

lib/beacon/template/markdown.ex

@@ -2,51 +2,26 @@ defmodule Beacon.Template.Markdown do
   @moduledoc """
   GitHub Flavored Markdown
 
-  Use https://github.com/github/cmark-gfm to convert Markdown to HTML
+  Use https://github.com/leandrocp/mdex to convert Markdown to HTML
   """
 
   # TODO: implement a markdown format that is aware of Phoenix features like link attrs and assigns
 
-  # TODO: replace cli with C or Rust lib
   @spec convert_to_html(Beacon.Template.t(), Beacon.Template.LoadMetadata.t()) :: {:cont, Beacon.Template.t()} | {:halt, Exception.t()}
   def convert_to_html(template, _metadata) do
-    cmark_gfm_bin = find_cmark_gfm_bin!()
-
-    random_file_name = Base.encode16(:crypto.strong_rand_bytes(12))
-    random_file_path = Path.join(System.tmp_dir!(), random_file_name)
-    File.write!(random_file_path, template)
-
-    # credo:disable-for-next-line
-    args = ~w(--unsafe --smart -e table -e autolink -e tasklist --to html) ++ [random_file_path]
-    # credo:disable-for-next-line
-
-    {output, exit_code} = System.cmd(cmark_gfm_bin, args, stderr_to_stdout: true)
-    File.rm(random_file_path)
-
-    if exit_code == 0 do
-      {:cont, output}
-    else
+    template = MDEx.to_html(template, extension: [table: true, autolink: true, tasklist: true], parse: [smart: true], render: [unsafe_: true])
+    {:cont, template}
+  rescue
+    exception ->
       message = """
       failed to convert markdown to html
 
       Got:
 
-          exit code: #{exit_code}
-          output: #{output}
+        #{Exception.message(exception)}
 
       """
 
       {:halt, %Beacon.ParserError{message: message}}
-    end
-  end
-
-  defp find_cmark_gfm_bin! do
-    message = """
-    failed to find cmark-gfm bin
-
-    make sure it's installed and defined in your $PATH env var
-    """
-
-    System.find_executable("cmark-gfm") || raise message
   end
 end

mix.exs

@@ -61,7 +61,9 @@ defmodule Beacon.MixProject do
       {:postgrex, "~> 0.16"},
       {:safe_code, github: "TheFirstAvenger/safe_code"},
       {:tailwind, "~> 0.2"},
-      live_monaco_editor_dep()
+      {:rustler, ">= 0.0.0", optional: true},
+      live_monaco_editor_dep(),
+      mdex_dep()
     ]
   end
 
@@ -73,6 +75,14 @@ defmodule Beacon.MixProject do
     end
   end
 
+  defp mdex_dep do
+    if path = System.get_env("MDEX_PATH") do
+      {:mdex, path: path}
+    else
+      {:mdex, "~> 0.1.2"}
+    end
+  end
+
   defp aliases do
     [
       setup: ["deps.get", "assets.setup", "assets.build", "ecto.setup"],

mix.lock

@@ -30,11 +30,12 @@
   "heroicons": {:hex, :heroicons, "0.5.3", "ee8ae8335303df3b18f2cc07f46e1cb6e761ba4cf2c901623fbe9a28c0bc51dd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.2", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "a210037e8a09ac17e2a0a0779d729e89c821c944434c3baa7edfc1f5b32f3502"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "image": {:hex, :image, "0.33.0", "f18e67f8dbffe069012f71ad1ed8331221c5d96496f182e057d2174aba5db40f", [:mix], [{:bumblebee, "~> 0.2", [hex: :bumblebee, repo: "hexpm", optional: true]}, {:evision, "~> 0.1.26", [hex: :evision, repo: "hexpm", optional: true]}, {:exla, "~> 0.5", [hex: :exla, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}, {:nx, "~> 0.5", [hex: :nx, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14 or ~> 3.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: false]}, {:vix, "~> 0.17", [hex: :vix, repo: "hexpm", optional: false]}], "hexpm", "1436a5e929f6c8296bf24c39aaaf3bb47fd24af95d0b4cb6dcfe6e02ddf55e58"},
-  "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
+  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "live_monaco_editor": {:hex, :live_monaco_editor, "0.1.2", "bf7ec7a12d62e5eb0b917839c93547d29b90aba201edafdfe7515149f2d05384", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "22981cc1479571638b5eba67d95dcd52e6bf6c34150f87676f0c36feb08841f2"},
   "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+  "mdex": {:hex, :mdex, "0.1.1", "17868737eac8603b2e1c409a5f6c0d5404c66cbecef80699d59267d31df6ca98", [:mix], [{:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.6", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "9ad888802b226834fb6f85ab36ce64e975415d08598337d56810de64035bf0fb"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
@@ -54,12 +55,15 @@
   "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
   "postgrex": {:hex, :postgrex, "0.17.1", "01c29fd1205940ee55f7addb8f1dc25618ca63a8817e56fac4f6846fc2cddcbe", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "14b057b488e73be2beee508fb1955d8db90d6485c6466428fe9ccf1d6692a555"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
+  "rustler": {:hex, :rustler, "0.29.1", "880f20ae3027bd7945def6cea767f5257bc926f33ff50c0d5d5a5315883c084d", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "109497d701861bfcd26eb8f5801fe327a8eef304f56a5b63ef61151ff44ac9b6"},
+  "rustler_precompiled": {:hex, :rustler_precompiled, "0.6.3", "f838d94bc35e1844973ee7266127b156fdc962e9e8b7ff666c8fb4fed7964d23", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "e18ecca3669a7454b3a2be75ae6c3ef01d550bc9a8cf5fbddcfff843b881d7c6"},
   "safe_code": {:git, "https://github.com/TheFirstAvenger/safe_code.git", "9907b385f1b71c0602183bf7fd8732570252b622", []},
   "solid": {:hex, :solid, "0.14.1", "61b454dc7e7bf8a56926e805a0d047f817a993c28af71c9b79b840482da1ba5a", [:mix], [{:nimble_parsec, "~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5fda2b9176d7a71f52cca7f694d8ca75aed3f1b5b76dd175ada30b2756f96bae"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"},
   "tailwind": {:hex, :tailwind, "0.2.1", "83d8eadbe71a8e8f67861fe7f8d51658ecfb258387123afe4d9dc194eddc36b0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "e8a13f6107c95f73e58ed1b4221744e1eb5a093cd1da244432067e19c8c9a277"},
   "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+  "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "vix": {:hex, :vix, "0.22.0", "17efba59fa1a5d9cab36dbf066aa5d0a40d6b53f7d66380877392212aa7a39c6", [:make, :mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:cc_precompiler, "~> 0.1.4 or ~> 0.2", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7.3 or ~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}], "hexpm", "04fb539e881539f00eefde22997c3aa05d79aba695f440d2556ee2ad35237bdd"},
   "websock": {:hex, :websock, "0.5.2", "b3c08511d8d79ed2c2f589ff430bd1fe799bb389686dafce86d28801783d8351", [:mix], [], "hexpm", "925f5de22fca6813dfa980fb62fd542ec43a2d1a1f83d2caec907483fe66ff05"},