From 90adf5f1914f7472d7027ee36870f5bce61c2b4c Mon Sep 17 00:00:00 2001
From: APB9785 <74077809+APB9785@users.noreply.github.com>
Date: Tue, 10 Dec 2024 21:11:51 -0600
Subject: [PATCH] Generate robots.txt from beacon_site

---
 lib/beacon/router.ex                          |  1 +
 .../web/controllers/robots_controller.ex      | 20 ++++++++++++++
 lib/beacon/web/robots/robots.txt.eex          |  3 +++
 lib/beacon/web/robots_txt.ex                  |  4 +++
 .../controllers/robots_controller_test.exs    | 26 +++++++++++++++++++
 5 files changed, 54 insertions(+)
 create mode 100644 lib/beacon/web/controllers/robots_controller.ex
 create mode 100644 lib/beacon/web/robots/robots.txt.eex
 create mode 100644 lib/beacon/web/robots_txt.ex
 create mode 100644 test/beacon_web/controllers/robots_controller_test.exs

diff --git a/lib/beacon/router.ex b/lib/beacon/router.ex
index 0bc1f891..fe303132 100644
--- a/lib/beacon/router.ex
+++ b/lib/beacon/router.ex
@@ -149,6 +149,7 @@ defmodule Beacon.Router do
           get "/__beacon_assets__/js-:md5", Beacon.Web.AssetsController, :js, assigns: %{site: opts[:site]}
 
           get "/sitemap.xml", Beacon.Web.SitemapController, :show, as: :beacon_sitemap, assigns: %{site: opts[:site]}
+          get "/robots.txt", Beacon.Web.RobotsController, :show, as: :beacon_robots, assigns: %{site: opts[:site]}
 
           live "/*path", Beacon.Web.PageLive, :path
         end
diff --git a/lib/beacon/web/controllers/robots_controller.ex b/lib/beacon/web/controllers/robots_controller.ex
new file mode 100644
index 00000000..fd906a90
--- /dev/null
+++ b/lib/beacon/web/controllers/robots_controller.ex
@@ -0,0 +1,20 @@
+defmodule Beacon.Web.RobotsController do
+  @moduledoc false
+  use Beacon.Web, :controller
+
+  def init(:show), do: :show
+
+  def call(%{assigns: %{site: site}} = conn, :show) do
+    conn
+    |> put_view(Beacon.Web.RobotsTxt)
+    |> put_resp_content_type("text/txt")
+    |> put_resp_header("cache-control", "public max-age=300")
+    |> render(:robots, sitemap_url: get_sitemap_url(conn, site))
+  end
+
+  defp get_sitemap_url(conn, site) do
+    routes_module = Beacon.Loader.fetch_routes_module(site)
+
+    Beacon.apply_mfa(site, routes_module, :beacon_sitemap_url, [conn])
+  end
+end
diff --git a/lib/beacon/web/robots/robots.txt.eex b/lib/beacon/web/robots/robots.txt.eex
new file mode 100644
index 00000000..6a683c9f
--- /dev/null
+++ b/lib/beacon/web/robots/robots.txt.eex
@@ -0,0 +1,3 @@
+# http://www.robotstxt.org
+User-agent: *
+Sitemap: <%= @sitemap_url %>
diff --git a/lib/beacon/web/robots_txt.ex b/lib/beacon/web/robots_txt.ex
new file mode 100644
index 00000000..22fc32a4
--- /dev/null
+++ b/lib/beacon/web/robots_txt.ex
@@ -0,0 +1,4 @@
+defmodule Beacon.Web.RobotsTxt do
+  import Phoenix.Template, only: [embed_templates: 1]
+  embed_templates "robots/*.txt"
+end
diff --git a/test/beacon_web/controllers/robots_controller_test.exs b/test/beacon_web/controllers/robots_controller_test.exs
new file mode 100644
index 00000000..570a194d
--- /dev/null
+++ b/test/beacon_web/controllers/robots_controller_test.exs
@@ -0,0 +1,26 @@
+defmodule Beacon.Web.RobotsControllerTest do
+  use Beacon.Web.ConnCase, async: false
+
+  test "show", %{conn: conn} do
+    conn = get(conn, "/robots.txt")
+
+    assert response(conn, 200) == """
+           # http://www.robotstxt.org
+           User-agent: *
+           Sitemap: http://localhost:4000/sitemap.xml
+           """
+
+    assert response_content_type(conn, :txt) =~ "charset=utf-8"
+
+    # site: :not_booted
+    conn = get(conn, "/other/robots.txt")
+
+    assert response(conn, 200) == """
+           # http://www.robotstxt.org
+           User-agent: *
+           Sitemap: http://localhost:4000/other/sitemap.xml
+           """
+
+    assert response_content_type(conn, :txt) =~ "charset=utf-8"
+  end
+end