Baseline SEO improvements (#2157)

nuxt.config.ts

@@ -31,12 +31,9 @@ const meta = [
     name: "viewport",
     content: "width=device-width,initial-scale=1",
   },
-  // By default, tell all robots not to index pages. Will be overridden in the
+  // By default, tell all robots not to index pages. Will be overwritten in the
   // search, content and home pages.
   { hid: "robots", name: "robots", content: "noindex" },
-  // Tell Googlebot to crawl Openverse when iframed
-  // TODO: remove after the iframe is removed
-  { hid: "googlebot", name: "googlebot", content: "indexifembedded" },
   {
     vmid: "monetization",
     name: "monetization",
@@ -204,8 +201,10 @@ const config: NuxtConfig = {
   ],
   serverMiddleware: [
     { path: "/healthcheck", handler: "~/server-middleware/healthcheck.js" },
+    { path: "/robots.txt", handler: "~/server-middleware/robots.js" },
   ],
   i18n: {
+    baseUrl: "https://openverse.org",
     locales: [
       {
         // unique identifier for the locale in Vue i18n

src/layouts/content-layout.vue

@@ -199,7 +199,10 @@ export default defineComponent({
     }
   },
   head() {
-    return this.$nuxtI18nHead({ addSeoAttributes: true, addDirAttribute: true })
+    return this.$nuxtI18nHead({
+      addSeoAttributes: true,
+      addDirAttribute: true,
+    })
   },
 })
 </script>

src/layouts/default.vue

@@ -71,7 +71,10 @@ export default defineComponent({
     }
   },
   head() {
-    return this.$nuxtI18nHead({ addSeoAttributes: true, addDirAttribute: true })
+    return this.$nuxtI18nHead({
+      addSeoAttributes: true,
+      addDirAttribute: true,
+    })
   },
 })
 </script>

src/pages/search/audio.vue

@@ -49,8 +49,6 @@ import { useUiStore } from "~/stores/ui"
 
 import { IsSidebarVisibleKey } from "~/types/provides"
 
-import { useFeatureFlagStore } from "~/stores/feature-flag"
-
 import VSnackbar from "~/components/VSnackbar.vue"
 import VAudioTrack from "~/components/VAudioTrack/VAudioTrack.vue"
 import VLoadMore from "~/components/VLoadMore.vue"
@@ -68,13 +66,8 @@ export default defineComponent({
   },
   props: propTypes,
   setup(props) {
-    const featureFlagStore = useFeatureFlagStore()
-
     useMeta({
       title: `${props.searchTerm} | Openverse`,
-      meta: featureFlagStore.isOn("new_header")
-        ? [{ hid: "robots", name: "robots", content: "all" }]
-        : undefined,
     })
 
     const uiStore = useUiStore()

src/pages/search/image.vue

@@ -12,7 +12,6 @@ import { computed, defineComponent, useMeta } from "@nuxtjs/composition-api"
 
 import { propTypes } from "~/pages/search/search-page.types"
 import { useFocusFilters } from "~/composables/use-focus-filters"
-import { useFeatureFlagStore } from "~/stores/feature-flag"
 
 import VImageGrid from "~/components/VImageGrid/VImageGrid.vue"
 
@@ -21,13 +20,8 @@ export default defineComponent({
   components: { VImageGrid },
   props: propTypes,
   setup(props) {
-    const featureFlagStore = useFeatureFlagStore()
-
     useMeta({
       title: `${props.searchTerm} | Openverse`,
-      meta: featureFlagStore.isOn("new_header")
-        ? [{ hid: "robots", name: "robots", content: "all" }]
-        : undefined,
     })
 
     const results = computed(() => props.resultItems.image)

src/pages/search/index.vue

@@ -6,7 +6,6 @@
 import { defineComponent, useMeta } from "@nuxtjs/composition-api"
 
 import { propTypes } from "~/pages/search/search-page.types"
-import { useFeatureFlagStore } from "~/stores/feature-flag"
 
 import VAllResultsGrid from "~/components/VAllResultsGrid/VAllResultsGrid.vue"
 
@@ -15,13 +14,8 @@ export default defineComponent({
   components: { VAllResultsGrid },
   props: propTypes,
   setup(props) {
-    const featureFlagStore = useFeatureFlagStore()
-
     useMeta({
       title: `${props.searchTerm} | Openverse`,
-      meta: featureFlagStore.isOn("new_header")
-        ? [{ hid: "robots", name: "robots", content: "all" }]
-        : undefined,
     })
   },
   head: {},

src/server-middleware/robots.js

@@ -0,0 +1,19 @@
+const { LOCAL, PRODUCTION } = require("../constants/deploy-env")
+
+/**
+ * Send the correct robots.txt information per-environment.
+ */
+export default function robots(_, res) {
+  const deployEnv = process.env.DEPLOYMENT_ENV ?? LOCAL
+
+  const contents =
+    deployEnv === PRODUCTION
+      ? `# This file is intentionally left blank`
+      : `# Block crawlers from the staging site
+User-agent: *
+Disallow: /
+`
+
+  res.setHeader("Content-Type", "text/plain")
+  res.end(contents)
+}