add robots.txt

xpack-dev-tools · Oct 7, 2024 · 3673425 · 3673425
1 parent 01fd23b
commit 3673425
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 1 deletion.
diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts
@@ -136,6 +136,18 @@ const config: Config = {
         theme: {
           customCss: './src/css/custom.css',
         },
+        // https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-sitemap
+        sitemap: {
+          lastmod: 'date',
+          changefreq: 'weekly',
+          priority: 0.5,
+          ignorePatterns: [
+            '/blog/archive/**',
+            '/blog/authors/**',
+            '/blog/tags/**'
+          ],
+          filename: 'sitemap.xml',
+        },
         // https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-google-gtag
         // https://tagassistant.google.com
         gtag: {

diff --git a/website/src/components/HomepageTools/index.tsx b/website/src/components/HomepageTools/index.tsx
@@ -18,6 +18,7 @@ import styles from './styles.module.css';
 
 import Link from '@docusaurus/Link';
 
+/* When updated, don't forget to add sitemap to robots.txt! */
 const PrimaryTools = [
   {
     appName: 'GNU AArch64 Embedded GCC',
@@ -73,7 +74,7 @@ const PrimaryTools = [
   },
 ]
 
-
+/* When updated, don't forget to add sitemap to robots.txt! */
 const SecondaryTools = [
   {
     appName: 'GNU bison',

diff --git a/website/static/robots.txt b/website/static/robots.txt
@@ -0,0 +1,26 @@
+Sitemap: xpack-dev-tools.github.io/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/aarch64-none-elf-gcc-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/arm-none-eabi-gcc-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/clang-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/cmake-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/gcc-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/meson-build-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/mingw-w64-gcc-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/ninja-build-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/openocd-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/qemu-arm-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/qemu-riscv-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/riscv-none-elf-gcc-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/windows-build-tools-xpack/sitemap.xml
+
+Sitemap: xpack-dev-tools.github.io/bison-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/flex-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/m4-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/patchelf-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/pkg-config-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/realpath-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/sed-xpack/sitemap.xml
+Sitemap: xpack-dev-tools.github.io/wine-xpack/sitemap.xml
+
+User-agent: *
+Allow: /