feat: Remove default "robots" meta (#1166)

cypress/e2e/norobots.spec.js

@@ -0,0 +1,38 @@
+describe('SEO Meta No Robots', () => {
+  it('App loads', () => {
+    cy.visit('http://localhost:3000');
+    cy.get('h1').should('contain', 'Default SEO');
+  });
+
+  it('SEO norobots', () => {
+    cy.visit('http://localhost:3000/norobots');
+    cy.get('head meta[name="robots"]').should('not.exist');
+  });
+
+  it('SEO overrides norobots with nofollow correctly', () => {
+    cy.visit('http://localhost:3000/norobots/nofollow');
+    cy.get('head meta[name="robots"]').should(
+      'have.attr',
+      'content',
+      'index,nofollow',
+    );
+  });
+
+  it('SEO overrides norobots with nofollow correctly', () => {
+    cy.visit('http://localhost:3000/norobots/noindex');
+    cy.get('head meta[name="robots"]').should(
+      'have.attr',
+      'content',
+      'noindex,follow',
+    );
+  });
+
+  it('SEO overrides norobots with robots props correctly', () => {
+    cy.visit('http://localhost:3000/norobots/robots');
+    cy.get('head meta[name="robots"]').should(
+      'have.attr',
+      'content',
+      'index,follow,nosnippet,max-snippet:-1,max-image-preview:none,noarchive,noimageindex,max-video-preview:-1,notranslate',
+    );
+  });
+});

e2e/pages/_app.tsx

@@ -16,6 +16,7 @@ function MyApp({ Component, pageProps, router }: AppProps) {
           router.pathname === '/dangerously/noindex' ||
           router.pathname === '/dangerously/nofollow-and-noindex'
         }
+        norobots={router.pathname.startsWith('/norobots')}
       />
       <Component {...pageProps} />
     </>

e2e/pages/norobots/index.tsx

@@ -0,0 +1,12 @@
+import React from 'react';
+import { NextSeo } from '../../..';
+import Links from '../../components/links';
+
+const NoRobots = () => (
+  <>
+    <NextSeo title="NoRobots" />
+    <h1>norobots</h1>
+    <Links />
+  </>
+);
+export default NoRobots;

e2e/pages/norobots/nofollow.tsx

@@ -0,0 +1,12 @@
+import React from 'react';
+import { NextSeo } from '../../..';
+import Links from '../../components/links';
+
+const NoRobotsNoFollow = () => (
+  <>
+    <NextSeo title="NoRobots NoFollow" nofollow />
+    <h1>norobots and nofollow</h1>
+    <Links />
+  </>
+);
+export default NoRobotsNoFollow;

e2e/pages/norobots/noindex.tsx

@@ -0,0 +1,12 @@
+import React from 'react';
+import { NextSeo } from '../../..';
+import Links from '../../components/links';
+
+const NoRobotsNoIndex = () => (
+  <>
+    <NextSeo title="NoRobots NoIndex" noindex />
+    <h1>norobots and noindex</h1>
+    <Links />
+  </>
+);
+export default NoRobotsNoIndex;

e2e/pages/norobots/robots.tsx

@@ -0,0 +1,24 @@
+import React from 'react';
+import { NextSeo } from '../../..';
+import Links from '../../components/links';
+
+const Robots = () => (
+  <>
+    <NextSeo
+      title="Robots meta title"
+      robotsProps={{
+        nosnippet: true,
+        notranslate: true,
+        noimageindex: true,
+        noarchive: true,
+        maxSnippet: -1,
+        maxImagePreview: 'none',
+        maxVideoPreview: -1,
+      }}
+    />
+    <h1>Norobots with Robots meta properties</h1>
+    <Links />
+  </>
+);
+
+export default Robots;

src/meta/buildTags.tsx

@@ -4,6 +4,7 @@ const defaults = {
   templateTitle: '',
   noindex: false,
   nofollow: false,
+  norobots: false,
   defaultOpenGraphImageWidth: 0,
   defaultOpenGraphImageHeight: 0,
   defaultOpenGraphVideoWidth: 0,
@@ -128,7 +129,10 @@ const buildTags = (config: BuildTagsParams) => {
       ? defaults.nofollow || config.dangerouslySetAllPagesToNoFollow
       : config.nofollow;
 
+  const norobots = config.norobots || defaults.norobots;
+
   let robotsParams = '';
+
   if (config.robotsProps) {
     const {
       nosnippet,
@@ -152,6 +156,10 @@ const buildTags = (config: BuildTagsParams) => {
     }`;
   }
 
+  if (config.norobots) {
+    defaults.norobots = true;
+  }
+
   if (noindex || nofollow) {
     if (config.dangerouslySetAllPagesToNoIndex) {
       defaults.noindex = true;
@@ -169,7 +177,7 @@ const buildTags = (config: BuildTagsParams) => {
         }${robotsParams}`}
       />,
     );
-  } else {
+  } else if (!norobots || robotsParams) {
     tagsToRender.push(
       <meta
         key="robots"

src/meta/defaultSEO.tsx

@@ -24,6 +24,7 @@ export const DefaultSeo = ({
   languageAlternates,
   additionalLinkTags,
   robotsProps,
+  norobots,
 }: DefaultSeoProps) => {
   return (
     <WithHead
@@ -47,6 +48,7 @@ export const DefaultSeo = ({
       languageAlternates={languageAlternates}
       additionalLinkTags={additionalLinkTags}
       robotsProps={robotsProps}
+      norobots={norobots}
     />
   );
 };

src/types.ts

@@ -498,6 +498,7 @@ export interface DefaultSeoProps {
   titleTemplate?: string;
   themeColor?: string;
   defaultTitle?: string;
+  norobots?: boolean;
   robotsProps?: AdditionalRobotsProps;
   description?: string;
   canonical?: string;