add html parsing to docs (#308)

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "editor.formatOnSave": false,
+  "editor.formatOnSave": true,
   "editor.codeActionsOnSave": ["source.formatDocument", "source.fixAll.eslint"],
   "search.exclude": {
     "**/.yarn": true,

next.config.js

@@ -1,16 +1,14 @@
 /* eslint-disable @typescript-eslint/no-var-requires */
 // eslint-disable-next-line function-paren-newline
-const withMarkdoc = require('@markdoc/next.js')(
-  /* config: https://markdoc.io/docs/nextjs#options */ {
-    schemaPath: './src/markdoc',
-  })
-const withTM = require('next-transpile-modules')(['@pluralsh/design-system', 'honorable', 'honorable-theme-default'],
+const withTM = require('next-transpile-modules')(
+  ['@pluralsh/design-system', 'honorable', 'honorable-theme-default'],
   {
     debug: false,
-  })
+  }
+)
 
 module.exports = () => {
-  const plugins = [withMarkdoc, withTM]
+  const plugins = [withTM]
 
   return plugins.reduce((acc, next) => next(acc), {
     reactStrictMode: false,
@@ -23,7 +21,14 @@ module.exports = () => {
       locales: ['en-US'],
       defaultLocale: 'en-US',
     },
-    pageExtensions: ['js', 'jsx', 'ts', 'tsx', 'md', 'mdoc'],
+    pageExtensions: ['js', 'jsx', 'ts', 'tsx'],
+    webpack: (config) => {
+      config.module.rules.push({
+        test: /\.md$/,
+        use: 'raw-loader',
+      })
+      return config
+    },
     async redirects() {
       return [
         {
@@ -53,12 +58,14 @@ module.exports = () => {
         },
         {
           source: '/getting-started/getting-started-with-runbooks/runbook-yaml',
-          destination: '/adding-new-application/getting-started-with-runbooks/runbook-yaml',
+          destination:
+            '/adding-new-application/getting-started-with-runbooks/runbook-yaml',
           permanent: true,
         },
         {
           source: '/basic-setup-and-deployment/setting-up-gitops',
-          destination: '/getting-started/managing-git-repository/setting-up-gitops',
+          destination:
+            '/getting-started/managing-git-repository/setting-up-gitops',
           permanent: true,
         },
         {
@@ -87,7 +94,8 @@ module.exports = () => {
           permanent: true,
         },
         {
-          source: '/reference/operator-guides/adding-kubecost-for-cost-analysis',
+          source:
+            '/reference/operator-guides/adding-kubecost-for-cost-analysis',
           destination: '/repositories/kubecost',
           permanent: true,
         },
@@ -132,8 +140,10 @@ module.exports = () => {
           permanent: true,
         },
         {
-          source: '/advanced-topics/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
-          destination: '/operations/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
+          source:
+            '/advanced-topics/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
+          destination:
+            '/operations/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
           permanent: true,
         },
         {
@@ -143,7 +153,8 @@ module.exports = () => {
         },
         {
           source: '/advanced-topics/security/secret-management',
-          destination: '/getting-started/manage-git-repositories/sharing-git-repositories',
+          destination:
+            '/getting-started/manage-git-repositories/sharing-git-repositories',
           permanent: true,
         },
         {
@@ -172,12 +183,14 @@ module.exports = () => {
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/introduction',
+          source:
+            '/advanced-topics/identity-and-access-management/introduction',
           destination: '/operations/auth-access-control',
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/openid-connect',
+          source:
+            '/advanced-topics/identity-and-access-management/openid-connect',
           destination: '/operations/auth-access-control/openid-connect',
           permanent: true,
         },
@@ -187,28 +200,37 @@ module.exports = () => {
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/identity-and-installations',
-          destination: '/operations/auth-access-control/identity-and-installations',
+          source:
+            '/advanced-topics/identity-and-access-management/identity-and-installations',
+          destination:
+            '/operations/auth-access-control/identity-and-installations',
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/identity-and-installations/audit-logging',
-          destination: '/operations/auth-access-control/identity-and-installations/audit-logging',
+          source:
+            '/advanced-topics/identity-and-access-management/identity-and-installations/audit-logging',
+          destination:
+            '/operations/auth-access-control/identity-and-installations/audit-logging',
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/identity-and-installations/service-accounts',
-          destination: '/operations/auth-access-control/identity-and-installations/service-accounts',
+          source:
+            '/advanced-topics/identity-and-access-management/identity-and-installations/service-accounts',
+          destination:
+            '/operations/auth-access-control/identity-and-installations/service-accounts',
           permanent: true,
         },
         {
-          source: '/advanced-topics/identity-and-access-management/identity-and-installations/sharing-existing-repos',
-          destination: '/getting-started/manage-git-repositories/sharing-git-repository',
+          source:
+            '/advanced-topics/identity-and-access-management/identity-and-installations/sharing-existing-repos',
+          destination:
+            '/getting-started/manage-git-repositories/sharing-git-repository',
           permanent: true,
         },
         {
           source: '/reference/workspaces',
-          destination: '/getting-started/manage-git-repositories/your-plural-workspace',
+          destination:
+            '/getting-started/manage-git-repositories/your-plural-workspace',
           permanent: true,
         },
       ]

package.json

@@ -48,6 +48,7 @@
     "graphql": "16.6.0",
     "honorable": "0.194.0",
     "honorable-theme-default": "0.77.0",
+    "htmlparser2": "9.1.0",
     "immer": "10.0.2",
     "js-yaml": "4.1.0",
     "lodash": "4.17.21",

pages/[...slug].tsx

@@ -0,0 +1,98 @@
+import fs from 'fs'
+import path from 'path'
+import { type ParsedUrlQuery } from 'querystring'
+
+import { type GetStaticPaths, type GetStaticProps } from 'next'
+
+import MarkdocComponent from '@src/components/MarkdocContent'
+import { readMdFileCached } from '@src/markdoc/mdParser'
+import { type MarkdocPage } from '@src/markdoc/mdSchema'
+
+interface Params extends ParsedUrlQuery {
+  slug: string[]
+}
+
+export default function MarkdocContent({
+  markdoc,
+}: {
+  markdoc: MarkdocPage | null
+}) {
+  return markdoc && <MarkdocComponent markdoc={markdoc} />
+}
+
+export const getStaticPaths: GetStaticPaths = async () => {
+  const pagesDirectory = path.join('pages')
+
+  // recursively get all .md files in the 'pages/' directory
+  function getAllMarkdownFiles(dir: string, files: string[] = []): string[] {
+    const entries = fs.readdirSync(dir, { withFileTypes: true })
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name)
+
+      if (entry.isDirectory()) {
+        getAllMarkdownFiles(fullPath, files)
+      } else if (entry.isFile() && entry.name.endsWith('.md')) {
+        files.push(fullPath)
+      }
+    }
+
+    return files
+  }
+
+  const markdownFiles = getAllMarkdownFiles(pagesDirectory)
+
+  const paths = markdownFiles.map((file) => {
+    const relativePath = path.relative(pagesDirectory, file)
+    const parsedPath = path.parse(relativePath)
+
+    const dirSegments = parsedPath.dir ? parsedPath.dir.split(path.sep) : []
+
+    let slug: string[]
+
+    if (parsedPath.name === 'index') slug = dirSegments
+    else slug = [...dirSegments, parsedPath.name]
+
+    return {
+      params: {
+        slug,
+      },
+    }
+  })
+
+  return {
+    paths,
+    fallback: 'blocking',
+  }
+}
+
+export const getStaticProps: GetStaticProps<
+  { markdoc: MarkdocPage | null },
+  Params
+> = async ({ params }) => {
+  if (!params?.slug) {
+    return { notFound: true }
+  }
+
+  const slugPath = params.slug.join('/')
+
+  // looks for folder/name/index.md first, then folder/name.md
+  const filePath =
+    [
+      path.join('pages', slugPath, 'index.md'),
+      path.join('pages', `${slugPath}.md`),
+    ].find(fs.existsSync) || null
+
+  if (!filePath) {
+    return { notFound: true }
+  }
+  const markdoc = await readMdFileCached(filePath)
+
+  return {
+    props: {
+      displayTitle: markdoc?.frontmatter?.title ?? '',
+      displayDescription: markdoc?.frontmatter?.description ?? '',
+      markdoc,
+    },
+  }
+}

pages/how-to/set-up/mgmt-cluster.md

@@ -46,4 +46,4 @@ There are a few reasons you'd consider using this over Plural Cloud:
 * Integration - Oftentimes resources needed by Plural are themselves hosted on private networks, for instance Git Repositories.  In that case, it's logistically easier to self-host and place it in an integrated network. 
 * Scaling - you want complete control as to how Plural Scales for your enterprise.  `dedicated` cloud hosting does this perfectly well too, but some orgs want their own hands on the wheel.
 
-Plural is meant to be architecturally simple and efficient.  Most organizations that do chose to self-host are shocked at how streamlined managing it is, especially compared to some more bloated CNCF projects, so it is a surprisingly viable way to manage the software if that is what your organization desires.
+Plural is meant to be architecturally simple and efficient.  Most organizations that do choose to self-host are shocked at how streamlined managing it is, especially compared to some more bloated CNCF projects, so it is a surprisingly viable way to manage the software if that is what your organization desires.

src/markdoc/mdParser.ts

@@ -2,6 +2,7 @@ import { promises as fs } from 'fs'
 import path from 'path'
 
 import Markdoc from '@markdoc/markdoc'
+import { Parser } from 'htmlparser2'
 import yaml from 'js-yaml'
 
 import { config as schemaConfig } from './mdSchema'
@@ -13,7 +14,7 @@ const fileCache = new Map<string, MarkdocPage | null>()
 export const readMdFileCached = async (
   filePath: string
 ): Promise<MarkdocPage | null> => {
-  if (!filePath.startsWith('/pages')) {
+  if (!filePath.startsWith('pages')) {
     return null
   }
   function cacheAndReturn(val: MarkdocPage | null) {
@@ -36,18 +37,23 @@ export const readMdFileCached = async (
       return cacheAndReturn(null)
     }
 
-    const ast = Markdoc.parse(file)
+    const tokenizer = new Markdoc.Tokenizer({ html: true })
+    const tokens = tokenizer.tokenize(file)
+    const processed = processHtmlTokens(tokens)
+    const ast = Markdoc.parse(processed)
+
     const frontmatter = ast.attributes.frontmatter
       ? yaml.load(ast.attributes.frontmatter)
       : {}
+
     const content = Markdoc.transform(ast, schemaConfig)
 
     const ret: MarkdocPage = JSON.parse(
       JSON.stringify({
         content,
         frontmatter,
         file: {
-          path: filePath.replace(/^\/pages/g, ''),
+          path: filePath.replace(/^pages/g, ''),
         },
       })
     )
@@ -59,3 +65,52 @@ export const readMdFileCached = async (
     return cacheAndReturn(null)
   }
 }
+
+// see https://github.com/markdoc/markdoc/issues/10
+// https://gist.github.com/rpaul-stripe/941eb22c4779ea87b1adf7715d76ca08
+function processHtmlTokens(tokens) {
+  const output: any[] = []
+
+  const parser = new Parser({
+    onopentag(name, attrs) {
+      output.push({
+        type: 'tag_open',
+        nesting: 1,
+        meta: {
+          tag: 'html-tag',
+          attributes: [
+            { type: 'attribute', name: 'name', value: name },
+            { type: 'attribute', name: 'attrs', value: attrs },
+          ],
+        },
+      })
+    },
+
+    ontext(content) {
+      if (typeof content === 'string' && content.trim().length > 0)
+        output.push({ type: 'text', content })
+    },
+
+    onclosetag() {
+      output.push({
+        type: 'tag_close',
+        nesting: -1,
+        meta: { tag: 'html-tag' },
+      })
+    },
+  })
+
+  for (const token of tokens) {
+    if (token.type.startsWith('html')) {
+      parser.write(token.content)
+      continue
+    }
+
+    if (token.type === 'inline')
+      token.children = processHtmlTokens(token.children)
+
+    output.push(token)
+  }
+
+  return output
+}

src/markdoc/mdSchema.ts

@@ -3,8 +3,8 @@ import merge from 'lodash/merge'
 
 import * as config from './config'
 import * as functions from './functions'
-import * as nodes from './nodes'
-import * as tags from './tags'
+import { nodes } from './nodes'
+import { tags } from './tags'
 
 import type { MarkdocNextJsPageProps } from '@markdoc/next.js'