feat: support MDX components in partials (#28)

* feat: support MDX components in partials

* 3.1.1-canary.0

* 3.2.0

* 3.2.0-canary.0

* Clean up comments

* Rename option to resolveMdx

* docs: update README with notes on resolveMdx

* docs: update missing import in README

* docs: tweak language around accepted value for resolveMdx

* patch: fix issue with comment processing

* 3.2.0-canary.1

* refactor: use createMdxAstCompiler proper

* 3.2.0-canary.2

* Revert "refactor: use createMdxAstCompiler proper"

This reverts commit c6b41e2.

* 3.2.0-canary.3

* patch: fix issue with nested includes

* 3.2.0-canary.4

* docs: add comment on md-ast-to-mdx-ast

* Fix bungled heading include-markdown README

Co-authored-by: Jeff Escalante <[email protected]>

Co-authored-by: Jeff Escalante <[email protected]>

package.json

@@ -1,15 +1,17 @@
 {
   "name": "@hashicorp/remark-plugins",
   "description": "A potpourri of remark plugins used to process .mdx files",
-  "version": "3.1.1",
+  "version": "3.2.0-canary.4",
   "author": "Jeff Escalante",
   "bugs": "https://github.com/hashicorp/remark-plugins/issues",
   "contributors": [
     "Kevin Pruett"
   ],
   "dependencies": {
+    "@mdx-js/util": "1.6.22",
     "github-slugger": "^1.3.0",
-    "remark": "^12.0.1",
+    "remark": "12.0.1",
+    "remark-mdx": "1.6.22",
     "to-vfile": "^6.1.0",
     "unist-util-flatmap": "^1.0.0",
     "unist-util-is": "^4.0.2",

plugins/include-markdown/README.md

@@ -73,14 +73,59 @@ function sayHello(name) {
 
 ### Options
 
-This plugin accepts one optional config option: `resolveFrom`. If you pass this option along with a path, all partials will resolve from the path that was passed in. For example:
+This plugin accepts two optional config options: `resolveFrom` and `resolveMdx`.
+
+#### `resolveFrom`
+
+If you pass this option along with a path, all partials will resolve from the path that was passed in. For example:
 
 ```js
 remark().use(includeMarkdown, { resolveFrom: path.join(__dirname, 'partials') })
 ```
 
 With this config, you'd be able to put all your includes in a partials folder and require only based on the filename regardless of the location of your markdown file.
 
+#### `resolveMdx`
+
+If you pass `true` for this option, `.mdx` partials will be processed using [`remark-mdx`](https://github.com/mdx-js/mdx/tree/main/packages/remark-mdx). This allows the use of custom components within partials. For example, with `next-mdx-remote`:
+
+```js
+import { serialize } from 'next-mdx-remote/serialize'
+import { MDXRemote } from 'next-mdx-remote'
+import { includeMarkdown } from '@hashicorp/remark-plugins'
+import CustomComponent from '../components/custom-component'
+
+const components = { CustomComponent }
+
+export default function TestPage({ source }) {
+  return (
+    <div className="wrapper">
+      <MDXRemote {...source} components={components} />
+    </div>
+  )
+}
+
+export async function getStaticProps() {
+  // Imagine "included-file.mdx" has <CustomComponent /> in it...
+  // it will render as expected, since the @include extension
+  // is .mdx and resolveMdx is true.
+  const source = 'Some **mdx** text.\n\n@include "included-file.mdx"'
+  const mdxSource = await serialize(source, {
+    mdxOptions: {
+      remarkPlugins: [[includeMarkdown, { resolveMdx: true }]],
+    },
+  })
+  return { props: { source: mdxSource } }
+}
+```
+
+**Note**: this option should only be used in MDX contexts. This option will likely break where `remark-stringify` is used as the stringify plugin, such as when using `remark` directly.
+
+```js
+// 🚨 DON'T DO THIS - it will likely just break.
+// remark().use(includeMarkdown, { resolveMdx: true })
+```
+
 ### Ordering
 
 It's important to note that remark applies transforms in the order that they are called. If you want your other plugins to also apply to the contents of includeed files, you need to make sure that you apply the include content plugin **before all other plugins**. For example, let's say you have two plugins, one is this one to include markdown, and the other capitalizes all text, because yelling makes you more authoritative and also it's easier to read capitalized text. If you want to ensure that your includeed content is also capitalized, here's how you'd order your plugins:
@@ -95,4 +140,4 @@ If you order them the opposite way, like this:
 remark().use(capitalizeAllText).use(includeMarkdown)
 ```
 
-...what will happen is that all your text will be capitalized _except_ for the text in includeed files. And on top of that, the include plugin wouldn't resolve the files properly, because it capitalized the word "include", which is the wrong syntax. So usually you want to make sure this plugin comes first in your plugin stack.
+...what will happen is that all your text will be capitalized _except_ for the text in included files. And on top of that, the include plugin wouldn't resolve the files properly, because it capitalized the word "include", which is the wrong syntax. So usually you want to make sure this plugin comes first in your plugin stack.

plugins/include-markdown/fixtures/include-nested-component.mdx

@@ -0,0 +1,3 @@
+text at depth one
+
+@include 'nested/include-component.mdx'

plugins/include-markdown/fixtures/include-with-comment.mdx

@@ -0,0 +1,9 @@
+We should now be able include custom MDX components in partials. For example, a `"official"` `PluginTierLabel` should render below:
+
+<PluginTierLabel tier="official" />
+
+Comments should NOT mess things up:
+
+<!-- HTML comment but nested -->
+
+But they seem to be messing things up, apparently due to differences in `remark` 12 vs 13.

plugins/include-markdown/fixtures/include-with-component.mdx

@@ -0,0 +1,3 @@
+some text in an include
+
+<CustomComponent />

plugins/include-markdown/fixtures/nested/include-component.mdx

@@ -0,0 +1,3 @@
+some text in a nested include
+
+<NestedComponent />

plugins/include-markdown/index.js

@@ -1,9 +1,14 @@
 const path = require('path')
 const remark = require('remark')
+const remarkMdx = require('remark-mdx')
 const flatMap = require('unist-util-flatmap')
 const { readSync } = require('to-vfile')
+const mdAstToMdxAst = require('./md-ast-to-mdx-ast')
 
-module.exports = function includeMarkdownPlugin({ resolveFrom } = {}) {
+module.exports = function includeMarkdownPlugin({
+  resolveFrom,
+  resolveMdx,
+} = {}) {
   return function transformer(tree, file) {
     return flatMap(tree, (node) => {
       if (node.type !== 'paragraph') return [node]
@@ -32,8 +37,16 @@ module.exports = function includeMarkdownPlugin({ resolveFrom } = {}) {
       // if any other file type, they are embedded into a code block
       if (includePath.match(/\.md(?:x)?$/)) {
         // return the file contents in place of the @include
-        // this takes a couple steps because we allow recursive includes
-        const processor = remark().use(includeMarkdownPlugin, { resolveFrom })
+        // (takes a couple steps because we're processing includes with remark)
+        const processor = remark()
+        // if the include is MDX, and the plugin consumer has confirmed their
+        // ability to stringify MDX nodes (eg "jsx"), then use remarkMdx to support
+        // custom components (which would otherwise appear as likely invalid HTML nodes)
+        const isMdx = includePath.match(/\.mdx$/)
+        if (isMdx && resolveMdx) processor.use(remarkMdx).use(mdAstToMdxAst)
+        // use the includeMarkdown plugin to allow recursive includes
+        processor.use(includeMarkdownPlugin, { resolveFrom, resolveMdx })
+        // Process the file contents, then return them
         const ast = processor.parse(includeContents)
         return processor.runSync(ast, includeContents).children
       } else {

plugins/include-markdown/index.test.js

@@ -25,6 +25,69 @@ describe('include-markdown', () => {
       })
   })
 
+  test('include custom mdx components', () => {
+    // Set up a basic snippet as an mdast tree
+    const sourceMdx = `hello\n\n@include 'include-with-component.mdx'\n\nworld`
+    const rawTree = remark().parse(sourceMdx)
+    // Set up the includes plugin which will also run remark-mdx
+    const resolveFrom = path.join(__dirname, 'fixtures')
+    const tree = includeMarkdown({ resolveFrom, resolveMdx: true })(rawTree)
+    // Expect the custom component to appear in the resulting tree as JSX
+    expect(tree.children.length).toBe(4)
+    const [beforeP, includedText, includedComponent, afterP] = tree.children
+    expect(beforeP.children[0].value).toBe('hello')
+    expect(includedText.children[0].value).toBe('some text in an include')
+    expect(includedComponent.type).toBe('jsx')
+    expect(includedComponent.value).toBe('<CustomComponent />')
+    expect(afterP.children[0].value).toBe('world')
+  })
+
+  test('include nested custom mdx components', () => {
+    // Set up a basic snippet as an mdast tree
+    const sourceMdx = `hello\n\n@include 'include-nested-component.mdx'\n\nworld`
+    const rawTree = remark().parse(sourceMdx)
+    // Set up the includes plugin which will also run remark-mdx
+    const resolveFrom = path.join(__dirname, 'fixtures')
+    const tree = includeMarkdown({ resolveFrom, resolveMdx: true })(rawTree)
+    // Expect the custom component to appear in the resulting tree as JSX
+    expect(tree.children.length).toBe(5)
+    const [beforeP, includedText, nestedText, nestedComponent, afterP] =
+      tree.children
+    expect(beforeP.children[0].value).toBe('hello')
+    expect(includedText.children[0].value).toBe('text at depth one')
+    expect(nestedText.children[0].value).toBe('some text in a nested include')
+    expect(nestedComponent.value).toBe('<NestedComponent />')
+    expect(nestedComponent.type).toBe('jsx')
+    expect(afterP.children[0].value).toBe('world')
+  })
+
+  test('handles HTML comments when MDX is enabled', () => {
+    // Set up a basic snippet as an mdast tree
+    const sourceMdx = `<!-- HTML comment -->\n\n@include 'include-with-comment.mdx'\n\nworld`
+    const rawTree = remark().parse(sourceMdx)
+    // Set up the includes plugin which will also run remark-mdx
+    const resolveFrom = path.join(__dirname, 'fixtures')
+    const tree = includeMarkdown({ resolveFrom, resolveMdx: true })(rawTree)
+    // Expect the tree to have the right number of nodes
+    expect(tree.children.length).toBe(7)
+    // Expect the direct comment to be an HTML node,
+    // as we're not using md-ast-to-mdx-ast at this top level
+    // (though in our usual MDX contexts, we would be)
+    const directComment = tree.children[0]
+    expect(directComment.type).toBe('html')
+    expect(directComment.value).toBe('<!-- HTML comment -->')
+    // Expect the custom component in the include to be a JSX node
+    const customComponent = tree.children[2]
+    expect(customComponent.type).toBe('jsx')
+    expect(customComponent.value).toBe('<PluginTierLabel tier="official" />')
+    // Expect the comment in the include to be a comment node,
+    // as it has been parsed with remark-mdx and md-ast-to-mdx-ast,
+    // the latter of which transforms comments from "html" to "comment" nodes
+    const includedComment = tree.children[4]
+    expect(includedComment.type).toBe('comment')
+    expect(includedComment.value).toBe(' HTML comment but nested ')
+  })
+
   test('include non-markdown', () => {
     remark()
       .use(includeMarkdown)

plugins/include-markdown/md-ast-to-mdx-ast.js

@@ -0,0 +1,26 @@
+/*
+
+NOTE:
+This file is swiped directly from @mdxjs/mdx's createMdxAstCompiler.
+ref: https://github.com/mdx-js/mdx/blob/510bae2580958598ae29047bf755b1a2ea26cf7e/packages/mdx/md-ast-to-mdx-ast.js
+
+I considered the possibility of using createMdxAstCompiler rather than remark-mdx on its own.
+however, crucially, we do NOT want to transform our AST into a MDXAST, we ONLY want to
+transform custom component nodes (ie HTML that is really JSX) into JSX nodes.
+So it felt duplicative, but necessary, to copypasta this utility in to meet our needs.
+
+*/
+
+const visit = require('unist-util-visit')
+const { isComment, getCommentContents } = require('@mdx-js/util')
+
+module.exports = (_options) => (tree) => {
+  visit(tree, 'jsx', (node) => {
+    if (isComment(node.value)) {
+      node.type = 'comment'
+      node.value = getCommentContents(node.value)
+    }
+  })
+
+  return tree
+}