From 262d8d8e9d5826c2c3bd50ca63d7945d4765bd51 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 9 Jul 2024 08:03:25 -0700 Subject: [PATCH 01/11] Spans and shortcodes --- packages/markdown-it-myst/src/index.ts | 6 +- packages/markdown-it-myst/src/roles.ts | 3 +- packages/markdown-it-myst/src/shortcode.ts | 31 +++++++++++ packages/markdown-it-myst/src/span.ts | 35 ++++++++++++ .../markdown-it-myst/tests/shortcode.spec.ts | 55 +++++++++++++++++++ packages/markdown-it-myst/tests/span.spec.ts | 25 +++++++++ packages/myst-parser/src/tokensToMyst.ts | 2 + 7 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 packages/markdown-it-myst/src/shortcode.ts create mode 100644 packages/markdown-it-myst/src/span.ts create mode 100644 packages/markdown-it-myst/tests/shortcode.spec.ts create mode 100644 packages/markdown-it-myst/tests/span.spec.ts diff --git a/packages/markdown-it-myst/src/index.ts b/packages/markdown-it-myst/src/index.ts index 95ebd9b79..e468d6865 100644 --- a/packages/markdown-it-myst/src/index.ts +++ b/packages/markdown-it-myst/src/index.ts @@ -2,13 +2,17 @@ import type MarkdownIt from 'markdown-it/lib'; import { rolePlugin } from './roles.js'; import { directivePlugin } from './directives.js'; import { citationsPlugin } from './citations.js'; +import { shortcodePlugin } from './shortcode.js'; +import { spanPlugin } from './span.js'; -export { rolePlugin, directivePlugin, citationsPlugin }; +export { rolePlugin, directivePlugin, citationsPlugin, shortcodePlugin }; /** * A markdown-it plugin for parsing MyST roles and directives to structured data */ export function mystPlugin(md: MarkdownIt): void { + md.use(shortcodePlugin); + md.use(spanPlugin); md.use(rolePlugin); md.use(directivePlugin); } diff --git a/packages/markdown-it-myst/src/roles.ts b/packages/markdown-it-myst/src/roles.ts index 4621a8af7..12077209b 100644 --- a/packages/markdown-it-myst/src/roles.ts +++ b/packages/markdown-it-myst/src/roles.ts @@ -63,13 +63,14 @@ function runRoles(state: StateCore): boolean { if (child.type === 'role') { try { const { map } = token; - const { content, col } = child as any; + const { content, col, meta } = child as any; const roleOpen = new state.Token('parsed_role_open', '', 1); roleOpen.content = content; roleOpen.hidden = true; roleOpen.info = child.meta.name; roleOpen.block = false; roleOpen.map = map; + roleOpen.meta = meta; (roleOpen as any).col = col; const contentTokens = roleContentToTokens(content, map ? map[0] : 0, state); const roleClose = new state.Token('parsed_role_close', '', -1); diff --git a/packages/markdown-it-myst/src/shortcode.ts b/packages/markdown-it-myst/src/shortcode.ts new file mode 100644 index 000000000..7e2a8fd2e --- /dev/null +++ b/packages/markdown-it-myst/src/shortcode.ts @@ -0,0 +1,31 @@ +import type MarkdownIt from 'markdown-it/lib'; +import type StateCore from 'markdown-it/lib/rules_core/state_core.js'; +import type StateInline from 'markdown-it/lib/rules_inline/state_inline.js'; +import { nestedPartToTokens } from './nestedParse.js'; + +export function shortcodePlugin(md: MarkdownIt): void { + md.inline.ruler.before('backticks', 'parse_short_codes', shortCodeRule); +} + +// Hugo short code syntax e.g. {{< role value >}} +const ROLE_PATTERN = /^\{\{\<\s*([a-z0-9_\-+:]{1,36})\s*([^>]*)\s*\>\}\}/; + +function shortCodeRule(state: StateInline, silent: boolean): boolean { + // Check if the role is escaped + if (state.src.charCodeAt(state.pos - 1) === 0x5c) { + /* \ */ + // TODO: this could be improved in the case of edge case '\\{', also multi-line + return false; + } + const match = ROLE_PATTERN.exec(state.src.slice(state.pos)); + if (match == null) return false; + const [str, name, content] = match; + if (!silent) { + const token = state.push('role', '', 0); + token.meta = { name }; + token.content = content?.trim(); + (token as any).col = [state.pos, state.pos + str.length]; + } + state.pos += str.length; + return true; +} diff --git a/packages/markdown-it-myst/src/span.ts b/packages/markdown-it-myst/src/span.ts new file mode 100644 index 000000000..28f10349c --- /dev/null +++ b/packages/markdown-it-myst/src/span.ts @@ -0,0 +1,35 @@ +import type MarkdownIt from 'markdown-it/lib'; +import type StateCore from 'markdown-it/lib/rules_core/state_core.js'; +import type StateInline from 'markdown-it/lib/rules_inline/state_inline.js'; +import { nestedPartToTokens } from './nestedParse.js'; + +export function spanPlugin(md: MarkdownIt): void { + md.inline.ruler.before('backticks', 'parse_span', spanRule); +} + +// Inline span syntax e.g. [markdown]{.class} +const ROLE_PATTERN = /^\[([^\]]*)\]\{([^\}]*)\}/; + +function spanRule(state: StateInline, silent: boolean): boolean { + // Check if the role is escaped + if (state.src.charCodeAt(state.pos - 1) === 0x5c) { + /* \ */ + // TODO: this could be improved in the case of edge case '\\[', also multi-line + return false; + } + const match = ROLE_PATTERN.exec(state.src.slice(state.pos)); + if (match == null) return false; + const [str, content, options] = match; + if (!silent) { + const token = state.push('role', '', 0); + const classes = options + .split(' ') + .map((c) => c.trim().replace(/^\./, '')) + .filter((c) => !!c); + token.meta = { name: 'span', options: { class: classes.join(' ') } }; + token.content = content?.trim(); + (token as any).col = [state.pos, state.pos + str.length]; + } + state.pos += str.length; + return true; +} diff --git a/packages/markdown-it-myst/tests/shortcode.spec.ts b/packages/markdown-it-myst/tests/shortcode.spec.ts new file mode 100644 index 000000000..728d19be7 --- /dev/null +++ b/packages/markdown-it-myst/tests/shortcode.spec.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; +import MarkdownIt from 'markdown-it'; +import plugin from '../src'; + +describe('parses roles', () => { + it('basic role parses', () => { + const mdit = MarkdownIt().use(plugin); + const tokens = mdit.parse('ok {{< var lang >}}', {}); + expect(tokens.map((t) => t.type)).toEqual(['paragraph_open', 'inline', 'paragraph_close']); + expect(tokens[1].children?.map((t) => t.type)).toEqual([ + 'text', + 'parsed_role_open', + 'role_body_open', + 'inline', + 'role_body_close', + 'parsed_role_close', + ]); + expect(tokens[1].content).toEqual('ok {{< var lang >}}'); + // Pass the column information for the role + expect((tokens[1].children?.[1] as any).col).toEqual([3, 19]); + expect(tokens[1].children?.[1].info).toEqual('var'); + expect(tokens[1].children?.[1].content).toEqual('lang'); + expect(tokens[1].children?.[3].content).toEqual('lang'); + }); + it('basic role parses', () => { + const mdit = MarkdownIt().use(plugin); + const content = `Notice that the value for \`some_numbers\` is {{< var np_or_r >}}, +and that this value *contains* 10 numbers.`; + const tokens = mdit.parse(content, {}); + expect(tokens.map((t) => t.type)).toEqual(['paragraph_open', 'inline', 'paragraph_close']); + expect(tokens[1].children?.map((t) => t.type)).toEqual([ + 'text', + 'code_inline', + 'text', + 'parsed_role_open', + 'role_body_open', + 'inline', + 'role_body_close', + 'parsed_role_close', + 'text', + 'softbreak', + 'text', + 'em_open', + 'text', + 'em_close', + 'text', + ]); + expect(tokens[1].content).toEqual(content); + // Pass the column information for the role + expect((tokens[1].children?.[3] as any).col).toEqual([44, 63]); + expect(tokens[1].children?.[3].info).toEqual('var'); + expect(tokens[1].children?.[3].content).toEqual('np_or_r'); + expect(tokens[1].children?.[3].content).toEqual('np_or_r'); + }); +}); diff --git a/packages/markdown-it-myst/tests/span.spec.ts b/packages/markdown-it-myst/tests/span.spec.ts new file mode 100644 index 000000000..87f83c7fc --- /dev/null +++ b/packages/markdown-it-myst/tests/span.spec.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import MarkdownIt from 'markdown-it'; +import plugin from '../src'; + +describe('parses spans', () => { + it('basic span parses', () => { + const mdit = MarkdownIt().use(plugin); + const tokens = mdit.parse('ok [content]{.python}', {}); + expect(tokens.map((t) => t.type)).toEqual(['paragraph_open', 'inline', 'paragraph_close']); + expect(tokens[1].children?.map((t) => t.type)).toEqual([ + 'text', + 'parsed_role_open', + 'role_body_open', + 'inline', + 'role_body_close', + 'parsed_role_close', + ]); + expect(tokens[1].content).toEqual('ok [content]{.python}'); + // Pass the column information for the role + expect((tokens[1].children?.[1] as any).col).toEqual([3, 21]); + expect(tokens[1].children?.[1].info).toEqual('span'); + expect(tokens[1].children?.[1].content).toEqual('content'); + expect(tokens[1].children?.[3].content).toEqual('content'); + }); +}); diff --git a/packages/myst-parser/src/tokensToMyst.ts b/packages/myst-parser/src/tokensToMyst.ts index 9a4e37d06..be5f71465 100644 --- a/packages/myst-parser/src/tokensToMyst.ts +++ b/packages/myst-parser/src/tokensToMyst.ts @@ -416,9 +416,11 @@ const defaultMdast: Record = { parsed_role: { type: 'mystRole', getAttrs(t) { + if (t.info !== 'var') console.log({ role: t }); return { name: t.info, value: t.content, + options: t.meta?.options, processed: false, }; }, From 91bd52f248fa7c4cc256bde05b054266a7240072 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 08:15:02 -0700 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=8F=B7=20Add=20label=20markdown-it?= =?UTF-8?q?=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 31 +++++++++++++++++++ packages/markdown-it-myst/src/index.ts | 5 ++- packages/markdown-it-myst/src/labels.ts | 29 +++++++++++++++++ packages/markdown-it-myst/tests/cases.spec.ts | 6 ++-- packages/markdown-it-myst/tests/labels.yml | 22 +++++++++++++ packages/myst-parser/src/fromMarkdown.ts | 1 + packages/myst-parser/src/myst.ts | 7 +++++ packages/myst-transforms/package.json | 1 + packages/myst-transforms/src/targets.ts | 28 +++++++++++++---- 9 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 packages/markdown-it-myst/src/labels.ts create mode 100644 packages/markdown-it-myst/tests/labels.yml diff --git a/package-lock.json b/package-lock.json index 504479b8c..ddfa9866e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13617,6 +13617,36 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-util-find-before": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-before/-/unist-util-find-before-4.0.0.tgz", + "integrity": "sha512-bdXFv1xM0O19Gn8B7mJjXuVOyVSdE7eQZs4Ulo4EWOBb8OihHLpCDhOLwDGjrpMZCRPlmpUhxMMqlDohvswEsA==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-find-before/node_modules/@types/unist": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", + "integrity": "sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==" + }, + "node_modules/unist-util-find-before/node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-generated": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unist-util-generated/-/unist-util-generated-2.0.1.tgz", @@ -15884,6 +15914,7 @@ "unified": "^10.0.0", "unist-builder": "^3.0.0", "unist-util-find-after": "^4.0.0", + "unist-util-find-before": "4.0.0", "unist-util-map": "^3.0.0", "unist-util-modify-children": "^3.1.0", "unist-util-remove": "^3.1.0", diff --git a/packages/markdown-it-myst/src/index.ts b/packages/markdown-it-myst/src/index.ts index e468d6865..296f96488 100644 --- a/packages/markdown-it-myst/src/index.ts +++ b/packages/markdown-it-myst/src/index.ts @@ -2,17 +2,16 @@ import type MarkdownIt from 'markdown-it/lib'; import { rolePlugin } from './roles.js'; import { directivePlugin } from './directives.js'; import { citationsPlugin } from './citations.js'; +import { labelsPlugin } from './labels.js'; import { shortcodePlugin } from './shortcode.js'; import { spanPlugin } from './span.js'; -export { rolePlugin, directivePlugin, citationsPlugin, shortcodePlugin }; +export { rolePlugin, directivePlugin, citationsPlugin, shortcodePlugin, labelsPlugin }; /** * A markdown-it plugin for parsing MyST roles and directives to structured data */ export function mystPlugin(md: MarkdownIt): void { - md.use(shortcodePlugin); - md.use(spanPlugin); md.use(rolePlugin); md.use(directivePlugin); } diff --git a/packages/markdown-it-myst/src/labels.ts b/packages/markdown-it-myst/src/labels.ts new file mode 100644 index 000000000..8b55a6f8b --- /dev/null +++ b/packages/markdown-it-myst/src/labels.ts @@ -0,0 +1,29 @@ +import MarkdownIt from 'markdown-it/lib'; +import StateInline from 'markdown-it/lib/rules_inline/state_inline.js'; + +const LABEL_PATTERN = /^\{\#(.+?)\}/im; + +const LABEL_TOKEN_NAME = 'myst_target'; + +function labelRule(state: StateInline, silent: boolean): boolean { + // Check if the label is escaped + if (state.src.charCodeAt(state.pos - 1) === 0x5c) { + /* \ */ + // TODO: this could be improved in the case of edge case '\\{', also multi-line + return false; + } + const match = LABEL_PATTERN.exec(state.src.slice(state.pos)); + if (match == null) return false; + const [str, content] = match; + if (!silent) { + const token = state.push(LABEL_TOKEN_NAME, '', 0); + token.content = content; + (token as any).col = [state.pos, state.pos + str.length]; + } + state.pos += str.length; + return true; +} + +export function labelsPlugin(md: MarkdownIt): void { + md.inline.ruler.before('backticks', `parse_${LABEL_TOKEN_NAME}`, labelRule); +} diff --git a/packages/markdown-it-myst/tests/cases.spec.ts b/packages/markdown-it-myst/tests/cases.spec.ts index bda36705e..040a405aa 100644 --- a/packages/markdown-it-myst/tests/cases.spec.ts +++ b/packages/markdown-it-myst/tests/cases.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, test, vi } from 'vitest'; import type Token from 'markdown-it/lib/token'; -import { citationsPlugin } from '../src'; +import { citationsPlugin, labelsPlugin } from '../src'; import fs from 'node:fs'; import path from 'node:path'; import yaml from 'js-yaml'; @@ -17,7 +17,7 @@ type TestCase = { }; const directory = path.join('tests'); -const files = ['citations.yml']; +const files = ['citations.yml', 'labels.yml']; const only = ''; // Can set this to a test title @@ -36,7 +36,7 @@ casesList.forEach(({ title, cases }) => { test.each(casesToUse.map((c): [string, TestCase] => [c.title, c]))( '%s', (_, { md, tokens }) => { - const mdit = MarkdownIt().use(citationsPlugin); + const mdit = MarkdownIt().use(citationsPlugin).use(labelsPlugin); const parsed = mdit.parse(md, {}); expect(parsed).containSubset(tokens); }, diff --git a/packages/markdown-it-myst/tests/labels.yml b/packages/markdown-it-myst/tests/labels.yml new file mode 100644 index 000000000..605ad0714 --- /dev/null +++ b/packages/markdown-it-myst/tests/labels.yml @@ -0,0 +1,22 @@ +title: Labels +cases: + - title: basic {#label} + md: '{#my-label}' + tokens: + - type: paragraph_open + - type: inline + children: + - type: myst_target + content: 'my-label' + - type: paragraph_close + - title: label stops at first } + md: '{#my-label}too}' + tokens: + - type: paragraph_open + - type: inline + children: + - type: myst_target + content: 'my-label' + - type: text + content: 'too}' + - type: paragraph_close diff --git a/packages/myst-parser/src/fromMarkdown.ts b/packages/myst-parser/src/fromMarkdown.ts index 66c7f98ed..07c1fc3dc 100644 --- a/packages/myst-parser/src/fromMarkdown.ts +++ b/packages/myst-parser/src/fromMarkdown.ts @@ -55,6 +55,7 @@ export type AllOptions = { tasklist?: boolean; tables?: boolean; blocks?: boolean; + rmd?: boolean; }; mdast: MdastOptions; directives: DirectiveSpec[]; diff --git a/packages/myst-parser/src/myst.ts b/packages/myst-parser/src/myst.ts index 4c1f4b20e..91c5e8984 100644 --- a/packages/myst-parser/src/myst.ts +++ b/packages/myst-parser/src/myst.ts @@ -23,6 +23,7 @@ import { applyRoles } from './roles.js'; import type { AllOptions } from './fromMarkdown.js'; import type { GenericParent } from 'myst-common'; import { visit } from 'unist-util-visit'; +import { labelsPlugin, shortcodePlugin, spanPlugin } from 'markdown-it-myst'; type Options = Partial; @@ -41,6 +42,7 @@ export const defaultOptions: Omit = { tasklist: true, tables: true, blocks: true, + rmd: true, }, mdast: {}, directives: defaultDirectives, @@ -82,6 +84,11 @@ export function createTokenizer(opts?: Options) { if (extensions.blocks) tokenizer.use(mystBlockPlugin); if (extensions.footnotes) tokenizer.use(footnotePlugin).disable('footnote_inline'); // not yet implemented in myst-parser if (extensions.citations) tokenizer.use(citationsPlugin); + if (extensions.rmd) { + tokenizer.use(labelsPlugin); + tokenizer.use(shortcodePlugin); + tokenizer.use(spanPlugin); + } tokenizer.use(mystPlugin); if (extensions.math) tokenizer.use(mathPlugin, extensions.math); if (extensions.deflist) tokenizer.use(deflistPlugin); diff --git a/packages/myst-transforms/package.json b/packages/myst-transforms/package.json index 06f361181..3550652d4 100644 --- a/packages/myst-transforms/package.json +++ b/packages/myst-transforms/package.json @@ -36,6 +36,7 @@ "unified": "^10.0.0", "unist-builder": "^3.0.0", "unist-util-find-after": "^4.0.0", + "unist-util-find-before": "4.0.0", "unist-util-modify-children": "^3.1.0", "unist-util-map": "^3.0.0", "unist-util-remove": "^3.1.0", diff --git a/packages/myst-transforms/src/targets.ts b/packages/myst-transforms/src/targets.ts index 973a22f4c..c99acf43a 100644 --- a/packages/myst-transforms/src/targets.ts +++ b/packages/myst-transforms/src/targets.ts @@ -1,11 +1,12 @@ import type { Plugin } from 'unified'; import { findAfter } from 'unist-util-find-after'; +import { findBefore } from 'unist-util-find-before'; import { visit } from 'unist-util-visit'; import { remove } from 'unist-util-remove'; import type { Target, Parent } from 'myst-spec'; import type { Heading } from 'myst-spec-ext'; import { selectAll } from 'unist-util-select'; -import type { GenericParent } from 'myst-common'; +import type { GenericNode, GenericParent } from 'myst-common'; import { normalizeLabel, toText } from 'myst-common'; /** @@ -25,15 +26,30 @@ import { normalizeLabel, toText } from 'myst-common'; * and other structural changes to the tree that don't preserve labels. */ export function mystTargetsTransform(tree: GenericParent) { + const paragraphs = selectAll('paragraph', tree) as GenericParent[]; + paragraphs.forEach((paragraph) => { + if (paragraph.children?.length !== 1) return; + const target = paragraph.children[0]; + if (target.type !== 'mystTarget') return; + paragraph.type = target.type; + paragraph.label = target.label; + paragraph.position = target.position; + }); visit(tree, 'mystTarget', (node: Target, index: number, parent: Parent) => { // TODO: have multiple targets and collect the labels - const nextNode = findAfter(parent, index) as any; const normalized = normalizeLabel(node.label); - if (nextNode && normalized) { + if (!normalized) return; + let targetedNode = findAfter(parent, index) as GenericNode; + if (!targetedNode && parent.type === 'heading') targetedNode = parent; + if (!targetedNode) { + const prevNode = findBefore(parent, index) as GenericNode; + if (prevNode?.type === 'image') targetedNode = prevNode; + } + if (targetedNode && normalized) { // TODO: raise error if the node is already labelled - nextNode.identifier = normalized.identifier; - nextNode.label = normalized.label; - nextNode.html_id = normalized.html_id; + targetedNode.identifier = normalized.identifier; + targetedNode.label = normalized.label; + targetedNode.html_id = normalized.html_id; } }); remove(tree, 'mystTarget'); From 93810cf0bfbc361db40d139f63cc5a3c59ae4aff Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 08:17:42 -0700 Subject: [PATCH 03/11] =?UTF-8?q?=F0=9F=94=A7=20Add=20export?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/markdown-it-myst/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/markdown-it-myst/src/index.ts b/packages/markdown-it-myst/src/index.ts index 296f96488..d56cc89b5 100644 --- a/packages/markdown-it-myst/src/index.ts +++ b/packages/markdown-it-myst/src/index.ts @@ -6,7 +6,7 @@ import { labelsPlugin } from './labels.js'; import { shortcodePlugin } from './shortcode.js'; import { spanPlugin } from './span.js'; -export { rolePlugin, directivePlugin, citationsPlugin, shortcodePlugin, labelsPlugin }; +export { rolePlugin, directivePlugin, citationsPlugin, shortcodePlugin, spanPlugin, labelsPlugin }; /** * A markdown-it plugin for parsing MyST roles and directives to structured data From 4cf552a9dd13d1d66d69a63e1551c31254dda97c Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 9 Jul 2024 08:47:00 -0700 Subject: [PATCH 04/11] Fix tests --- packages/markdown-it-myst/tests/shortcode.spec.ts | 6 +++--- packages/markdown-it-myst/tests/span.spec.ts | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/markdown-it-myst/tests/shortcode.spec.ts b/packages/markdown-it-myst/tests/shortcode.spec.ts index 728d19be7..36b69bd87 100644 --- a/packages/markdown-it-myst/tests/shortcode.spec.ts +++ b/packages/markdown-it-myst/tests/shortcode.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest'; import MarkdownIt from 'markdown-it'; -import plugin from '../src'; +import { default as plugin, shortcodePlugin } from '../src'; describe('parses roles', () => { it('basic role parses', () => { - const mdit = MarkdownIt().use(plugin); + const mdit = MarkdownIt().use(shortcodePlugin).use(plugin); const tokens = mdit.parse('ok {{< var lang >}}', {}); expect(tokens.map((t) => t.type)).toEqual(['paragraph_open', 'inline', 'paragraph_close']); expect(tokens[1].children?.map((t) => t.type)).toEqual([ @@ -23,7 +23,7 @@ describe('parses roles', () => { expect(tokens[1].children?.[3].content).toEqual('lang'); }); it('basic role parses', () => { - const mdit = MarkdownIt().use(plugin); + const mdit = MarkdownIt().use(shortcodePlugin).use(plugin); const content = `Notice that the value for \`some_numbers\` is {{< var np_or_r >}}, and that this value *contains* 10 numbers.`; const tokens = mdit.parse(content, {}); diff --git a/packages/markdown-it-myst/tests/span.spec.ts b/packages/markdown-it-myst/tests/span.spec.ts index 87f83c7fc..c4f3789c3 100644 --- a/packages/markdown-it-myst/tests/span.spec.ts +++ b/packages/markdown-it-myst/tests/span.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest'; import MarkdownIt from 'markdown-it'; -import plugin from '../src'; +import { default as plugin, spanPlugin } from '../src'; describe('parses spans', () => { it('basic span parses', () => { - const mdit = MarkdownIt().use(plugin); + const mdit = MarkdownIt().use(spanPlugin).use(plugin); const tokens = mdit.parse('ok [content]{.python}', {}); expect(tokens.map((t) => t.type)).toEqual(['paragraph_open', 'inline', 'paragraph_close']); expect(tokens[1].children?.map((t) => t.type)).toEqual([ @@ -19,6 +19,7 @@ describe('parses spans', () => { // Pass the column information for the role expect((tokens[1].children?.[1] as any).col).toEqual([3, 21]); expect(tokens[1].children?.[1].info).toEqual('span'); + expect(tokens[1].children?.[1].meta.options.class).toEqual('python'); expect(tokens[1].children?.[1].content).toEqual('content'); expect(tokens[1].children?.[3].content).toEqual('content'); }); From d5e5cfa58a6a78b98ffe78fc0fcfd1296c4abded Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 09:31:24 -0700 Subject: [PATCH 05/11] =?UTF-8?q?=E2=97=BB=EF=B8=8F=20Remove=20heading=20w?= =?UTF-8?q?hitespace=20before=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/targets.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/myst-transforms/src/targets.ts b/packages/myst-transforms/src/targets.ts index c99acf43a..699b22bcb 100644 --- a/packages/myst-transforms/src/targets.ts +++ b/packages/myst-transforms/src/targets.ts @@ -40,7 +40,13 @@ export function mystTargetsTransform(tree: GenericParent) { const normalized = normalizeLabel(node.label); if (!normalized) return; let targetedNode = findAfter(parent, index) as GenericNode; - if (!targetedNode && parent.type === 'heading') targetedNode = parent; + if (!targetedNode && parent.type === 'heading') { + targetedNode = parent; + // Strip trailing whitespace if there is a label + const headingText = selectAll('text', targetedNode) as GenericNode[]; + const lastHeadingText = headingText[headingText.length - 1]; + lastHeadingText.value = lastHeadingText.value?.trimEnd(); + } if (!targetedNode) { const prevNode = findBefore(parent, index) as GenericNode; if (prevNode?.type === 'image') targetedNode = prevNode; From 8f218f3c212346cbab96cff9e5e5b6a39caaf8e8 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 09:44:47 -0700 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=8F=B7=20Lift=20labels=20from=20con?= =?UTF-8?q?tainer=20contents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/containers.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/myst-transforms/src/containers.ts b/packages/myst-transforms/src/containers.ts index 15c927a0a..9f4a9053d 100644 --- a/packages/myst-transforms/src/containers.ts +++ b/packages/myst-transforms/src/containers.ts @@ -197,6 +197,13 @@ export function containerChildrenTransform(tree: GenericParent, vfile: VFile) { if (subfigures.length > 1 && !container.noSubcontainers) { subfigures = subfigures.map((node) => createSubfigure(node, container)); } + if (subfigures.length === 1 && !container.label && !container.identifier) { + const { label, identifier } = normalizeLabel(subfigures[0].label) ?? {}; + container.label = label; + container.identifier = identifier; + delete subfigures[0].label; + delete subfigures[0].identifier; + } const children: GenericNode[] = [...subfigures]; if (placeholderImage) children.push(placeholderImage); // Caption is above tables and below all other figures From 70c86d7078ce0351bbd310f0c07c4a8f7b82a4ee Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 10:11:22 -0700 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=94=A7=20Lift=20htmlid=20from=20ima?= =?UTF-8?q?ge=20to=20figure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/containers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/myst-transforms/src/containers.ts b/packages/myst-transforms/src/containers.ts index 9f4a9053d..13ba111bb 100644 --- a/packages/myst-transforms/src/containers.ts +++ b/packages/myst-transforms/src/containers.ts @@ -198,11 +198,13 @@ export function containerChildrenTransform(tree: GenericParent, vfile: VFile) { subfigures = subfigures.map((node) => createSubfigure(node, container)); } if (subfigures.length === 1 && !container.label && !container.identifier) { - const { label, identifier } = normalizeLabel(subfigures[0].label) ?? {}; + const { label, identifier, html_id } = normalizeLabel(subfigures[0].label) ?? {}; container.label = label; container.identifier = identifier; + container.html_id = html_id; delete subfigures[0].label; delete subfigures[0].identifier; + delete subfigures[0].html_id; } const children: GenericNode[] = [...subfigures]; if (placeholderImage) children.push(placeholderImage); From dd872dffd9f55e8c5a57ef7b8ecf4aa5b9252e7d Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 9 Jul 2024 10:18:29 -0700 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=8E=86=20Convert=20labeled=20image?= =?UTF-8?q?=20to=20figure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/targets.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/myst-transforms/src/targets.ts b/packages/myst-transforms/src/targets.ts index 699b22bcb..93095dbbd 100644 --- a/packages/myst-transforms/src/targets.ts +++ b/packages/myst-transforms/src/targets.ts @@ -7,7 +7,7 @@ import type { Target, Parent } from 'myst-spec'; import type { Heading } from 'myst-spec-ext'; import { selectAll } from 'unist-util-select'; import type { GenericNode, GenericParent } from 'myst-common'; -import { normalizeLabel, toText } from 'myst-common'; +import { copyNode, normalizeLabel, toText } from 'myst-common'; /** * Propagate target identifier/value to subsequent node @@ -49,9 +49,24 @@ export function mystTargetsTransform(tree: GenericParent) { } if (!targetedNode) { const prevNode = findBefore(parent, index) as GenericNode; - if (prevNode?.type === 'image') targetedNode = prevNode; + if (prevNode?.type === 'image') { + // Convert labeled image to figure + const image = copyNode(prevNode); + targetedNode = prevNode; + targetedNode.type = 'container'; + targetedNode.kind = 'figure'; + targetedNode.children = [image]; + if (image.alt) { + targetedNode.children.push({ + type: 'paragraph', + children: [{ type: 'text', value: image.alt }], + }); + } + delete targetedNode.alt; + delete targetedNode.url; + } } - if (targetedNode && normalized) { + if (targetedNode) { // TODO: raise error if the node is already labelled targetedNode.identifier = normalized.identifier; targetedNode.label = normalized.label; From 856dc4f40a21ea693dfbf5c3c70c2a5fdd98b21d Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 9 Jul 2024 11:02:20 -0700 Subject: [PATCH 09/11] Remove log --- packages/myst-parser/src/tokensToMyst.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/myst-parser/src/tokensToMyst.ts b/packages/myst-parser/src/tokensToMyst.ts index be5f71465..bfe9c7ada 100644 --- a/packages/myst-parser/src/tokensToMyst.ts +++ b/packages/myst-parser/src/tokensToMyst.ts @@ -416,7 +416,6 @@ const defaultMdast: Record = { parsed_role: { type: 'mystRole', getAttrs(t) { - if (t.info !== 'var') console.log({ role: t }); return { name: t.info, value: t.content, From e73a9811fcd3e8d8f8d857656b727a34f445556b Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 9 Jul 2024 11:02:46 -0700 Subject: [PATCH 10/11] Move document transforms up --- packages/myst-cli/src/process/mdast.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 686402009..7ea2a2704 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -181,15 +181,18 @@ export async function transformMdast( }) .use(inlineMathSimplificationPlugin) .use(mathPlugin, { macros: frontmatter.math }) - .use(glossaryPlugin) // This should be before the enumerate plugins - .use(abbreviationPlugin, { abbreviations: frontmatter.abbreviations }) - .use(enumerateTargetsPlugin, { state }) // This should be after math/container transforms .use(joinGatesPlugin); // Load custom transform plugins session.plugins?.transforms.forEach((t) => { if (t.stage !== 'document') return; pipe.use(t.plugin, undefined, pluginUtils); }); + + pipe + .use(glossaryPlugin) // This should be before the enumerate plugins + .use(abbreviationPlugin, { abbreviations: frontmatter.abbreviations }) + .use(enumerateTargetsPlugin, { state }); // This should be after math/container transforms + await pipe.run(mdast, vfile); // This needs to come after basic transformations since meta tags are added there From f1107bba714547b66c6e27403c5623995cda0785 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 9 Jul 2024 11:03:15 -0700 Subject: [PATCH 11/11] Allow options to be defined on directive token --- packages/markdown-it-myst/src/directives.ts | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/markdown-it-myst/src/directives.ts b/packages/markdown-it-myst/src/directives.ts index dc36073ca..6439e71e0 100644 --- a/packages/markdown-it-myst/src/directives.ts +++ b/packages/markdown-it-myst/src/directives.ts @@ -32,6 +32,11 @@ function replaceFences(state: StateCore): boolean { token.info = match[1].trim(); token.meta = { arg: match[2] }; } + if (!match && token.markup.startsWith(':::')) { + token.type = 'directive'; + token.meta = { options: { class: token.info?.trim() } }; + token.info = 'div'; + } } } return true; @@ -49,6 +54,7 @@ function runDirectives(state: StateCore): boolean { token.content.trim() ? token.content.split(/\r?\n/) : [], info, state, + token.meta.options, ); const { body, options } = content; let { bodyOffset } = content; @@ -106,6 +112,7 @@ function parseDirectiveContent( content: string[], info: string, state: StateCore, + optionsIn: Record = {}, ): { body: string[]; bodyOffset: number; @@ -136,7 +143,11 @@ function parseDirectiveContent( try { const options = yaml.load(yamlBlock.join('\n')) as Record; if (options && typeof options === 'object') { - return { body: newContent, options: Object.entries(options), bodyOffset }; + return { + body: newContent, + options: Object.entries({ ...optionsIn, ...options }), + bodyOffset, + }; } } catch (err) { stateWarn( @@ -145,7 +156,7 @@ function parseDirectiveContent( ); } } else if (content.length && COLON_OPTION_REGEX.exec(content[0])) { - const options: [string, string][] = []; + const options: Record = {}; let foundDivider = false; for (const line of content) { if (!foundDivider && !COLON_OPTION_REGEX.exec(line)) { @@ -158,13 +169,15 @@ function parseDirectiveContent( } else { const match = COLON_OPTION_REGEX.exec(line); const { option, value } = match?.groups ?? {}; - if (option) options.push([option, value || 'true']); + if (option) { + options[option] = value || 'true'; + } bodyOffset++; } } - return { body: newContent, options, bodyOffset }; + return { body: newContent, options: Object.entries({ ...optionsIn, ...options }), bodyOffset }; } - return { body: content, bodyOffset: 1 }; + return { body: content, bodyOffset: 1, options: Object.entries({ ...optionsIn }) }; } function directiveArgToTokens(arg: string, lineNumber: number, state: StateCore) {