Andreas/markdownlint
1
0
Fork 0
mirror of https://github.com/DavidAnson/markdownlint.git synced 2025-09-22 05:40:48 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Refactor micromark parse code to stop using micromark's TokenizeContext.sliceSerialize (less well supported) in favor of Token.start/end.offset.

Browse source
This commit is contained in:
David Anson 2025-02-08 14:43:38 -08:00
parent b23fc96ab2
commit 3cbe1cb6c5
6 changed files with 71 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

5
helpers/helpers.cjs
View file

@ -378,6 +378,7 @@ module.exports.frontMatterHasTitle =
*/ */
function getReferenceLinkImageData(tokens) { function getReferenceLinkImageData(tokens) {
const normalizeReference = (s) => s.toLowerCase().trim().replace(/\s+/g, " "); const normalizeReference = (s) => s.toLowerCase().trim().replace(/\s+/g, " ");
const getText = (t) => t?.children.filter((c) => c.type !== "blockQuotePrefix").map((c) => c.text).join("");
const references = new Map(); const references = new Map();
const shortcuts = new Map(); const shortcuts = new Map();
const addReferenceToDictionary = (token, label, isShortcut) => { const addReferenceToDictionary = (token, label, isShortcut) => {
@ -449,7 +450,7 @@ function getReferenceLinkImageData(tokens) {
const isFullOrCollapsed = (token.children.length === 2) && !token.children.some((t) => t.type === "resource"); const isFullOrCollapsed = (token.children.length === 2) && !token.children.some((t) => t.type === "resource");
const [ labelText ] = micromark.getDescendantsByType(token, [ "label", "labelText" ]); const [ labelText ] = micromark.getDescendantsByType(token, [ "label", "labelText" ]);
const [ referenceString ] = micromark.getDescendantsByType(token, [ "reference", "referenceString" ]); const [ referenceString ] = micromark.getDescendantsByType(token, [ "reference", "referenceString" ]);
let label = labelText?.text; let label = getText(labelText);
// Identify if footnote // Identify if footnote
if (!isShortcut && !isFullOrCollapsed) { if (!isShortcut && !isFullOrCollapsed) {
const [ footnoteCallMarker, footnoteCallString ] = token.children.filter( const [ footnoteCallMarker, footnoteCallString ] = token.children.filter(
@ -462,7 +463,7 @@ function getReferenceLinkImageData(tokens) {
} }
// Track link (handle shortcuts separately due to ambiguity in "text [text] text") // Track link (handle shortcuts separately due to ambiguity in "text [text] text")
if (isShortcut || isFullOrCollapsed) { if (isShortcut || isFullOrCollapsed) {
addReferenceToDictionary(token, referenceString?.text || label, isShortcut); addReferenceToDictionary(token, getText(referenceString) || label, isShortcut);
} }
} }
break; break;

17
lib/micromark-parse.mjs
View file

@ -20,6 +20,17 @@ import { flatTokensSymbol, htmlFlowSymbol, newLineRe } from "../helpers/shared.c
/** @typedef {import("markdownlint").MicromarkToken} MicromarkToken */ /** @typedef {import("markdownlint").MicromarkToken} MicromarkToken */
/** @typedef {import("./micromark-types.d.mts")} */ /** @typedef {import("./micromark-types.d.mts")} */
/**
* Gets the Markdown text for a Micromark token.
*
* @param {string} markdown Markdown content.
* @param {Token} token Micromark token.
* @returns {string} Token text.
*/
function getText(markdown, token) {
return markdown.slice(token.start.offset, token.end.offset);
}
/** /**
* Parse options. * Parse options.
* *
@ -126,7 +137,7 @@ export function getEvents(
// Create artificial event list and replicate content // Create artificial event list and replicate content
const text = eventsToReplicate const text = eventsToReplicate
.filter((event) => event[0] === "enter") .filter((event) => event[0] === "enter")
.map((event) => tokenizeContext.sliceSerialize(event[1])) .map((event) => getText(markdown, event[1]))
.join("") .join("")
.trim(); .trim();
if ((text.length > 0) && !text.includes("]")) { if ((text.length > 0) && !text.includes("]")) {
@ -221,11 +232,11 @@ function parseInternal(
let lines = null; let lines = null;
let skipHtmlFlowChildren = false; let skipHtmlFlowChildren = false;
for (const event of events) { for (const event of events) {
const [ kind, token, context ] = event; const [ kind, token ] = event;
const { type, start, end } = token; const { type, start, end } = token;
const { "column": startColumn, "line": startLine } = start; const { "column": startColumn, "line": startLine } = start;
const { "column": endColumn, "line": endLine } = end; const { "column": endColumn, "line": endLine } = end;
const text = context.sliceSerialize(token); const text = getText(markdown, token);
if ((kind === "enter") && !skipHtmlFlowChildren) { if ((kind === "enter") && !skipHtmlFlowChildren) {
const previous = current; const previous = current;
history.push(previous); history.push(previous);

37
test/snapshots/markdownlint-test-custom-rules.mjs.md
View file

@ -4860,7 +4860,7 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 3, startColumn: 3,
startLine: 21, startLine: 21,
text: '', text: '> Nested',
type: 'blockQuote', type: 'blockQuote',
}, },
], ],
@ -4869,7 +4869,8 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 20, startLine: 20,
text: '', text: `> Block quote␊
> > Nested`,
type: 'blockQuote', type: 'blockQuote',
}, },
{ {
@ -5149,7 +5150,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 5, startColumn: 5,
startLine: 25, startLine: 25,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'paragraph', type: 'paragraph',
}, },
], ],
@ -5159,7 +5160,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 5, startColumn: 5,
startLine: 25, startLine: 25,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'content', type: 'content',
}, },
{ {
@ -5242,7 +5243,10 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 3, startColumn: 3,
startLine: 25, startLine: 25,
text: '', text: `- Items␊
Indented␊
␊
Content`,
type: 'listUnordered', type: 'listUnordered',
}, },
], ],
@ -5251,7 +5255,12 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 23, startLine: 23,
text: '', text: `- Unordered␊
- List␊
- Items␊
Indented␊
␊
Content`,
type: 'listUnordered', type: 'listUnordered',
}, },
{ {
@ -5561,7 +5570,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 7, startColumn: 7,
startLine: 32, startLine: 32,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'paragraph', type: 'paragraph',
}, },
], ],
@ -5571,7 +5580,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 7, startColumn: 7,
startLine: 32, startLine: 32,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'content', type: 'content',
}, },
{ {
@ -5654,7 +5663,10 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 4, startColumn: 4,
startLine: 32, startLine: 32,
text: '', text: `1. Items␊
Indented␊
␊
Content`,
type: 'listOrdered', type: 'listOrdered',
}, },
], ],
@ -5663,7 +5675,12 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 30, startLine: 30,
text: '', text: `1. Ordered␊
2. List␊
1. Items␊
Indented␊
␊
Content`,
type: 'listOrdered', type: 'listOrdered',
}, },
{ {

BIN
test/snapshots/markdownlint-test-custom-rules.mjs.snap
View file

Binary file not shown.

37
test/snapshots/markdownlint-test-micromark.mjs.md
View file

@ -1798,7 +1798,7 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 3, startColumn: 3,
startLine: 21, startLine: 21,
text: '', text: '> Nested',
type: 'blockQuote', type: 'blockQuote',
}, },
], ],
@ -1807,7 +1807,8 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 20, startLine: 20,
text: '', text: `> Block quote␊
> > Nested`,
type: 'blockQuote', type: 'blockQuote',
}, },
{ {
@ -2087,7 +2088,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 5, startColumn: 5,
startLine: 25, startLine: 25,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'paragraph', type: 'paragraph',
}, },
], ],
@ -2097,7 +2098,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 5, startColumn: 5,
startLine: 25, startLine: 25,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'content', type: 'content',
}, },
{ {
@ -2180,7 +2181,10 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 3, startColumn: 3,
startLine: 25, startLine: 25,
text: '', text: `- Items␊
Indented␊
␊
Content`,
type: 'listUnordered', type: 'listUnordered',
}, },
], ],
@ -2189,7 +2193,12 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 23, startLine: 23,
text: '', text: `- Unordered␊
- List␊
- Items␊
Indented␊
␊
Content`,
type: 'listUnordered', type: 'listUnordered',
}, },
{ {
@ -2499,7 +2508,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 7, startColumn: 7,
startLine: 32, startLine: 32,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'paragraph', type: 'paragraph',
}, },
], ],
@ -2509,7 +2518,7 @@ Generated by [AVA](https://avajs.dev).
startColumn: 7, startColumn: 7,
startLine: 32, startLine: 32,
text: `Items␊ text: `Items␊
Indented`, Indented`,
type: 'content', type: 'content',
}, },
{ {
@ -2592,7 +2601,10 @@ Generated by [AVA](https://avajs.dev).
parent: [Circular], parent: [Circular],
startColumn: 4, startColumn: 4,
startLine: 32, startLine: 32,
text: '', text: `1. Items␊
Indented␊
␊
Content`,
type: 'listOrdered', type: 'listOrdered',
}, },
], ],
@ -2601,7 +2613,12 @@ Generated by [AVA](https://avajs.dev).
parent: null, parent: null,
startColumn: 1, startColumn: 1,
startLine: 30, startLine: 30,
text: '', text: `1. Ordered␊
2. List␊
1. Items␊
Indented␊
␊
Content`,
type: 'listOrdered', type: 'listOrdered',
}, },
{ {

BIN
test/snapshots/markdownlint-test-micromark.mjs.snap
View file

Binary file not shown.