Remove params.tokenLists to simplify custom rule API.

lib/markdownlint.js

@@ -141,7 +141,6 @@ function removeFrontMatter(content, frontMatter) {
 // Annotate tokens with line/lineNumber
 function annotateTokens(tokens, lines) {
   var tbodyMap = null;
-  var tokenLists = {};
   tokens.forEach(function forToken(token) {
     // Handle missing maps for table body
     if (token.type === "tbody_open") {
@@ -172,12 +171,7 @@ function annotateTokens(tokens, lines) {
         }
       });
     }
-    if (!tokenLists[token.type]) {
-      tokenLists[token.type] = [];
-    }
-    tokenLists[token.type].push(token);
   });
-  return tokenLists;
 }
 
 // Map rule names/tags to canonical rule name
@@ -292,19 +286,16 @@ function lintContent(
   ruleList, content, config, frontMatter, noInlineConfig, resultVersion,
   callback) {
   // Remove UTF-8 byte order marker (if present)
-  if (content.charCodeAt(0) === 0xfeff) {
+  content = content.replace(/^\ufeff/, "");
-    content = content.slice(1);
-  }
   // Remove front matter
   var removeFrontMatterResult = removeFrontMatter(content, frontMatter);
-  content = removeFrontMatterResult.content;
   var frontMatterLines = removeFrontMatterResult.frontMatterLines;
   // Ignore the content of HTML comments
-  content = shared.clearHtmlCommentText(content);
+  content = shared.clearHtmlCommentText(removeFrontMatterResult.content);
   // Parse content into tokens and lines
   var tokens = md.parse(content, {});
   var lines = content.split(shared.newLineRe);
-  var tokenLists = annotateTokens(tokens, lines);
+  annotateTokens(tokens, lines);
   var aliasToRuleNames = mapAliasToRuleNames(ruleList);
   var effectiveConfig = getEffectiveConfig(ruleList, config, aliasToRuleNames);
   var enabledRulesPerLineNumber = getEnabledRulesPerLineNumber(
@@ -313,10 +304,10 @@ function lintContent(
   // Create parameters for rules
   var params = {
     "tokens": tokens,
-    "tokenLists": tokenLists,
     "lines": lines,
     "frontMatterLines": frontMatterLines
   };
+  shared.makeTokenCache(params);
   // Function to run for each rule
   var result = (resultVersion === 0) ? {} : [];
   function forRule(rule) {
@@ -402,8 +393,10 @@ function lintContent(
   try {
     ruleList.forEach(forRule);
   } catch (ex) {
+    shared.makeTokenCache(null);
     return callback(ex);
   }
+  shared.makeTokenCache(null);
   callback(null, result);
 }
 

lib/shared.js

@@ -117,51 +117,104 @@ module.exports.headingStyleFor = function headingStyleFor(token) {
 
 // Calls the provided function for each matching token
 function filterTokens(params, type, callback) {
-  (params.tokenLists[type] || []).forEach(callback);
+  params.tokens.forEach(function forToken(token) {
+    if (token.type === type) {
+      callback(token);
+    }
+  });
 }
 module.exports.filterTokens = filterTokens;
 
-var lastForEachLineParams = null;
+var tokenCache = null;
-var lastForEachLineResult = null;
+// Caches line metadata and flattened lists for reuse
+function makeTokenCache(params) {
+  if (!params) {
+    tokenCache = null;
+    return;
+  }
+
+  // Populate line metadata array
+  var lineMetadata = new Array(params.lines.length);
+  var fenceStart = null;
+  var inFence = false;
+  // Find fenced code by pattern (parser ignores "``` close fence")
+  params.lines.forEach(function forLine(line, lineIndex) {
+    var metadata = 0;
+    var match = /^(`{3,}|~{3,})/.exec(line);
+    var fence = match && match[1];
+    if (fence &&
+        (!inFence || (fence.substr(0, fenceStart.length) === fenceStart))) {
+      metadata = inFence ? 2 : 6;
+      fenceStart = inFence ? null : fence;
+      inFence = !inFence;
+    } else if (inFence) {
+      metadata = 1;
+    }
+    lineMetadata[lineIndex] = metadata;
+  });
+  // Find code blocks normally
+  filterTokens(params, "code_block", function forToken(token) {
+    for (var i = token.map[0]; i < token.map[1]; i++) {
+      lineMetadata[i] = 1;
+    }
+  });
+  // Find tables normally
+  filterTokens(params, "table_open", function forToken(token) {
+    for (var i = token.map[0]; i < token.map[1]; i++) {
+      lineMetadata[i] += 8;
+    }
+  });
+
+  // Flatten lists
+  var flattenedLists = [];
+  var stack = [];
+  var current = null;
+  var lastWithMap = { "map": [ 0, 1 ] };
+  params.tokens.forEach(function forToken(token) {
+    if ((token.type === "bullet_list_open") ||
+        (token.type === "ordered_list_open")) {
+      // Save current context and start a new one
+      stack.push(current);
+      current = {
+        "unordered": (token.type === "bullet_list_open"),
+        "parentsUnordered": !current ||
+          (current.unordered && current.parentsUnordered),
+        "open": token,
+        "items": [],
+        "nesting": stack.length - 1,
+        "lastLineIndex": -1,
+        "insert": flattenedLists.length
+      };
+    } else if ((token.type === "bullet_list_close") ||
+               (token.type === "ordered_list_close")) {
+      // Finalize current context and restore previous
+      current.lastLineIndex = lastWithMap.map[1];
+      flattenedLists.splice(current.insert, 0, current);
+      delete current.insert;
+      current = stack.pop();
+    } else if (token.type === "list_item_open") {
+      // Add list item
+      current.items.push(token);
+    } else if (token.map) {
+      // Track last token with map
+      lastWithMap = token;
+    }
+  });
+
+  // Cache results
+  tokenCache = {
+    "params": params,
+    "lineMetadata": lineMetadata,
+    "flattenedLists": flattenedLists
+  };
+}
+module.exports.makeTokenCache = makeTokenCache;
+
 // Calls the provided function for each line (with context)
 module.exports.forEachLine = function forEachLine(params, callback) {
-  if (params !== lastForEachLineParams) {
-    var lineMetadata = new Array(params.lines.length);
-    var fenceStart = null;
-    var inFence = false;
-    // Find fenced code by pattern (parser ignores "``` close fence")
-    params.lines.forEach(function forLine(line, lineIndex) {
-      var metadata = 0;
-      var match = /^(`{3,}|~{3,})/.exec(line);
-      var fence = match && match[1];
-      if (fence &&
-          (!inFence || (fence.substr(0, fenceStart.length) === fenceStart))) {
-        metadata = inFence ? 2 : 6;
-        fenceStart = inFence ? null : fence;
-        inFence = !inFence;
-      } else if (inFence) {
-        metadata = 1;
-      }
-      lineMetadata[lineIndex] = metadata;
-    });
-    // Find code blocks normally
-    filterTokens(params, "code_block", function forToken(token) {
-      for (var i = token.map[0]; i < token.map[1]; i++) {
-        lineMetadata[i] = 1;
-      }
-    });
-    // Find tables normally
-    filterTokens(params, "table_open", function forToken(token) {
-      for (var i = token.map[0]; i < token.map[1]; i++) {
-        lineMetadata[i] += 8;
-      }
-    });
-    lastForEachLineParams = params;
-    lastForEachLineResult = lineMetadata;
-  }
   // Invoke callback
-  params.lines.forEach(function forLine(line, lineIndex) {
+  tokenCache.params.lines.forEach(function forLine(line, lineIndex) {
-    var metadata = lastForEachLineResult[lineIndex];
+    var metadata = tokenCache.lineMetadata[lineIndex];
     callback(
       line,
       lineIndex,
@@ -197,49 +250,9 @@ module.exports.forEachHeading = function forEachHeading(params, callback) {
   });
 };
 
-var lastFlattenListsParams = null;
-var lastFlattenListsResult = null;
 // Returns (nested) lists as a flat array (in order)
-module.exports.flattenLists = function flattenLists(params) {
+module.exports.flattenLists = function flattenLists() {
-  if (lastFlattenListsParams !== params) {
+  return tokenCache.flattenedLists;
-    var lists = [];
-    var stack = [];
-    var current = null;
-    var lastWithMap = { "map": [ 0, 1 ] };
-    params.tokens.forEach(function forToken(token) {
-      if ((token.type === "bullet_list_open") ||
-          (token.type === "ordered_list_open")) {
-        // Save current context and start a new one
-        stack.push(current);
-        current = {
-          "unordered": (token.type === "bullet_list_open"),
-          "parentsUnordered": !current ||
-            (current.unordered && current.parentsUnordered),
-          "open": token,
-          "items": [],
-          "nesting": stack.length - 1,
-          "lastLineIndex": -1,
-          "insert": lists.length
-        };
-      } else if ((token.type === "bullet_list_close") ||
-                 (token.type === "ordered_list_close")) {
-        // Finalize current context and restore previous
-        current.lastLineIndex = lastWithMap.map[1];
-        lists.splice(current.insert, 0, current);
-        delete current.insert;
-        current = stack.pop();
-      } else if (token.type === "list_item_open") {
-        // Add list item
-        current.items.push(token);
-      } else if (token.map) {
-        // Track last token with map
-        lastWithMap = token;
-      }
-    });
-    lastFlattenListsParams = params;
-    lastFlattenListsResult = lists;
-  }
-  return lastFlattenListsResult;
 };
 
 // Adds a generic error object via the onError callback