Add lint-test-repos scripts and Workflow for real-world regression testing.

.github/workflows/test-repos.yml

@@ -0,0 +1,28 @@
+name: TestRepos
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [12.x]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install Dependencies
+      run: npm install --no-package-lock
+    - name: Clone Test Repos
+      run: npm run clone-test-repos
+    - name: Lint Test Repos
+      run: npm run lint-test-repos

.gitignore

@@ -4,4 +4,5 @@ demo/markdownlint-browser.min.js
 lib-es3
 node_modules
 npm-debug.log
+test-repos
 .vscode

.npmignore

@@ -10,3 +10,4 @@ example
 lib-es3
 npm-debug.log
 test
+test-repos

package.json

@@ -24,7 +24,10 @@
     "build-declaration": "tsc --allowJs --declaration --outDir declaration --resolveJsonModule lib/markdownlint.js && cpy declaration/lib/markdownlint.d.ts lib && rimraf declaration",
     "build-demo": "cpy node_modules/markdown-it/dist/markdown-it.min.js demo && cd demo && rimraf markdownlint-browser.* && cpy file-header.js . --rename=markdownlint-browser.js && tsc --allowJs --resolveJsonModule --outDir ../lib-es3 ../lib/markdownlint.js && cpy ../helpers/package.json ../lib-es3/helpers && browserify ../lib-es3/lib/markdownlint.js --standalone markdownlint >> markdownlint-browser.js && uglifyjs markdownlint-browser.js --compress --mangle --comments --output markdownlint-browser.min.js",
     "build-example": "npm install --no-save --ignore-scripts grunt grunt-cli gulp through2",
-    "example": "cd example && node standalone.js && grunt markdownlint --force && gulp markdownlint"
+    "example": "cd example && node standalone.js && grunt markdownlint --force && gulp markdownlint",
+    "clone-test-repos": "make-dir test-repos && cd test-repos && git clone https://github.com/eslint/eslint eslint-eslint --depth 1 --no-tags --quiet && git clone https://github.com/mkdocs/mkdocs mkdocs-mkdocs --depth 1 --no-tags --quiet && git clone https://github.com/pi-hole/docs pi-hole-docs --depth 1 --no-tags --quiet",
+    "lint-test-repos": "node test/markdownlint-test-repos.js",
+    "clean-test-repos": "rimraf test-repos"
   },
   "engines": {
     "node": ">=10"
@@ -40,13 +43,16 @@
     "eslint": "~6.8.0",
     "eslint-plugin-jsdoc": "~22.1.0",
     "glob": "~7.1.6",
+    "globby": "~11.0.0",
     "js-yaml": "~3.13.1",
+    "make-dir-cli": "~2.0.0",
     "markdown-it-for-inline": "~0.1.1",
     "markdown-it-katex": "~2.0.3",
     "markdown-it-sub": "~1.0.0",
     "markdown-it-sup": "~1.0.0",
     "markdownlint-rule-helpers": "~0.7.0",
     "rimraf": "~3.0.2",
+    "strip-json-comments": "~3.1.0",
     "tape": "~4.13.2",
     "tape-player": "~0.1.0",
     "toml": "~3.0.0",

test/markdownlint-test-repos.js

@@ -0,0 +1,93 @@
+// @ts-check
+
+"use strict";
+
+const { join } = require("path");
+const { promisify } = require("util");
+const globby = require("globby");
+const jsYaml = require("js-yaml");
+const stripJsonComments = require("strip-json-comments");
+const tape = require("tape");
+require("tape-player");
+const markdownlint = require("../lib/markdownlint");
+const markdownlintPromise = promisify(markdownlint);
+const readConfigPromise = promisify(markdownlint.readConfig);
+
+/**
+ * Parses JSONC text.
+ *
+ * @param {string} json JSON to parse.
+ * @returns {Object} Object representation.
+ */
+function jsoncParse(json) {
+  return JSON.parse(stripJsonComments(json));
+}
+
+/**
+ * Parses YAML text.
+ *
+ * @param {string} yaml YAML to parse.
+ * @returns {Object} Object representation.
+ */
+function yamlParse(yaml) {
+  return jsYaml.safeLoad(yaml);
+}
+
+/**
+ * Lints a test repository.
+ *
+ * @param {Object} test Test instance.
+ * @param {string[]} globPatterns Array of files to in/exclude.
+ * @param {string} configPath Path to config file.
+ */
+function lintTestRepo(test, globPatterns, configPath) {
+  test.plan(1);
+  Promise.all([
+    globby(globPatterns),
+    // @ts-ignore
+    readConfigPromise(configPath, [ jsoncParse, yamlParse ])
+  ]).then((globbyAndReadConfigResults) => {
+    const [ files, config ] = globbyAndReadConfigResults;
+    const options = {
+      files,
+      config
+    };
+    return markdownlintPromise(options).then((results) => {
+      const resultsString = results.toString();
+      if (resultsString.length) {
+        // eslint-disable-next-line no-console
+        console.log(resultsString);
+      }
+      test.ok(!resultsString.length, "Unexpected linting violations");
+      test.end();
+    });
+  });
+}
+
+// Run markdownlint the same way the corresponding repositories do
+
+tape("https://github.com/eslint/eslint", (test) => {
+  const rootDir = "./test-repos/eslint-eslint";
+  const globPatterns = [ join(rootDir, "docs/**/*.md") ];
+  const configPath = join(rootDir, ".markdownlint.yml");
+  lintTestRepo(test, globPatterns, configPath);
+});
+
+tape("https://github.com/mkdocs/mkdocs", (test) => {
+  const rootDir = "./test-repos/mkdocs-mkdocs";
+  const globPatterns = [
+    join(rootDir, "README.md"),
+    join(rootDir, "CONTRIBUTING.md"),
+    join(rootDir, "docs/*"),
+    "!" + join(rootDir, "docs/CNAME")
+  ];
+  const configPath = join(rootDir, ".markdownlintrc");
+  lintTestRepo(test, globPatterns, configPath);
+});
+
+tape("https://github.com/pi-hole/docs", (test) => {
+  const rootDir = "./test-repos/pi-hole-docs";
+  const globPatterns = [ join(rootDir, "**/*.md") ];
+  const configPath = join(rootDir, ".markdownlint.json");
+  lintTestRepo(test, globPatterns, configPath);
+});