Travis optimization

.gitignore

@@ -8,3 +8,4 @@ src/**/_*.md
 npm-debug.log
 yarn-error.log
 package-lock.json
+.cache

.travis.yml

@@ -1,15 +1,52 @@
-branches:
-  only:
-    - master
-    - develop
+sudo: false
 language: node_js
 node_js:
   - "8"
-sudo: required
-install:
-  - yarn
-  - sudo pip install proselint
-before_script:
-  - source ./src/scripts/env.sh
-script:
-  - bash ./src/scripts/deploy.sh
+
+cache:
+  yarn: true
+  directories:
+    - node_modules
+    - dist
+    - .cache
+
+stages:
+  - Build
+  - Post-build
+
+jobs:
+  fast_finish: true
+  allow_failures:
+    - name: External Link Check
+
+  include:
+    - stage: Build
+      name: Lint and Build
+      before_install: npm install --global yarn
+      install: yarn
+      script:
+        - yarn lint:js
+        - yarn lint:markdown
+        - yarn lint:social
+        - yarn build
+        - yarn lint:links
+
+    - stage: Build
+      name: Proselint
+      language: python
+      python: 3.6
+      cache:
+        pip: true
+        directories:
+          - $HOME/.cache
+      install: pip install -r requirements.txt
+      script: cp .proselintrc ~/ && proselint src/content
+
+    - stage: Post-build
+      name: Deploy
+      if: branch = master
+      script: bash ./src/scripts/deploy.sh
+
+    - stage: Post-build
+      name: External Link Check
+      script: yarn linkcheck

package.json

@@ -26,18 +26,24 @@
   "scripts": {
     "clean": "rimraf ./dist src/content/**/_*.md src/**/_*.json",
     "start": "cross-env NODE_ENV=development webpack-dev-server --config webpack.dev.js --env.dev",
+    "update-repos": "node src/utilities/fetch-package-repos.js",
     "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json",
     "build-test": "npm run build && http-server build/",
-    "fetch": "sh src/scripts/fetch.sh",
+    "fetch": "run-p fetch:*",
+    "fetch:readmes": "node src/utilities/fetch-package-readmes.js",
+    "fetch:supporters": "node src/utilities/fetch-supporters.js",
+    "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js",
     "prebuild": "npm run clean",
-    "build": "npm run fetch && npm run content && NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME",
+    "build": "run-s fetch content && NODE_ENV=production webpack --config webpack.prod.js",
+    "postbuild": "npm run sitemap",
     "test": "npm run lint",
     "lint": "run-s lint:*",
-    "lint:js": "eslint src --ext .js,.jsx,.md",
+    "lint:js": "eslint src --ext .js,.jsx,.md --cache true --cache-location .cache/.eslintcache",
     "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'",
     "lint:social": "alex . -q",
     "lint:prose": "cp .proselintrc ~/ && proselint src/content",
-    "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --todo external-redirect | tee master.tap | tap-spot",
+    "lint:links": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ -i | tee internal-links.tap | tap-spot",
+    "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee external-links.tap | tap-spot",
     "sitemap": "cd dist && sitemap-static --prefix=https://webpack.js.org/ > sitemap.xml",
     "serve": "npm run build && sirv start ./dist --port 4000",
     "deploy": "gh-pages -d dist"
@@ -56,7 +62,7 @@
     ]
   },
   "devDependencies": {
-    "@octokit/rest": "^15.2.6",
+    "@octokit/rest": "^15.9.4",
     "alex": "^5.1.0",
     "autoprefixer": "^7.2.3",
     "babel-core": "^6.26.0",
@@ -86,7 +92,7 @@
     "html-webpack-template": "^6.1.0",
     "http-server": "^0.10.0",
     "husky": "^1.0.0-rc.8",
-    "hyperlink": "github:Munter/hyperlink#070916d",
+    "hyperlink": "^4.0.0",
     "lint-staged": "^7.2.0",
     "loader-utils": "^1.1.0",
     "lodash": "^4.17.4",

repositories/README.md

@@ -0,0 +1,3 @@
+# Webpack documentation repositories
+
+The files in this directory are auto generated from `src/utils/fetch-package-repos.js` and should not be edited by hand. Any manual changes will be overwritten by the automation next time it runs.

repositories/loaders.json

@@ -0,0 +1,45 @@
+[
+  "webpack-contrib/json-loader",
+  "webpack-contrib/raw-loader",
+  "webpack-contrib/coffee-loader",
+  "webpack-contrib/css-loader",
+  "webpack-contrib/style-loader",
+  "webpack-contrib/script-loader",
+  "webpack-contrib/less-loader",
+  "webpack-contrib/bundle-loader",
+  "webpack-contrib/val-loader",
+  "webpack-contrib/file-loader",
+  "webpack-contrib/url-loader",
+  "webpack-contrib/i18n-loader",
+  "webpack-contrib/json5-loader",
+  "webpack-contrib/worker-loader",
+  "webpack-contrib/jshint-loader",
+  "webpack-contrib/imports-loader",
+  "webpack-contrib/exports-loader",
+  "webpack-contrib/mocha-loader",
+  "webpack-contrib/coverjs-loader",
+  "webpack-contrib/expose-loader",
+  "webpack-contrib/node-loader",
+  "webpack-contrib/coffee-redux-loader",
+  "webpack-contrib/transform-loader",
+  "webpack-contrib/html-loader",
+  "webpack-contrib/sass-loader",
+  "webpack-contrib/source-map-loader",
+  "webpack-contrib/react-proxy-loader",
+  "webpack-contrib/null-loader",
+  "webpack-contrib/multi-loader",
+  "webpack-contrib/istanbul-instrumenter-loader",
+  "webpack-contrib/eslint-loader",
+  "webpack-contrib/yaml-frontmatter-loader",
+  "webpack-contrib/svg-inline-loader",
+  "webpack-contrib/restyle-loader",
+  "webpack-contrib/gzip-loader",
+  "webpack-contrib/cache-loader",
+  "webpack-contrib/thread-loader",
+  "webpack-contrib/polymer-webpack-loader",
+  "webpack-contrib/workerize-loader",
+  "webpack-contrib/config-loader",
+  "babel/babel-loader",
+  "postcss/postcss-loader",
+  "peerigon/extract-loader"
+]

repositories/plugins.json

@@ -0,0 +1,15 @@
+[
+  "webpack-contrib/i18n-webpack-plugin",
+  "webpack-contrib/component-webpack-plugin",
+  "webpack-contrib/compression-webpack-plugin",
+  "webpack-contrib/extract-text-webpack-plugin",
+  "webpack-contrib/copy-webpack-plugin",
+  "webpack-contrib/npm-install-webpack-plugin",
+  "webpack-contrib/stylelint-webpack-plugin",
+  "webpack-contrib/babel-minify-webpack-plugin",
+  "webpack-contrib/uglifyjs-webpack-plugin",
+  "webpack-contrib/zopfli-webpack-plugin",
+  "webpack-contrib/closure-webpack-plugin",
+  "webpack-contrib/css-webpack-plugin",
+  "webpack-contrib/mini-css-extract-plugin"
+]

requirements.txt

@@ -0,0 +1 @@
+proselint

src/CNAME

@@ -0,0 +1 @@
+webpack.js.org

src/content/configuration/optimization.md

@@ -293,7 +293,7 @@ module.exports = {
 
 `bool`
 
-Tells webpack to determine used exports for each module. This depends on [`optimization.providedExports`](#optimization-occurrenceorder). Information collected by `optimization.usedExports` is used by other optimizations or code generation i.e. exports are not generated for unused exports, export names are mangled to single char identifiers when all usages are compatible. 
+Tells webpack to determine used exports for each module. This depends on [`optimization.providedExports`](#optimizationoccurrenceorder). Information collected by `optimization.usedExports` is used by other optimizations or code generation i.e. exports are not generated for unused exports, export names are mangled to single char identifiers when all usages are compatible.
 Dead code elimination in minimizers will benefit from this and can remove unused exports.
 By default `optimization.usedExports` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise. 
 
@@ -312,8 +312,8 @@ module.exports = {
 
 `bool`
 
-Tells webpack to find segments of the module graph which can be safely concatenated into a single module. Depends on [`optimization.providedExports`](#optimization-providedexports) and [`optimization.usedExports`](#optimization-usedexports).
-By default `optimization.concatenateModules` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise. 
+Tells webpack to find segments of the module graph which can be safely concatenated into a single module. Depends on [`optimization.providedExports`](#optimizationprovidedexports) and [`optimization.usedExports`](#optimizationusedexports).
+By default `optimization.concatenateModules` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise.
 
 __webpack.config.js__
 

src/content/plugins/commons-chunk-plugin.md

@@ -132,7 +132,7 @@ module.exports = {
 <script src="app.js" charset="utf-8"></script>
 ```
 
-T> In combination with long term caching you may need to use the [`ChunkManifestWebpackPlugin`](https://github.com/soundcloud/chunk-manifest-webpack-plugin) to avoid the vendor chunk changes. You should also use records to ensure stable module ids, e.g. using [`NamedModulesPlugin`](/plugins/named-modules-plugin) or [`HashedModuleIdsPlugin`](/plugins/hashed-module-ids-plugin).
+T> In combination with long term caching you may need to use the [`ChunkManifestWebpackPlugin`](https://github.com/soundcloud/chunk-manifest-webpack-plugin) to avoid the vendor chunk changes. You should also use records to ensure stable module ids, e.g. using `NamedModulesPlugin` or [`HashedModuleIdsPlugin`](/plugins/hashed-module-ids-plugin).
 
 
 ### Move common modules into the parent chunk

src/scripts/deploy.sh

@@ -2,26 +2,10 @@
 # see https://gist.github.com/domenic/ec8b0fc8ab45f39403dd
 set -e # Exit with nonzero exit code if anything fails
 
-SOURCE_BRANCH="master"
-
-# Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
-    echo "Skipping deploy; just doing a build and linting links/prose/js."
-    yarn test
-    yarn build
-    exit 0
-fi
-
 # Save some useful information
 REPO=`git config remote.origin.url`
 SSH_REPO=${REPO/https:\/\/github.com\//[email protected]:}
 
-# Run tests
-yarn test
-
-# Run our build
-yarn build
-
 # Set some git options
 git config --global user.name "Travis CI"
 git config --global user.email "[email protected]"

src/utilities/fetch-package-readmes.js

@@ -0,0 +1,66 @@
+const _ = require('lodash');
+const fs = require('fs');
+const path = require('path');
+const { promisify } = require('util');
+const mkdirp = promisify(require('mkdirp'));
+const request = require('request-promise');
+
+const yamlHeadmatter = require('./yaml-headmatter.js');
+const processReadme = require('./process-readme.js');
+
+const writeFile = promisify(fs.writeFile);
+const readFile = promisify(fs.readFile);
+const cwd = process.cwd();
+
+const types = ['loaders', 'plugins'];
+
+const pathMap = {
+  loaders: path.resolve(__dirname, '../content/loaders'),
+  plugins: path.resolve(__dirname, '../content/plugins')
+};
+
+async function main() {
+  for (const type of types) {
+    const outputDir = pathMap[type];
+
+    await mkdirp(outputDir);
+
+    const repos = JSON.parse(await readFile(path.resolve(__dirname, `../../repositories/${type}.json`)));
+
+    for (const repo of repos) {
+      const [org, packageName] = repo.split('/');
+      const url = `https://raw.githubusercontent.com/${repo}/master/README.md`;
+      const htmlUrl = `https://github.com/${repo}`;
+      const editUrl = `${htmlUrl}/edit/master/README.md`;
+      const fileName = path.resolve(outputDir, `_${packageName}.md`);
+
+      let title = packageName;
+
+      if (type === 'plugins') {
+        title = _.camelCase(title);
+        title = _.upperFirst(title);
+        title = title.replace(/I18N/, 'I18n');
+      }
+
+      // generate yaml matter for file
+      let headmatter = yamlHeadmatter({
+        title: title,
+        source: url,
+        edit: editUrl,
+        repo: htmlUrl
+      });
+
+      request(url)
+        .then(async content => {
+          const body = processReadme(content, { source: url });
+
+          await writeFile(fileName, headmatter + body);
+
+          console.log('Generated:', path.relative(cwd, fileName));
+        });
+    }
+  }
+}
+
+main();
+