Support params in spec paths, by using regexes as Trie keys

This requires a fairly sweeping change everywhere from
objects to ES6, which is the main thrust of all this,
and then lots of careful handling of paths that are
arrays of string & regex, rather than just single strings.

Author: pimterry

custom-typings/@httptoolkit_serialize-javascript.d.ts

@@ -0,0 +1,4 @@
+declare module '@httptoolkit/serialize-javascript' {
+    import * as serializeToJs from 'serialize-javascript';
+    export = serializeToJs;
+}

package-lock.json

@@ -40,12 +40,14 @@
     "openapi3-ts": "^1.2.0"
   },
   "devDependencies": {
+    "@httptoolkit/serialize-javascript": "^1.7.0-sets-and-maps",
     "@types/chai": "^4.1.7",
     "@types/fs-extra": "^5.0.4",
     "@types/globby": "^8.0.0",
     "@types/lodash": "^4.14.120",
     "@types/mocha": "^5.2.5",
     "@types/node": "^11.9.0",
+    "@types/serialize-javascript": "^1.5.0",
     "@types/swagger-parser": "^4.0.2",
     "chai": "^4.2.0",
     "fs-extra": "^7.0.1",

package.json

@@ -1,5 +1,6 @@
 import * as path from 'path';
 import * as fs from 'fs-extra';
+import * as serializeToJs from '@httptoolkit/serialize-javascript';
 
 import { generateApis } from "./generate-apis";
 import { buildTrie } from './build-index';
@@ -12,12 +13,13 @@ export async function buildAll(globs: string[]) {
     console.log('APIs generated and written to disk');
 
     await fs.writeFile(
-        path.join('api', '_index.json'),
-        JSON.stringify(buildTrie(index))
+        path.join('api', '_index.js'),
+        'module.exports = ' + serializeToJs(buildTrie(index), {
+            unsafe: true // We're not embedded in HTML, we don't want XSS escaping
+        })
     );
 
-    const indexSize = Object.keys(index).length;
-    console.log(`Index trie for ${indexSize} entries generated and written to disk`);
+    console.log(`Index trie for ${index.size} entries generated and written to disk`);
 }
 
 if (require.main === module) {

src/buildtime/build-all.ts

@@ -1,53 +1,67 @@
 import * as _ from 'lodash';
-import { TrieData, isLeafValue } from "../runtime/trie";
+import { TrieData, isLeafValue, TrieValue } from "../runtime/trie";
 
-export function buildTrie(map: { [key: string]: string | string[] }): TrieData {
+type TrieInput = Map<
+    Array<string | RegExp>,
+    string | string[]
+>;
+
+export function buildTrie(map: TrieInput): TrieData {
     return optimizeTrie(buildNaiveTrie(map));
 }
 
 // Build a simple naive trie. One level per char, string nodes at the leaves,
 // and '' keys for leaf nodes half way down paths.
-export function buildNaiveTrie(map: { [key: string]: string | string[] }): TrieData {
-    const root = <TrieData>{};
+export function buildNaiveTrie(map: TrieInput): TrieData {
+    const root = <TrieData>new Map();
 
     // For each key, make a new level for each char in the key (or use an existing
     // level), and place the leaf when we get to the end of the key.
 
-    _.forEach(map, (value, key) => {
+    for (let [keys, value] of map) {
         let trie: TrieData = root;
-        _.forEach(key, (char, i) => {
-            let nextStep = trie[char];
 
-            if (i === key.length - 1) {
+        const keyChunks = _.flatMap<string | RegExp, string | RegExp>(keys, (key) => {
+            if (_.isRegExp(key)) return key;
+            else return key.split('');
+        });
+        _.forEach(keyChunks, (chunk, i) => {
+            let nextStep = chunk instanceof RegExp ?
+                trie.get(_.find([...trie.keys()], k => _.isEqual(chunk, k))!) :
+                trie.get(chunk);
+
+            let isLastChunk = i === keyChunks.length - 1;
+
+            if (isLastChunk) {
                 // We're done - write our value into trie[char]
 
                 if (isLeafValue(nextStep)) {
                     throw new Error('Duplicate key'); // Should really never happen
                 } else if (typeof nextStep === 'object') {
                     // We're half way down another key - add an empty branch
-                    nextStep[''] = value;
+                    nextStep.set('', value);
                 } else {
                     // We're a fresh leaf at the end of a branch
-                    trie[char] = value;
+                    trie.set(chunk, value);
                 }
             } else {
                 // We have more to go - iterate into trie[char]
 
                 if (isLeafValue(nextStep)) {
                     // We're at what is currently a leaf value
                     // Transform it into a node with '' for the value.
-                    nextStep = { '': nextStep };
-                    trie[char] = nextStep;
+                    nextStep = new Map([['', nextStep]]);
+                    trie.set(chunk, nextStep);
                 } else if (typeof nextStep === 'undefined') {
                     // We're adding a new branch to the trie
-                    nextStep = {};
-                    trie[char] = nextStep;
+                    nextStep = new Map();
+                    trie.set(chunk, nextStep);
                 }
 
                 trie = nextStep;
             }
         });
-    });
+    }
 
     return root;
 }
@@ -59,45 +73,55 @@ export function buildNaiveTrie(map: { [key: string]: string | string[] }): TrieD
 export function optimizeTrie(trie: TrieData): TrieData {
     if (_.isString(trie)) return trie;
 
-    const keys = Object.keys(trie);
+    const keys = [...trie.keys()].filter(k => k !== '');
 
     if (keys.length === 0) return trie;
 
     if (keys.length === 1) {
+        // If this level has one string key, combine it with the level below
         const [key] = keys;
-        const child = trie[key]!;
+        const child = trie.get(key)!;
 
+        // If the child is a final value, we can't combine this key with it, and we're done
+        // TODO: Could optimize further here, and pull the child up in this case?
+        // (Only if trie.size === 1 too). Seems unnecessary for now, a little risky.
         if (isLeafValue(child)) return trie;
 
-        // Don't combine if our child has a leaf node attached - this would break
-        // search (en route leaf nodes need to always be under '' keys)
-        if (!child['']) {
-            // Return the only child, with every key prefixed with this key
-            const newChild = optimizeTrie(
-                _.mapKeys(child, (_value, childKey) => key + childKey)
+        if (
+            // Don't combine if our child has a leaf node attached - this would break
+            // search (en route leaf nodes need to always be under '' keys)
+            !child.get('') &&
+            // If this key or any child key is a regex, we don't try to combine the
+            // keys together. It's possible to do so, but a little messy,
+            // not strictly necessary, and hurts runtime perf (testing up to N regexes
+            // is worse than testing 1 regex + 1 string hash lookup).
+            !_.isRegExp(keys[0]) &&
+            !_.some([...child.keys()], k => _.isRegExp(k))
+        ) {
+            // Replace this node with the only child, with every key prefixed with this key
+            const collapsedChild = mapMap(child, (childKey, value) =>
+                // We know keys are strings because we checked above
+                [key + (childKey as string), value]
             );
-
-            return newChild;
-        }
-    }
-
-    if (keys.length === 2 && _.includes(keys, '')) {
-        const [key] = keys.filter(k => k !== '');
-        const child = trie[key]!;
-
-        const childKeys = Object.keys(child);
-        if (!isLeafValue(child) && childKeys.length === 1 && childKeys[0] !== '') {
-            // If child has only one key and it's not '', pull it up.
-            return optimizeTrie({
-                '': trie[''],
-                [key + childKeys[0]]: child[childKeys[0]]
-            });
+            // We might still have an en-route leaf node at this level - don't lose it.
+            if (trie.get('')) collapsedChild.set('', trie.get(''));
+            // Then we reoptimize this same level again (we might be able to to collapse further)
+            return optimizeTrie(collapsedChild);
         }
     }
 
     // Recursive DFS through the child values to optimize them in turn
-    return _.mapValues(trie, (child) => {
-        if (isLeafValue(child)) return child;
-        else return optimizeTrie(child!);
+    return mapMap(trie, (key, child): [string | RegExp, TrieValue] => {
+        if (isLeafValue(child)) return [key, child];
+        else return [key, optimizeTrie(child!)];
     });
+}
+
+function mapMap<K, V, K2, V2>(
+    map: Map<K, V>,
+    mapping: (a: K, b: V) => [K2, V2]
+): Map<K2, V2> {
+    return new Map(
+        Array.from(map, ([k, v]) => mapping(k, v))
+    );
 }