updated backend

Author: standlove

config/default.js

@@ -17,6 +17,9 @@ let config = {
   PORT: process.env.PORT || 3003,
   API_VERSION: 'api/1.0',
   WORK_ROOT: path.join(__dirname, '..'),
+  IDLE_TIMEOUT_IN_MINS: 30,
+  IDLE_CONNECTION_POLL_INTERVAL: 1,
+  MAX_RETRIES: 5,
   defaultExternalConfigLink: 'https://raw.githubusercontent.com/jiangliwu/static-files/master/config.json',
   defaultExternalConfigFileName: 'defaultExternalCache.json',
   externalConfigFileName: 'externalCache.json',

lib/redis.js

@@ -2,6 +2,11 @@ const Redis = require('ioredis');
 const errors = require('./common/errors');
 const _ = require('lodash');
 const redisDump = require('./tools/redis-dump');
+const {
+  IDLE_TIMEOUT_IN_MINS,
+  IDLE_CONNECTION_POLL_INTERVAL,
+  MAX_RETRIES
+ } = require('config');
 
 /**
  * cached redis instance
@@ -16,21 +21,41 @@ const DISCONNECTED = 'disconnected';
  * @return {Promise<any>}
  */
 async function connect(body) {
+  let redisInstance = redisInstanceCache[body.id];
+  if (redisInstance) {
+    redisInstance.retries = 0;
+    redisInstance.lastAccessed = Date.now();
+  }
+
+  const newProps = getRedisInstanceProperties(body);
   // already exist
-  if (redisInstanceCache[body.id]) {
-    const redis = redisInstanceCache[body.id];
-    if (redis.redisStatus == DISCONNECTED) {
-      redis.connect();
+  if (redisInstance && _.isEqual(redisInstance.props, newProps)) {
+    if (redisInstance.status === DISCONNECTED) {
+      throw errors.newConnectFailedError(body.id);
     }
     return Promise.resolve(body);
   }
 
+  // properties changed
+  if (redisInstance) {
+    redisInstanceCache[body.id] = null;
+    redisInstance.connection.disconnect();
+  } else {
+    redisInstance = {
+      props: getRedisInstanceProperties(body),
+      retries: 0,
+    };
+  }
+
   return new Promise((resolve, reject) => {
     const redis = new Redis({
-      host: body.serverModel.ip, port: body.serverModel.port, db: body.serverModel.db,
+      host: body.serverModel.ip, 
+      port: body.serverModel.port, 
+      db: body.serverModel.db,
       password: body.serverModel.password,
       showFriendlyErrorStack: true,
       autoResendUnfulfilledCommands: false,
+      maxRetriesPerRequest: MAX_RETRIES,
     });
 
     const timeoutHandler = setTimeout(() => {
@@ -40,34 +65,98 @@ async function connect(body) {
 
     redis.on('error', (e) => {
       console.error(e);
-      redis.redisStatus = DISCONNECTED;
+      redisInstance.status = DISCONNECTED;
     });
 
     redis.on('end', () => {
+      console.log('end');
       redisInstanceCache[body.id] = null;
     });
 
+    redis.on('reconnecting', () => {
+      console.log('reconnecting');
+      redisInstance.retries++;
+      if(redisInstance.retries >= MAX_RETRIES) {
+        redisInstanceCache[body.id] = null;
+        redis.disconnect();
+      }
+    })
+
     redis.on('ready', () => {
-      redis.redisStatus = CONNECTED;
-      redisInstanceCache[body.id] = redis;
-      
+      console.log('ready');
+      redisInstance.connection = redis;
+      redisInstance.status = CONNECTED;
+      redisInstance.lastAccessed = Date.now();
+      redisInstance.retries = 0;
+
+      redisInstanceCache[body.id] = redisInstance;
+
       resolve(body);
       clearTimeout(timeoutHandler);
     });
   });
 }
 
+
+/**
+ * disconnect redis
+ * @param body
+ * @return {Promise<any>}
+ */
+async function disconnect(body) {
+  const redisInstance = redisInstanceCache[body.id];
+
+  if (redisInstance) {
+    redisInstanceCache[body.id] = null;
+    redisInstance.connection.disconnect();
+  }
+}
+
+/**
+ * Get the properties of redis instance
+ * @param body
+ * @return {Object} - the properties used to identify modifications in instance config
+ */
+function getRedisInstanceProperties(body) {
+  return _.pick(
+    body,
+    'serverModel.ip',
+    'serverModel.port',
+    'serverModel.db',
+    'serverModel.password'
+  );
+}
+
+/**
+ * Poll for idle instances. If found, disconnect and remove them from cache
+ */
+function pollIdleConnections() {
+  const idleTimeoutInMS = IDLE_TIMEOUT_IN_MINS * 60 * 1000;
+  setInterval(() => {
+    const idleThreshold = Date.now() - idleTimeoutInMS;
+    for (let id in redisInstanceCache) {
+      const redisInstance = redisInstanceCache[id];
+      if (redisInstance && redisInstance.lastAccessed < idleThreshold) {
+        redisInstanceCache[id] = null;
+        redisInstance.connection.disconnect();
+      }
+    }
+  }, IDLE_CONNECTION_POLL_INTERVAL * 60 * 1000);
+}
+
 /**
  * Get redis connection
- * 
+ *
  * @param query the query params
  */
 function getRedisConnection(query) {
-  const redis = redisInstanceCache[query.id];
-  if (!redis || redis.redisStatus !== CONNECTED) {
+  const redisInstance = redisInstanceCache[query.id];
+  if (!redisInstance || redisInstance.status !== CONNECTED) {
     throw errors.newConnectFailedError(query.id);
   }
-  return redis;
+
+  redisInstance.lastAccessed = Date.now();
+  return redisInstance.connection;
 }
 
 /**
@@ -88,7 +177,7 @@ async function fetchTree(query) {
   let keys;
   try {
     keys = query.keys || (await redis.keys('*'));
-    
+
     for (let i = 0; i < keys.length; i++) { // process types
       const key = keys[i];
       const type = await redis.type(key);
@@ -203,6 +292,9 @@ async function dump(query) {
   }
 }
 
+// trigger the poller on start
+pollIdleConnections();
+
 module.exports = {
-  connect, fetchTree, call, dump
+  connect, fetchTree, call, dump, disconnect
 };

lib/route.js

@@ -24,6 +24,11 @@ module.exports = {
       method: async req => await redis.connect(req.payload),
     },
   },
+  '/redis/disconnect': {
+    post: {
+      method: async req => await redis.disconnect(req.payload),
+    },
+  },
   '/redis/fetch': {
     get: {
       method: async req => await redis.fetchTree(req.query),

redis-cluster.conf

@@ -0,0 +1,5 @@
+port 7000
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

verification.md

@@ -0,0 +1,30 @@
+## Verfication Guide
+
+#1 Support redis cluster
+* To verify the first point, set up a redis cluster following the updated readme and connect from the UI.
+* Once connected, you should be able to send commands and recieve response from cluster like this:
+    ```
+      > get foo
+      (nil)
+      > set foo bar
+      Ok
+      > get foo
+      bar
+    ```
+#1.1 And in the connect endpoint, when the connection with id already exists, you should also check if the properties (e.g. host name, port, cluster etc...) are changed or not. If changed, disconnect the old one and reconnect. 
+
+* To verify this, manually edit the localStorage, refresh the page and try to connect.
+
+#2 Add a disconnect endpoint to the backend. 
+
+* This is implemented and integrated with the disconnect button above the instance info
+
+#3 To verify if the backend removes the cache, 
+* run the server in dev mode (using ndb is recommended for running in debug mode) put a breakpoint in reconnecting callback in redis.js. 
+* set MAX_RETRIES in config/default.js. 
+
+#3.1 To verify idle timeout
+* set a very low value (say 1) in the config/default.js > IDLE_TIMEOUT_IN_MINS variable. 
+* wait for 1 minute and run `get foo` from the UI. This will fail unless you manually reconnected in the meantime.
+
+