Merge pull request #41 from wmantly/wildcards

Wildcards

README.md

@@ -45,6 +45,12 @@ like Digital Ocean will do just fine.
 ```bash
 apt install luarocks
 sudo luarocks install lua-resty-auto-ssl
+sudo luarocks install lua-resty-socket
+sudo luarocks install lua-socket
+sudo luarocks install socket
+sudo luarocks install luasocket
+sudo luarocks install luasocket-unix
+sudo luarocks install lua-cjson
 ```
 
 * openresty config

Vagrantfile

@@ -73,7 +73,7 @@ Vagrant.configure("2") do |config|
 
     if ! which berks >/dev/null; then
       gem install ruby-shadow berkshelf --no-document
-      # ln -s /opt/chef/embedded/bin/berks /usr/local/bin/berks
+      ln -s /opt/chef/embedded/bin/berks /usr/local/bin/berks
     fi
 
     cd /vagrant

nodejs/app.js

@@ -40,6 +40,10 @@ app.use('/api/user', middleware.auth, require('./routes/user'));
 // API routes for working with hosts. All endpoints need to be have valid user.
 app.use('/api/host', middleware.auth, require('./routes/host'));
 
+app.controler = {
+  host: require('./controler/host')
+}
+
 // Catch 404 and forward to error handler. If none of the above routes are
 // used, this is what will be called.
 app.use(function(req, res, next) {

nodejs/conf/base.js

@@ -9,5 +9,6 @@ module.exports = {
 		searchBase: 'ou=people,dc=theta42,dc=com',
 		userFilter: '(objectClass=inetOrgPerson)',
 		userNameAttribute: 'uid'
-	}
+	},
+	socketFile: '/var/run/proxy_lookup.socket'
 };

nodejs/controler/host.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const {Host} = require('../models/host');
+const {SocketServerJson} = require('../models/socket_server_json');
+const conf = require('../app').conf;
+
+
+const socket = new SocketServerJson({
+	socketFile: conf.socketFile,
+	onData: function(data, clientSocket) {
+		let host = Host.lookUp(data['domain']);
+		clientSocket.write(JSON.stringify(host || {host: 'none'}));
+		if(host){
+			try{
+				Host.addCache(data['domain'], host)
+			}catch(error){
+				console.error('Should never get this error...', error)
+			}
+		}
+	},
+	onListen: function(){
+		console.log('listening')
+	}
+});

nodejs/models/host.js

@@ -1,6 +1,8 @@
 'use strict';
 
-const Host = require('../utils/redis_model')({
+const RedisModel = require('../utils/redis_model');
+
+const Host = RedisModel({
 	_name: 'host',
 	_key: 'host',
 	_keyMap: {
@@ -13,7 +15,220 @@ const Host = require('../utils/redis_model')({
 		'targetPort': {isRequired: true, type: 'number', min:0, max:65535},
 		'forcessl': {isRequired: false, default: true, type: 'boolean'},
 		'targetssl': {isRequired: false, default: false, type: 'boolean'},
+		'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
+		'is_cache': {default: false, isRequired: false, type: 'boolean',},
+
+
 	}
 });
 
+const Cached = RedisModel({
+	_name: 'cached',
+	_key: 'host',
+	_keyMap: {
+		'host': {isRequired: true, type: 'string', min: 3, max: 500},
+		'parent': {isRequired: true, type: 'string', min: 3, max: 500},
+	}
+});
+
+
+Host.addCache = async function(host, parentOBJ){
+	try{
+		await Host.__proto__.add.apply(this, [{...parentOBJ, host, is_cache: true}, true])
+		await Cached.add({
+			host: host,
+			parent: parentOBJ.host
+		});
+	}catch(error){
+		console.error('add cahce error', {...parentOBJ, host, is_cache: true}, error)
+		throw error;
+	}
+};
+
+Host.bustCache = async function(parent){
+	try{
+		let cached = await Cached.listDetail();
+		for(let cache of cached){
+			if(cache.parent == parent){
+				let host = await Host.get(cache.host);
+				await Host.__proto__.remove.apply(host);
+				await cache.remove();
+			}
+		}
+
+	}catch(error){
+		console.error('bust cache error', error)
+
+		throw error;
+	}
+}
+
+Host.add = async function(){
+	try{
+		let out = await Host.__proto__.add.apply(this, arguments)
+		await Host.buildLookUpObj()
+
+		return out;
+	} catch(error){
+		throw error;
+	}
+};
+
+Host.update = async function(data, key){
+	try{
+		let out = await Host.__proto__.update.apply(this, arguments)
+		await Host.bustCache(this.host)
+		await Host.buildLookUpObj()
+
+		return out;
+	} catch(error){
+		throw error;
+	}
+};
+
+Host.remove = async function(){
+	try{
+		let out = await Host.__proto__.remove.apply(this, arguments)
+		await Host.buildLookUpObj()
+		await Host.bustCache(this.host)
+
+		return out;
+	} catch(error){
+		throw error;
+	}
+};
+
+Host.lookUpObj = {};
+
+Host.buildLookUpObj = async function(){
+	/*
+	Build a look up tree for domain records in the redis back end to allow
+	complex looks with wildcards.
+	*/
+
+	// Hold lookUp ready while the look up object is being built.
+	this.__lookUpIsReady = false;
+	this.lookUpObj = {};
+
+	try{
+
+		// Loop over all the hosts in the redis.
+		for(let host of await this.list()){
+
+			// Spit the hosts on "." into its fragments .
+			let fragments = host.split('.');
+
+			// Hold a pointer to the root of the lookup tree.
+			let pointer = this.lookUpObj;
+
+			// Walk over each fragment, popping from right to left. 
+			while(fragments.length){
+				let fragment = fragments.pop();
+
+				// Add a branch to the lookup at the current position
+				if(!pointer[fragment]){
+					pointer[fragment] = {};
+				}
+
+				// Add the record(leaf) when we hit the a full host name.
+				// #record denotes a leaf node on this tree.
+				if(fragments.length === 0){
+					pointer[fragment]['#record'] = await this.get(host)
+				}
+
+				// Advance the pointer to the next level of the tree.
+				pointer = pointer[fragment];
+			}
+		}
+
+		// When the look up tree is finished, remove the ready hold.
+		this.__lookUpIsReady = true;
+
+	}catch(error){
+		console.error(error);
+	}
+};
+
+Host.lookUp = function(host){
+	/*
+	Perform a complex lookup of @host on the look up tree.
+	*/
+
+
+	// Hold a pointer to the root of the look up tree
+	let place = this.lookUpObj;
+
+	// Hold the last passed long wild card.
+	let last_resort = {};
+
+	// Walk over each fragment of the host, from right to left
+	for(let fragment of host.split('.').reverse()){
+
+		// If a long wild card is found on this level, hold on to it
+		if(place['**']) last_resort = place['**'];
+
+		// If we have a match for the current fragment, update the current pointer
+		// A match in the lookup tree takes priority being a more exact match.
+		if({...last_resort, ...place}[fragment]){
+			place = {...last_resort, ...place}[fragment];
+		// If we have a not exact fragment match, a wild card will do.
+		}else if(place['*']){
+			place = place['*']
+		// If no fragment can be matched, continue with the long wild card branch.
+		}else if(last_resort){
+			place = last_resort;
+		}
+	}
+
+	// After the tree has been traversed, see if we have leaf node to return. 
+	if(place && place['#record']) return place['#record'];
+};
+
+Host.__lookUpIsReady = false;
+
+Host.lookUpReady = async function(){
+	/*
+	Wait for the lookup tree to be built.
+	*/
+
+	// Check every 5ms to see if the look up tree is ready
+	while(!this.__lookUpIsReady) await new Promise(r => setTimeout(r, 5));
+	return true;
+};
+
+(async function(){
+	await Host.buildLookUpObj();
+})()
+
 module.exports = {Host};
+
+// (async function(){
+
+// 	await Host.lookUpReady();
+
+// 	// console.log(Host.lookUpObj)
+
+// 	// console.log(Host.lookUpObj['com']['vm42'])
+
+// 	// console.log('test-res', await Host.lookUp('payments.718it.biz'))
+
+// 	let count = 6
+// 	console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz')
+// 	console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined)
+// 	console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**')
+// 	console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com')
+// 	// console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com')
+// 	console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com')
+// 	console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com')
+// 	console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**')	
+// 	console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz')
+// 	console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined)
+// 	console.log(count++, Host.lookUp('blah.biz') === undefined)
+// 	console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net')
+// 	console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com')
+// 	console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com')
+// 	console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**')
+// 	console.log(count++, Host.lookUp('718it.biz').host === '718it.biz')
+
+
+// })()

nodejs/models/socket_server_json.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const net = require('net');
+const fs = require('fs');
+const {CallbackQueue} = require('../utils/callback_queue')
+
+class SocketServerJson {
+	constructor(args){
+		this.socketFile = args.socketFile;
+		this.onData = new CallbackQueue(args.onData, this);
+		this.onListen = new CallbackQueue(args.onListen, this);
+		this.onError = new CallbackQueue(args.onError);
+		this.onCLientNew = new CallbackQueue(args.onCLientNew);
+		this.onCLientClose = new CallbackQueue(args.onCLientClose);
+		this.onCLientError = new CallbackQueue(args.onCLientClose);
+
+		this.onListen.push(function(){
+			fs.chmodSync(args.socketFile, '777');
+		})
+
+		this.listen();
+
+	}
+
+	__resetSocketFile(callback){
+		let instance = this;
+
+		fs.stat(this.socketFile, function (err, stats) {
+			if (stats) {
+				fs.unlink(instance.socketFile, function(err){
+					if(err){
+						// This should never happen.
+						console.error(err);
+					}
+					callback(...arguments)
+				});
+			}else{
+				callback()
+			}
+
+		});
+	}
+
+
+	__setUpServer(){
+
+		let instance = this;
+		this.socket = net.createServer();
+
+		this.socket.on('connection', function(clientSocket){
+			let buffer = '';
+
+			clientSocket.on('data', function(data){
+				buffer += data.toString();
+				try{
+					instance.onData.call(JSON.parse(data), clientSocket)
+					buffer = ''
+					// clientSocket.write(JSON.stringify(Host.lookUp(buffer)|| {host: 'none'}));
+
+				}catch(error){
+					;
+				}
+			});
+
+			clientSocket.on('close', instance.onCLientClose.call.bind(instance.onCLientClose));
+
+			clientSocket.on('error', instance.onCLientError.call.bind(instance.onCLientError));
+
+		});
+
+		this.socket.on('error', this.onError.call.bind(this.onError))
+
+		this.socket.on('listening', this.onListen.call.bind(this.onListen));
+	}
+
+	listen(){
+		let instance = this;
+
+		this.__setUpServer();
+
+		this.__resetSocketFile(function(){
+			instance.socket.listen(instance.socketFile);
+		});
+
+	}
+};
+
+module.exports = {SocketServerJson};

nodejs/package.json

@@ -13,10 +13,11 @@
   },
   "dependencies": {
     "authenticate-pam": "github:WeiAnAn/node-authenticate-pam",
-    "ldapts": "^2.2.1",
-    "extend": "^3.0.2",
+    "bcrypt": "^5.0.0",
     "ejs": "^3.0.1",
     "express": "~4.16.1",
+    "extend": "^3.0.2",
+    "ldapts": "^2.2.1",
     "linux-sys-user": "^1.1.0",
     "redis": "^2.8.0"
   },

nodejs/routes/host.js

@@ -4,38 +4,11 @@ const router = require('express').Router();
 const {Host} = require('../models/host');
 
 
-router.get('/:host', async function(req, res, next){
-	try{
-
-		return res.json({
-			host: req.params.host,
-			results: await Host.get({host: req.params.host})
-		});
-	}catch(error){
-		return next(error);
-	}
-
-});
-
 router.get('/', async function(req, res, next){
 	try{
 		return res.json({
 			hosts:  await Host[req.query.detail ? "listDetail" : "list"]()
 		});
-	}catch(error){
-		next(error)
-	}
-});
-
-router.put('/:host', async function(req, res, next){
-	try{
-		req.body.updated_by = req.user.username;
-		let host = await Host.get(req.params.host);
-		await host.update(req.body);
-
-		return res.json({
-			message: `Host "${req.params.host}" updated.`
-		});
 	}catch(error){
 		return next(error);
 	}
@@ -50,23 +23,61 @@ router.post('/', async function(req, res, next){
 			message: `Host "${req.body.host}" added.`
 		});
 	} catch (error){
-		next(error);
+		return next(error);
 	}
+});
 
+router.get('/:host', async function(req, res, next){
+	try{
+
+		return res.json({
+			host: req.params.host,
+			results: await Host.get({host: req.params.host})
+		});
+	}catch(error){
+		return next(error);
+	}
+});
+
+router.put('/:host', async function(req, res, next){
+	try{
+		req.body.updated_by = req.user.username;
+		let host = await Host.get(req.params.host);
+		await host.update.call(host, req.body);
+
+		return res.json({
+			message: `Host "${req.params.host}" updated.`
+		});
+
+	}catch(error){
+		return next(error);
+
+	}
 });
 
 router.delete('/:host', async function(req, res, next){
-	
 	try{
 		let host = await Host.get(req.params);
-		let count = await host.remove(host);
+		let count = await host.remove.call(host, host);
 
 		return res.json({
 			message: `Host ${req.params.host} deleted`,
 		});
 
 	}catch(error){
-		next(error);
+		return next(error);
+	}
+});
+
+router.get('/lookup/:host', async function(req, res, next){
+	try{
+		return res.json({
+			string: req.params.host,
+			results: await Host.lookUp(req.params.host),
+		});
+
+	}catch(error){
+		return next(error);
 	}
 });
 

nodejs/routes/index.js

@@ -1,19 +1,23 @@
+'use strict';
+
 var express = require('express');
 var router = express.Router();
+const {Host} = require('../models/host');
+
 
 /* GET home page. */
 router.get('/', async function(req, res, next) {
-  res.render('hosts', { title: 'Express' });
+  res.render('hosts', {});
 });
 
 /* GET home page. */
 router.get('/hosts', function(req, res, next) {
-  res.render('hosts', { title: 'Express' });
+  res.render('hosts', {});
 });
 
 /* GET home page. */
 router.get('/users', function(req, res, next) {
-  res.render('users', { title: 'Express' });
+  res.render('users', {});
 });
 
 /* GET home page. */

nodejs/utils/callback_queue.js

@@ -0,0 +1,26 @@
+'use strict';
+
+class CallbackQueue{
+	constructor(callbacks){
+		this.__callbacks = [];
+
+		for(let callback of Array.isArray(callbacks) ? callbacks : [callbacks]){
+			this.push(callback);
+		}
+	}
+
+	push(callback){
+		if(callback instanceof Function){
+			this.__callbacks.push(callback);
+		}
+	}
+
+	call(){
+		let args = arguments;
+		this.__callbacks.forEach(function(callback){
+			callback(...args);
+		}.bind(this))
+	}
+}
+
+module.exports = {CallbackQueue};

nodejs/utils/redis_model.js

@@ -83,7 +83,7 @@ table.listDetail = async function(){
 	return out
 };
 
-table.add = async function(data){
+table.add = async function(data, noMemberAdd){
 	// Add a entry to this redis table.
 	try{
 
@@ -102,7 +102,7 @@ table.add = async function(data){
 		}
 
 		// Add the key to the members for this redis table
-		await client.SADD(this._name, data[this._key]);
+		if(!noMemberAdd) await client.SADD(this._name, data[this._key]);
 
 		// Add the values for this entry.
 		for(let key of Object.keys(data)){