From 1ff7d3cb4fddb3873260cc92bca0398493215af4 Mon Sep 17 00:00:00 2001 From: William Mantly Date: Tue, 29 Dec 2020 13:15:27 -0500 Subject: [PATCH] added caching for complex lookups --- nodejs/controler/host.js | 10 +++- nodejs/models/host.js | 97 ++++++++++++++++++++++-------- nodejs/utils/redis_model.js | 4 +- proxy.conf | 116 ++++++++++++++++++++++++++++++++++++ test.lua | 30 ++++++++++ 5 files changed, 230 insertions(+), 27 deletions(-) create mode 100644 proxy.conf create mode 100644 test.lua diff --git a/nodejs/controler/host.js b/nodejs/controler/host.js index 72a1e59..778563e 100644 --- a/nodejs/controler/host.js +++ b/nodejs/controler/host.js @@ -8,7 +8,15 @@ const conf = require('../app').conf; const socket = new SocketServerJson({ socketFile: conf.socketFile, onData: function(data, clientSocket) { - clientSocket.write(JSON.stringify(Host.lookUp(data['domain']) || {host: 'none'})); + let host = Host.lookUp(data['domain']); + clientSocket.write(JSON.stringify(host || {host: 'none'})); + if(host){ + try{ + Host.addCache(data['domain'], host) + }catch(error){ + console.error('Should never get this error...', error) + } + } }, onListen: function(){ console.log('listening') diff --git a/nodejs/models/host.js b/nodejs/models/host.js index 96e01c7..f03b344 100755 --- a/nodejs/models/host.js +++ b/nodejs/models/host.js @@ -1,6 +1,8 @@ 'use strict'; -const Host = require('../utils/redis_model')({ +const RedisModel = require('../utils/redis_model'); + +const Host = RedisModel({ _name: 'host', _key: 'host', _keyMap: { @@ -13,9 +15,54 @@ const Host = require('../utils/redis_model')({ 'targetPort': {isRequired: true, type: 'number', min:0, max:65535}, 'forcessl': {isRequired: false, default: true, type: 'boolean'}, 'targetssl': {isRequired: false, default: false, type: 'boolean'}, + 'created_by': {isRequired: true, type: 'string', min: 3, max: 500}, + 'is_cahced': {default: false, isRequired: false, type: 'boolean',}, + + } }); +const Cached = RedisModel({ + _name: 'cached', + _key: 'host', + _keyMap: { + 'host': {isRequired: true, type: 'string', min: 3, max: 500}, + 'parent': {isRequired: true, type: 'string', min: 3, max: 500}, + } +}); + + +Host.addCache = async function(host, parentOBJ){ + try{ + await Host.__proto__.add.apply(this, [{...parentOBJ, host, is_cahced: true}, true]) + await Cached.add({ + host: host, + parent: parentOBJ.host + }); + }catch(error){ + console.error('add cahce error', {...parentOBJ, host, is_cahced: true}, error) + throw error; + } +}; + +Host.bustCache = async function(parent){ + try{ + let cached = await Cached.listDetail(); + for(let cache of cached){ + if(cache.parent == parent){ + let host = await Host.get(cache.host); + await Host.__proto__.remove.apply(host); + await cache.remove(); + } + } + + }catch(error){ + console.error('bust cache error', error) + + throw error; + } +} + Host.add = async function(){ try{ let out = await Host.__proto__.add.apply(this, arguments) @@ -30,6 +77,7 @@ Host.add = async function(){ Host.update = async function(data, key){ try{ let out = await Host.__proto__.update.apply(this, arguments) + await Host.bustCache(this.host) await Host.buildLookUpObj() return out; @@ -42,6 +90,7 @@ Host.remove = async function(){ try{ let out = await Host.__proto__.remove.apply(this, arguments) await Host.buildLookUpObj() + await Host.bustCache(this.host) return out; } catch(error){ @@ -153,33 +202,33 @@ Host.lookUpReady = async function(){ module.exports = {Host}; -// (async function(){ +(async function(){ -// await Host.lookUpReady(); + await Host.lookUpReady(); -// // console.log(Host.lookUpObj) + // console.log(Host.lookUpObj) -// // console.log(Host.lookUpObj['com']['vm42']) + // console.log(Host.lookUpObj['com']['vm42']) -// // console.log('test-res', await Host.lookUp('payments.718it.biz')) + // console.log('test-res', await Host.lookUp('payments.718it.biz')) -// let count = 6 -// console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz') -// console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined) -// console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**') -// console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com') -// console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com') -// console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com') -// console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com') -// console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**') -// console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz') -// console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined) -// console.log(count++, Host.lookUp('blah.biz') === undefined) -// console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net') -// console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com') -// console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com') -// console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**') -// console.log(count++, Host.lookUp('718it.biz').host === '718it.biz') + let count = 6 + console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz') + console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined) + console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**') + console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com') + // console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com') + console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com') + console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com') + console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**') + console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz') + console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined) + console.log(count++, Host.lookUp('blah.biz') === undefined) + console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net') + console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com') + console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com') + console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**') + console.log(count++, Host.lookUp('718it.biz').host === '718it.biz') -// })() +})() diff --git a/nodejs/utils/redis_model.js b/nodejs/utils/redis_model.js index dffe9d0..4bf70d6 100644 --- a/nodejs/utils/redis_model.js +++ b/nodejs/utils/redis_model.js @@ -83,7 +83,7 @@ table.listDetail = async function(){ return out }; -table.add = async function(data){ +table.add = async function(data, noMemberAdd){ // Add a entry to this redis table. try{ @@ -102,7 +102,7 @@ table.add = async function(data){ } // Add the key to the members for this redis table - await client.SADD(this._name, data[this._key]); + if(!noMemberAdd) await client.SADD(this._name, data[this._key]); // Add the values for this entry. for(let key of Object.keys(data)){ diff --git a/proxy.conf b/proxy.conf new file mode 100644 index 0000000..0ea3aeb --- /dev/null +++ b/proxy.conf @@ -0,0 +1,116 @@ +map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; +} + +server { + listen 80; + + include autossl.conf; + + set_real_ip_from 192.168.1.0/24; + real_ip_header X-Real-IP; + real_ip_recursive on; + + location / { + + set $target ''; + set $target_scheme 'http'; + set $target_port ''; + set $header_host $host; + + access_by_lua ' + local host = ngx.var.host + local uri = ngx.var.uri + local scheme = ngx.var.scheme + + local json = require "cjson" + local socket = assert(require "socket.unix"()) + local function connect(path) + assert(socket:settimeout(.1)) + local status,err = pcall(function() assert(socket:connect(path)) end) + if status then return true end + return false + end + + if not host then + ngx.log(ngx.ERR, "no host header found") + return ngx.exit(499) + end + + local redis = require "resty.redis" + local red = redis:new() + + red:set_timeout(1000) -- 1 second + + local ok, err = red:connect("127.0.0.1", 6379) + if not ok then + ngx.log(ngx.ERR, "failed to connect to redis: ", err) + return ngx.exit(598) + end + + local res, err = red:hgetall("proxy_host_"..host) + local res = red:array_to_hash(res) + + if not res["ip"] then + if connect("/var/run/proxy_lookup.socket") then + assert(socket:send(json.encode({domain = host}))) + while 1 do + local s, status, partial = socket:receive() + if partial then + res = json.decode(partial) + socket:close() + break + end + end + end + end + + if not res["ip"] then + ngx.log(ngx.ERR, "no host found for key ", host) + return ngx.exit(406) + end + + if scheme == "http" then + if res["forcessl"] == "true" then + return ngx.redirect("https://"..host..uri, 301) + end + end + + if res["targetssl"] == "true" then + ngx.var.target_scheme = "https" + end + + if res["host-pass-though"] == "false" then + ngx.var.header_host = res["ip"] + end + + ngx.var.target = res["ip"] + ngx.var.target_port = res["targetPort"] + '; + + + resolver 192.168.1.1 ipv6=off; #8.8.4.4; # use Google's open DNS server + + proxy_http_version 1.1; + proxy_pass_request_headers on; + proxy_pass $target_scheme://$target:$target_port; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_ssl_session_reuse on; + proxy_intercept_errors off; + + proxy_set_header Host $header_host; + add_header X-Target-Host $target; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $target_scheme; + proxy_set_header Referer $target_scheme://$header_host; + proxy_set_header Accept-Language $http_accept_language; + proxy_set_header User-Agent $http_user_agent; + + sub_filter $target $host; + sub_filter_once off; + } +} diff --git a/test.lua b/test.lua new file mode 100644 index 0000000..5457a16 --- /dev/null +++ b/test.lua @@ -0,0 +1,30 @@ +local socket = assert(require "socket.unix"()) +local function connect(path) + assert(socket:settimeout(.05)) + local status,err = pcall(function() assert(socket:connect(path)) end) + if status then return true end + io.stderr:write(err.." ("..path..")\n") + return false +end + +local json = require "lunajson" +local res + +if connect("/var/run/proxy_lookup.socket") then + host = "payments.blah.com" + + assert(socket:send(json.encode({domain = host}))) + while 1 do + local s, status, partial = socket:receive() + if partial then + res = json.decode(partial) + break + end + end +end + + +print(res['ip']) + + +