theta42/proxy
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added caching for complex lookups

Browse Source
This commit is contained in:
William Mantly
2020-12-29 13:15:27 -05:00
parent bce23ab9fe
commit 1ff7d3cb4f
5 changed files with 230 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/controler/host.js
+9 -1
View File
@@ -8,7 +8,15 @@ const conf = require('../app').conf;
const socket = new SocketServerJson({
socketFile: conf.socketFile,
onData: function(data, clientSocket) {
clientSocket.write(JSON.stringify(Host.lookUp(data['domain']) || {host: 'none'}));
let host = Host.lookUp(data['domain']);
clientSocket.write(JSON.stringify(host || {host: 'none'}));
if(host){
try{
Host.addCache(data['domain'], host)
}catch(error){
console.error('Should never get this error...', error)
}
}
},
onListen: function(){
console.log('listening')
nodejs/models/host.js
+73 -24
View File
@@ -1,6 +1,8 @@
'use strict';
const Host = require('../utils/redis_model')({
const RedisModel = require('../utils/redis_model');
const Host = RedisModel({
_name: 'host',
_key: 'host',
_keyMap: {
@@ -13,9 +15,54 @@ const Host = require('../utils/redis_model')({
'targetPort': {isRequired: true, type: 'number', min:0, max:65535},
'forcessl': {isRequired: false, default: true, type: 'boolean'},
'targetssl': {isRequired: false, default: false, type: 'boolean'},
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'is_cahced': {default: false, isRequired: false, type: 'boolean',},
}
});
const Cached = RedisModel({
_name: 'cached',
_key: 'host',
_keyMap: {
'host': {isRequired: true, type: 'string', min: 3, max: 500},
'parent': {isRequired: true, type: 'string', min: 3, max: 500},
}
});
Host.addCache = async function(host, parentOBJ){
try{
await Host.__proto__.add.apply(this, [{...parentOBJ, host, is_cahced: true}, true])
await Cached.add({
host: host,
parent: parentOBJ.host
});
}catch(error){
console.error('add cahce error', {...parentOBJ, host, is_cahced: true}, error)
throw error;
}
};
Host.bustCache = async function(parent){
try{
let cached = await Cached.listDetail();
for(let cache of cached){
if(cache.parent == parent){
let host = await Host.get(cache.host);
await Host.__proto__.remove.apply(host);
await cache.remove();
}
}
}catch(error){
console.error('bust cache error', error)
throw error;
}
}
Host.add = async function(){
try{
let out = await Host.__proto__.add.apply(this, arguments)
@@ -30,6 +77,7 @@ Host.add = async function(){
Host.update = async function(data, key){
try{
let out = await Host.__proto__.update.apply(this, arguments)
await Host.bustCache(this.host)
await Host.buildLookUpObj()
return out;
@@ -42,6 +90,7 @@ Host.remove = async function(){
try{
let out = await Host.__proto__.remove.apply(this, arguments)
await Host.buildLookUpObj()
await Host.bustCache(this.host)
return out;
} catch(error){
@@ -153,33 +202,33 @@ Host.lookUpReady = async function(){
module.exports = {Host};
// (async function(){
(async function(){
// await Host.lookUpReady();
await Host.lookUpReady();
// // console.log(Host.lookUpObj)
// console.log(Host.lookUpObj)
// // console.log(Host.lookUpObj['com']['vm42'])
// console.log(Host.lookUpObj['com']['vm42'])
// // console.log('test-res', await Host.lookUp('payments.718it.biz'))
// console.log('test-res', await Host.lookUp('payments.718it.biz'))
// let count = 6
// console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz')
// console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined)
// console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**')
// console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com')
// console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com')
// console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com')
// console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com')
// console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**')
// console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz')
// console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined)
// console.log(count++, Host.lookUp('blah.biz') === undefined)
// console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net')
// console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com')
// console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com')
// console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**')
// console.log(count++, Host.lookUp('718it.biz').host === '718it.biz')
let count = 6
console.log(count++, Host.lookUp('payments.718it.biz').host === 'payments.718it.biz')
console.log(count++, Host.lookUp('sd.blah.test.vm42.com') === undefined)
console.log(count++, Host.lookUp('payments.test.com').host === 'payments.**')
console.log(count++, Host.lookUp('test.sample.other.exmaple.com').host === '**.exmaple.com')
// console.log(count++, Host.lookUp('stan.test.vm42.com').host === 'stan.test.vm42.com')
console.log(count++, Host.lookUp('test.vm42.com').host === 'test.vm42.com')
console.log(count++, Host.lookUp('blah.test.vm42.com').host === '*.test.vm42.com')
console.log(count++, Host.lookUp('payments.example.com').host === 'payments.**')
console.log(count++, Host.lookUp('info.wma.users.718it.biz').host === 'info.*.users.718it.biz')
console.log(count++, Host.lookUp('infof.users.718it.biz') === undefined)
console.log(count++, Host.lookUp('blah.biz') === undefined)
console.log(count++, Host.lookUp('test.1.2.718it.net').host === 'test.*.*.718it.net')
console.log(count++, Host.lookUp('test1.exmaple.com').host === 'test1.exmaple.com')
console.log(count++, Host.lookUp('other.exmaple.com').host === '*.exmaple.com')
console.log(count++, Host.lookUp('info.payments.example.com').host === 'info.**')
console.log(count++, Host.lookUp('718it.biz').host === '718it.biz')
// })()
})()
nodejs/utils/redis_model.js
+2 -2
View File
@@ -83,7 +83,7 @@ table.listDetail = async function(){
return out
};
table.add = async function(data){
table.add = async function(data, noMemberAdd){
// Add a entry to this redis table.
try{
@@ -102,7 +102,7 @@ table.add = async function(data){
}
// Add the key to the members for this redis table
await client.SADD(this._name, data[this._key]);
if(!noMemberAdd) await client.SADD(this._name, data[this._key]);
// Add the values for this entry.
for(let key of Object.keys(data)){
proxy.conf
+116
View File
@@ -0,0 +1,116 @@
map $http_upgrade $connection_upgrade {
default Upgrade;
'' close;
}
server {
listen 80;
include autossl.conf;
set_real_ip_from 192.168.1.0/24;
real_ip_header X-Real-IP;
real_ip_recursive on;
location / {
set $target '';
set $target_scheme 'http';
set $target_port '';
set $header_host $host;
access_by_lua '
local host = ngx.var.host
local uri = ngx.var.uri
local scheme = ngx.var.scheme
local json = require "cjson"
local socket = assert(require "socket.unix"())
local function connect(path)
assert(socket:settimeout(.1))
local status,err = pcall(function() assert(socket:connect(path)) end)
if status then return true end
return false
end
if not host then
ngx.log(ngx.ERR, "no host header found")
return ngx.exit(499)
end
local redis = require "resty.redis"
local red = redis:new()
red:set_timeout(1000) -- 1 second
local ok, err = red:connect("127.0.0.1", 6379)
if not ok then
ngx.log(ngx.ERR, "failed to connect to redis: ", err)
return ngx.exit(598)
end
local res, err = red:hgetall("proxy_host_"..host)
local res = red:array_to_hash(res)
if not res["ip"] then
if connect("/var/run/proxy_lookup.socket") then
assert(socket:send(json.encode({domain = host})))
while 1 do
local s, status, partial = socket:receive()
if partial then
res = json.decode(partial)
socket:close()
break
end
end
end
end
if not res["ip"] then
ngx.log(ngx.ERR, "no host found for key ", host)
return ngx.exit(406)
end
if scheme == "http" then
if res["forcessl"] == "true" then
return ngx.redirect("https://"..host..uri, 301)
end
end
if res["targetssl"] == "true" then
ngx.var.target_scheme = "https"
end
if res["host-pass-though"] == "false" then
ngx.var.header_host = res["ip"]
end
ngx.var.target = res["ip"]
ngx.var.target_port = res["targetPort"]
';
resolver 192.168.1.1 ipv6=off; #8.8.4.4; # use Google's open DNS server
proxy_http_version 1.1;
proxy_pass_request_headers on;
proxy_pass $target_scheme://$target:$target_port;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_ssl_session_reuse on;
proxy_intercept_errors off;
proxy_set_header Host $header_host;
add_header X-Target-Host $target;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $target_scheme;
proxy_set_header Referer $target_scheme://$header_host;
proxy_set_header Accept-Language $http_accept_language;
proxy_set_header User-Agent $http_user_agent;
sub_filter $target $host;
sub_filter_once off;
}
}
test.lua
+30
View File
@@ -0,0 +1,30 @@
local socket = assert(require "socket.unix"())
local function connect(path)
assert(socket:settimeout(.05))
local status,err = pcall(function() assert(socket:connect(path)) end)
if status then return true end
io.stderr:write(err.." ("..path..")\n")
return false
end
local json = require "lunajson"
local res
if connect("/var/run/proxy_lookup.socket") then
host = "payments.blah.com"
assert(socket:send(json.encode({domain = host})))
while 1 do
local s, status, partial = socket:receive()
if partial then
res = json.decode(partial)
break
end
end
end
print(res['ip'])