From 4d7bfa1cd19331b2d7ce7f6e5b633dda8dac0a44 Mon Sep 17 00:00:00 2001 From: Thierry FOURNIER Date: Sun, 25 Feb 2018 21:28:05 +0100 Subject: [PATCH] MINOR/DOC: spoe-server: Add documentation This is the documentation and examples. --- contrib/spoa_server/README | 105 ++++++++++------------ contrib/spoa_server/print_r.lua | 68 ++++++++++++++ contrib/spoa_server/ps_lua.lua | 17 ++++ contrib/spoa_server/ps_python.py | 20 +++++ contrib/spoa_server/spoa-server.conf | 33 +++++++ contrib/spoa_server/spoa-server.spoe.conf | 13 +++ 6 files changed, 196 insertions(+), 60 deletions(-) create mode 100644 contrib/spoa_server/print_r.lua create mode 100644 contrib/spoa_server/ps_lua.lua create mode 100644 contrib/spoa_server/ps_python.py create mode 100644 contrib/spoa_server/spoa-server.conf create mode 100644 contrib/spoa_server/spoa-server.spoe.conf diff --git a/contrib/spoa_server/README b/contrib/spoa_server/README index 7e376ee1d6..57ec9c414d 100644 --- a/contrib/spoa_server/README +++ b/contrib/spoa_server/README @@ -1,10 +1,18 @@ -A Random IP reputation service acting as a Stream Processing Offload Agent --------------------------------------------------------------------------- +Multi script langyage Stream Processing Offload Agent +----------------------------------------------------- -This is a very simple service that implement a "random" ip reputation -service. It will return random scores for all checked IP addresses. It only -shows you how to implement a ip reputation service or such kind of services -using the SPOE. +This agent receive SPOP message and process it with script languages. The +language register callback with a message. Each callback receive the list +of arguments with types according with the language capabilities. The +callback write variables which are sent as response when the processing +is done. + + + Compilation +--------------- + +Actually, the server support Lua and Python. Type "make" with the options: +USE_LUA=1 and/or USE_PYTHON=1. Start the service @@ -19,70 +27,47 @@ binary: -d Enable the debug mode -p Specify the port to listen on (default: 12345) -n Specify the number of workers (default: 5) + -f Load script according with the supported languages -Note: A worker is a thread. +The file processor is recognized using the extension. .lua or .luac for lua and +.py for python. Start example: + + $> ./spoa -d -f ps_lua.lua + + $> ./spoa -d -f ps_pyhton.py - Configure a SPOE to use the service ---------------------------------------- + Configure +------------- -All information about SPOE configuration can be found in "doc/SPOE.txt". Here is -the configuration template to use for your SPOE: +Sample configuration are join to this server: - [ip-reputation] - - spoe-agent iprep-agent - messages check-client-ip - - option var-prefix iprep - - timeout hello 100ms - timeout idle 30s - timeout processing 15ms - - use-backend iprep-backend - - spoe-message check-client-ip - args src - event on-client-session + spoa-server.conf : The HAProxy configuration file using SPOE server + spoa-server.spoe.conf : The SPOP description file used by HAProxy + ps_lua.lua : Processing Lua example + ps_python.py : Processing Python example -The engine is in the scope "ip-reputation". So to enable it, you must set the -following line in a frontend/listener section: + Considerations +------------------ - frontend my-front - ... - filter spoe engine ip-reputation config /path/spoe-ip-reputation.conf - .... +This server is a beta version. It works fine, but some improvement will be +welcome: -where "/path/spoe-ip-reputation.conf" is the path to your SPOE configuration -file. The engine name is important here, it must be the same than the one used -in the SPOE configuration file. +Main process: -IMPORTANT NOTE: - Because we want to send a message on the "on-client-session" event, this - SPOE must be attached to a proxy with the frontend capability. If it is - declared in a backend section, it will have no effet. + * Improve log management: Today the log are sent on stdout. + * Improve process management: The dead process are ignored. + * Implement systemd integration. + * Implement threads: It would be fine to implement thread working. Shared + memory is welcome for managing database connection pool and something like + that. + * Add PHP support and some other languages. +Python: -Because, in SPOE configuration file, we declare to use the backend -"iprep-backend" to communicate with the service, you must define it in HAProxy -configuration. For example: + * Improve repporting: Catch python error message and repport it in the right + place. Today the error are dumped on stdout. How using syslog for logging + stack traces ? - backend iprep-backend - mode tcp - timeout server 1m - server iprep-srv 127.0.0.1:12345 check maxconn 5 - - -In reply to the "check-client-ip" message, this service will set the variable -"ip_score" for the session, an integer between 0 and 100. If unchanged, the -variable prefix is "iprep". So the full variable name will be -"sess.iprep.ip_score". - -You can use it in ACLs to experiment the SPOE feature. For example: - - tcp-request content reject if { var(sess.iprep.ip_score) -m int lt 20 } - -With this rule, all IP address with a score lower than 20 will be rejected -(Remember, this score is random). +Maybe some other things... diff --git a/contrib/spoa_server/print_r.lua b/contrib/spoa_server/print_r.lua new file mode 100644 index 0000000000..2fa57e7867 --- /dev/null +++ b/contrib/spoa_server/print_r.lua @@ -0,0 +1,68 @@ +function color(index, str) + return "\x1b[" .. index .. "m" .. str .. "\x1b[00m" +end + +function nocolor(index, str) + return str +end + +function sp(count) + local spaces = "" + while count > 0 do + spaces = spaces .. " " + count = count - 1 + end + return spaces +end + +function print_rr(p, indent, c, wr) + local i = 0 + local nl = "" + + if type(p) == "table" then + wr(c("33", "(table)") .. " " .. c("34", tostring(p)) .. " [") + + mt = getmetatable(p) + if mt ~= nil then + wr("\n" .. sp(indent+1) .. c("31", "METATABLE") .. ": ") + print_rr(mt, indent+1, c, wr) + end + + for k,v in pairs(p) do + if i > 0 then + nl = "\n" + else + wr("\n") + end + wr(nl .. sp(indent+1)) + if type(k) == "number" then + wr(c("32", tostring(k))) + else + wr("\"" .. c("32", tostring(k)) .. "\"") + end + wr(": ") + print_rr(v, indent+1, c, wr) + i = i + 1 + end + if i == 0 then + wr(" " .. c("35", "/* empty */") .. " ]") + else + wr("\n" .. sp(indent) .. "]") + end + elseif type(p) == "string" then + wr(c("33", "(string)") .. " \"" .. c("34", p) .. "\"") + else + wr(c("33", "(" .. type(p) .. ")") .. " " .. c("34", tostring(p))) + end +end + +function print_r(p, col, wr) + if col == nil then col = true end + if wr == nil then wr = function(msg) io.stdout:write(msg) end end + if col == true then + print_rr(p, 0, color, wr) + else + print_rr(p, 0, nocolor, wr) + end + wr("\n") +end diff --git a/contrib/spoa_server/ps_lua.lua b/contrib/spoa_server/ps_lua.lua new file mode 100644 index 0000000000..2662045996 --- /dev/null +++ b/contrib/spoa_server/ps_lua.lua @@ -0,0 +1,17 @@ +require("print_r") + +print_r("Load lua message processors") + +spoa.register_message("check-client-ip", function(args) + print_r(args) + spoa.set_var_null("null", spoa.scope.txn) + spoa.set_var_boolean("boolean", spoa.scope.txn, true) + spoa.set_var_int32("int32", spoa.scope.txn, 1234) + spoa.set_var_uint32("uint32", spoa.scope.txn, 1234) + spoa.set_var_int64("int64", spoa.scope.txn, 1234) + spoa.set_var_uint64("uint64", spoa.scope.txn, 1234) + spoa.set_var_ipv4("ipv4", spoa.scope.txn, "127.0.0.1") + spoa.set_var_ipv6("ipv6", spoa.scope.txn, "1::f") + spoa.set_var_str("str", spoa.scope.txn, "1::f") + spoa.set_var_bin("bin", spoa.scope.txn, "1::f") +end) diff --git a/contrib/spoa_server/ps_python.py b/contrib/spoa_server/ps_python.py new file mode 100644 index 0000000000..108eb48a67 --- /dev/null +++ b/contrib/spoa_server/ps_python.py @@ -0,0 +1,20 @@ +from pprint import pprint +import spoa +import ipaddress + +def check_client_ip(args): + pprint(args) + spoa.set_var_null("null", spoa.scope_txn) + spoa.set_var_boolean("boolean", spoa.scope_txn, True) + spoa.set_var_int32("int32", spoa.scope_txn, 1234) + spoa.set_var_uint32("uint32", spoa.scope_txn, 1234) + spoa.set_var_int64("int64", spoa.scope_txn, 1234) + spoa.set_var_uint64("uint64", spoa.scope_txn, 1234) + spoa.set_var_ipv4("ipv4", spoa.scope_txn, ipaddress.IPv4Address(u"127.0.0.1")) + spoa.set_var_ipv6("ipv6", spoa.scope_txn, ipaddress.IPv6Address(u"1::f")) + spoa.set_var_str("str", spoa.scope_txn, "1::f") + spoa.set_var_bin("bin", spoa.scope_txn, "1:\x01:\x02f\x00\x00") + return + + +spoa.register_message("check-client-ip", check_client_ip) diff --git a/contrib/spoa_server/spoa-server.conf b/contrib/spoa_server/spoa-server.conf new file mode 100644 index 0000000000..13bd126ae0 --- /dev/null +++ b/contrib/spoa_server/spoa-server.conf @@ -0,0 +1,33 @@ +global + debug + +defaults + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 5000 + timeout server 5000 + +listen test + mode http + bind :10001 + filter spoe engine spoa-server config spoa-server.spoe.conf + http-request set-var(req.a) var(txn.iprep.null),debug + http-request set-var(req.a) var(txn.iprep.boolean),debug + http-request set-var(req.a) var(txn.iprep.int32),debug + http-request set-var(req.a) var(txn.iprep.uint32),debug + http-request set-var(req.a) var(txn.iprep.int64),debug + http-request set-var(req.a) var(txn.iprep.uint64),debug + http-request set-var(req.a) var(txn.iprep.ipv4),debug + http-request set-var(req.a) var(txn.iprep.ipv6),debug + http-request set-var(req.a) var(txn.iprep.str),debug + http-request set-var(req.a) var(txn.iprep.bin),debug + http-request redirect location /%[var(sess.iprep.ip_score)] + +backend spoe-server + mode tcp + balance roundrobin + timeout connect 5s + timeout server 3m + server spoe-server 127.0.0.1:12345 diff --git a/contrib/spoa_server/spoa-server.spoe.conf b/contrib/spoa_server/spoa-server.spoe.conf new file mode 100644 index 0000000000..dab4e5a9ef --- /dev/null +++ b/contrib/spoa_server/spoa-server.spoe.conf @@ -0,0 +1,13 @@ +[spoa-server] + +spoe-agent spoa-server + messages check-client-ip + option var-prefix iprep + timeout hello 100ms + timeout idle 30s + timeout processing 15ms + use-backend spoe-server + +spoe-message check-client-ip + args always_true int(1234) src ipv6(::55) req.fhdr(host) + event on-frontend-http-request