haproxy/examples/lua/mailers.lua
Aurelien DARRAGON a92694800c EXAMPLES: add lua mailers script to replace tcpcheck mailers
Legacy mailers implemented using tcpchecks may now be replaced using a
pure lua implementation.

Simply loading the file "mailers.lua" using lua-load directive is enough
to disable the legacy mailer implementation and make the lua script
subscribe to server events in order to generate messages and send email
alerts to configured mailservers according to the mailers configuration
specified by the user in the config file.

lua-load-per-thread directive is supported, the script will automatically
force itself on a single thread to prevent multiple mails from being sent
for the same event.

The email sending from lua in itself is handled with smtp_send_email()
function from Thierry FOURNIER.
(see: https://www.arpalert.org/how-to-send-email.html)

The function was slightly adapted to send HELO instead of EHLO when
beginning the SMTP handshake to make it more compatible with basic SMTP
stacks and to comply with the legacy SMTP handshake performed in
mailers.c.

Authentication is not yet handled by smtp_send_email(), but it may be
further improved to do so.

Note that existing lua libraries may also support sending emails (even
with authentication support maybe?), I did not do much researchs about
this, so I'm not aware of existing solutions at the time of writing this
script.

The only restriction is that when using an external library, the library
function calls must not be blocking, since haproxy relies on lua
executions to be yieldable and rescheduled.

As long as the script complies with this limitation, it may be customized
or improved in many ways, including templating, making calls to APIs
services.. (ie: triggering automation flows, sending SMS alerts...
you name it)

The purpose of this script is to provide a basic working replacement for
legacy mailers implementation based on tcpchecks, which is planned for
removal in the future. tcpcheck based mailers is kind of a hack which is
not suitable as a long term solution.
(hard to maintain and not customizable)

Note: Email content for email alerts sent using this script might slightly
differ from the legacy implementation (some optional info might be missing
such as server's check dynamic description, or included statistics such as
currently active servers may appear out of sync) due the email generation
now being performed asynchronously. However the output format complies with
the original one and essential informations are consistently reported.

Current known script limitation: if multiple events are generated
simultaneously it is possible that emails could be received out of
order since emails are sent asynchronously using smtp_send_email()
and there is no sending queue. Relying on the email "date" should
help to know which email was generated first..
2023-05-05 16:28:32 +02:00

392 lines
11 KiB
Lua

-- haproxy mailers implementation in lua
--
-- Provides a pure lua alternative to tcpcheck mailers.
--
-- To be loaded using "lua-load" from haproxy configuration to handle
-- email-alerts directly from lua and disable legacy tcpcheck implementation.
local SYSLOG_LEVEL = {
["EMERG"] = 0,
["ALERT"] = 1,
["CRIT"] = 2,
["ERROR"] = 3,
["WARN"] = 4,
["NOTICE"] = 5,
["INFO"] = 6,
["DEBUG"] = 7
}
-- smtp : send SMTP message
--
-- Copyright 2018 Thierry Fournier
--
-- This function is compliant with HAProxy cosockets
-- EHLO was replaced with HELO for better compatibility with
-- basic mail server implementations
--
function smtp_send_email(server, port, domain, from, to, data)
local ret
local reason
local tcp = core.tcp()
local smtp_wait_code = function(tcp, code)
local ret
-- Read headers until we reac a 2.. code.
while true do
-- read line
ret = tcp:receive("*l")
if ret == nil then
return false, "Connection unexpectly closed"
end
-- expected code
if string.match(ret, code) ~= nil then
return true, nil
end
-- other code
if string.match(ret, '^%d%d%d ') ~= nil then
return false, ret
end
-- other informational message, wait.
end
end
if tcp:connect(server, port) == nil then
return false, "Can't connect to \""..server..":"..port.."\""
end
ret, reason = smtp_wait_code(tcp, '^220 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send("HELO " .. domain .. "\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^250 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send("MAIL FROM: <" .. from .. ">\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^250 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send("RCPT TO: <" .. to .. ">\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^250 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send("DATA\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^354 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send(data .. "\r\n.\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^250 ')
if ret == false then
tcp:close()
return false, reason
end
if tcp:send("QUIT\r\n") == nil then
tcp:close()
return false, "Connection unexpectly closed"
end
ret, reason = smtp_wait_code(tcp, '^221 ')
if ret == false then
tcp:close()
return false, reason
end
tcp:close()
return true, nil
end
local function send_email_alert(srv, level, message, when)
local mailers = srv:get_proxy():get_mailers()
if mailers == nil then
return -- nothing to do
end
if level > mailers.log_level then
return
end
-- email sending is performed asynchronously
core.register_task(function(mailers, msg, when)
for name, mailsrv in pairs(mailers.mailservers) do
local mailsrv_ip, mailsrv_port = string.match(mailsrv, "([^:]+):([^:]+)")
local date = os.date("%a, %d %b %Y %T %z (%Z)", when)
local c = core.concat()
c:add(string.format("From: %s\r\n", mailers.smtp_from))
c:add(string.format("To: %s\r\n", mailers.smtp_to))
c:add(string.format("Date: %s\r\n", date))
c:add(string.format("Subject: [HAProxy Alert] %s\r\n", msg))
c:add("\r\n")
c:add(string.format("%s\r\n", msg))
local ret, reason = smtp_send_email(mailsrv_ip, mailsrv_port,
mailers.smtp_hostname, mailers.smtp_from,
mailers.smtp_to, c:dump())
if ret == false then
core.Warning("Can't send email: " .. reason)
end
end
end, mailers, message, when)
end
local function srv_get_check_details(check)
local c = core.concat()
c:add(", ")
c:add(string.format("reason: %s", check.reason.desc))
if check.reason.code ~= nil
then
c:add(string.format(", code: %d", check.reason.code))
end
if check.duration >= 0
then
c:add(string.format(", check duration: %dms", check.duration))
end
return c:dump()
end
local function srv_get_status_details(srv, requeued)
local c = core.concat()
c:add(string.format("%d active and %d backup servers left.",
srv:get_proxy():get_srv_act(),
srv:get_proxy():get_srv_bck()))
c:add(" ")
c:add(string.format("%d sessions active, %d requeued, %d remaining in queue",
srv:get_cur_sess(),
requeued,
srv:get_pend_conn()))
return c:dump()
end
local function srv_state_handler(event, data, when)
local server = data.reference
local state = data.state
local c = core.concat()
local log_level = SYSLOG_LEVEL["ALERT"]
local message
if server == nil then
-- server already removed, can't do much
return
end
if state.admin then
-- don't report if is related to an administrative change and not
-- directly due to an operational change
return
end
-- we don't send an alert if the server was previously stopping
if state.old_state == "STOPPING" or server:is_draining() then
log_level = SYSLOG_LEVEL["NOTICE"]
end
-- prepare the message
c:add(string.format("Server %s/%s is %s",
server:get_proxy():get_name(),
server:get_name(),
state.new_state == "RUNNING" and "UP" or "DOWN"))
if server:tracking()
then
-- server is tracking another server, it means that the operational
-- state change is inherited
c:add(string.format(" via %s/%s",
server:tracking():get_proxy():get_name(),
server:tracking():get_name()))
end
if state.check ~= nil
then
c:add(srv_get_check_details(state.check))
else
c:add(state.cause)
end
c:add(". ")
c:add(srv_get_status_details(server, state.requeued))
send_email_alert(server, log_level, c:dump(), when)
end
local function srv_admin_handler(event, data, when)
local server = data.reference
local admin = data.admin
local c = core.concat()
if server == nil then
-- server already removed, can't do much
return
end
-- only send an email when server is entering drain state and not under maint
if not (not admin.old_admin["DRAIN"] and
admin.new_admin["DRAIN"] and
not admin.new_admin["MAINT"]) then
return
end
-- prepare the message
c:add(string.format("Server %s/%s enters drain state",
server:get_proxy():get_name(),
server:get_name()))
if server:tracking() and admin.new_admin["IDRAIN"]
then
-- server is tracking another server and IDRAIN is set, it means
-- that the admin state change is inherited
c:add(string.format(" via %s/%s",
server:tracking():get_proxy():get_name(),
server:tracking():get_name()))
end
c:add(". ")
c:add(srv_get_status_details(server, admin.requeued))
send_email_alert(server, SYSLOG_LEVEL["NOTICE"], c:dump(), when)
end
local function srv_check_handler(event, data, when)
local server = data.reference
local check = data.check
local c = core.concat()
if server == nil then
-- server already removed, can't do much
return
end
-- we will always send an email, prepare the message
c:add(string.format("%s check for %sserver %s/%s ",
check.agent and "Agent" or "Health",
server:is_backup() and "backup " or "",
server:get_proxy():get_name(),
server:get_name()))
if check.result == "CONDPASS" then
c:add("conditionally succeeded")
elseif check.result == "PASSED" then
c:add("succeeded")
else
c:add("failed")
end
c:add(srv_get_check_details(check))
c:add(", status: ")
if check.health.cur >= check.health.rise then
-- good
c:add(string.format("%d/%d %s",
check.health.cur - check.health.rise + 1,
check.health.fall,
server:get_weight() and "UP" or "DRAIN"))
else
-- bad
c:add(string.format("%d/%d DOWN",
check.health.cur,
check.health.rise))
end
send_email_alert(server, SYSLOG_LEVEL["INFO"], c:dump(), when)
end
-- single function for multiple event types since all events come
-- from the same subscription to reduce memory footprint
local function srv_event_dispatch(event, data, mgmt, when)
if event == "SERVER_STATE" then srv_state_handler(event, data, when) end
if event == "SERVER_ADMIN" then srv_admin_handler(event, data, when) end
if event == "SERVER_CHECK" then srv_check_handler(event, data, when) end
end
local function mailers_track_server_events(srv)
local mailer_conf = srv:get_proxy():get_mailers()
-- don't track server events if the parent proxy did not enable email alerts
if mailer_conf == nil
then return
end
-- email alerts are enabled, track server state and admin changes
local subscriptions = {"SERVER_STATE", "SERVER_ADMIN"}
if mailer_conf.track_server_health
then
-- track server check events as well (this event source is expensive)
table.insert(subscriptions, "SERVER_CHECK")
end
-- perform the event subscription from the server
srv:event_sub(subscriptions, srv_event_dispatch)
end
local function srv_event_add(event, data)
-- do nothing if the server was already removed
if data.reference == nil
then return
end
-- server still exists, check if it can be tracked for email alerts
mailers_track_server_events(data.reference)
end
-- disable legacy email-alerts since email-alerts will be sent from lua directly
core.disable_legacy_mailers()
-- event subscription are purposely performed in a task to prevent email
-- alerts from being generated too early (when process is starting up)
core.register_task(function()
-- do nothing if not on primary thread
-- this prevents emails from being sent multiple times when
-- lua-load-per-thread is used to load the script since the task
-- will be started on each haproxy thread
if core.thread > 1 then core.done() end
-- subscribe to SERVER_ADD to be notified when new servers are added
core.event_sub({"SERVER_ADD"}, srv_event_add)
-- loop through existing backends to detect existing servers
for backend_name, backend in pairs(core.backends) do
for srv_name, srv in pairs(backend.servers) do
mailers_track_server_events(srv)
end
end
end)