mirror of
http://git.haproxy.org/git/haproxy.git/
synced 2025-01-02 10:12:03 +00:00
a92694800c
Legacy mailers implemented using tcpchecks may now be replaced using a pure lua implementation. Simply loading the file "mailers.lua" using lua-load directive is enough to disable the legacy mailer implementation and make the lua script subscribe to server events in order to generate messages and send email alerts to configured mailservers according to the mailers configuration specified by the user in the config file. lua-load-per-thread directive is supported, the script will automatically force itself on a single thread to prevent multiple mails from being sent for the same event. The email sending from lua in itself is handled with smtp_send_email() function from Thierry FOURNIER. (see: https://www.arpalert.org/how-to-send-email.html) The function was slightly adapted to send HELO instead of EHLO when beginning the SMTP handshake to make it more compatible with basic SMTP stacks and to comply with the legacy SMTP handshake performed in mailers.c. Authentication is not yet handled by smtp_send_email(), but it may be further improved to do so. Note that existing lua libraries may also support sending emails (even with authentication support maybe?), I did not do much researchs about this, so I'm not aware of existing solutions at the time of writing this script. The only restriction is that when using an external library, the library function calls must not be blocking, since haproxy relies on lua executions to be yieldable and rescheduled. As long as the script complies with this limitation, it may be customized or improved in many ways, including templating, making calls to APIs services.. (ie: triggering automation flows, sending SMS alerts... you name it) The purpose of this script is to provide a basic working replacement for legacy mailers implementation based on tcpchecks, which is planned for removal in the future. tcpcheck based mailers is kind of a hack which is not suitable as a long term solution. (hard to maintain and not customizable) Note: Email content for email alerts sent using this script might slightly differ from the legacy implementation (some optional info might be missing such as server's check dynamic description, or included statistics such as currently active servers may appear out of sync) due the email generation now being performed asynchronously. However the output format complies with the original one and essential informations are consistently reported. Current known script limitation: if multiple events are generated simultaneously it is possible that emails could be received out of order since emails are sent asynchronously using smtp_send_email() and there is no sending queue. Relying on the email "date" should help to know which email was generated first..
392 lines
11 KiB
Lua
392 lines
11 KiB
Lua
-- haproxy mailers implementation in lua
|
|
--
|
|
-- Provides a pure lua alternative to tcpcheck mailers.
|
|
--
|
|
-- To be loaded using "lua-load" from haproxy configuration to handle
|
|
-- email-alerts directly from lua and disable legacy tcpcheck implementation.
|
|
|
|
local SYSLOG_LEVEL = {
|
|
["EMERG"] = 0,
|
|
["ALERT"] = 1,
|
|
["CRIT"] = 2,
|
|
["ERROR"] = 3,
|
|
["WARN"] = 4,
|
|
["NOTICE"] = 5,
|
|
["INFO"] = 6,
|
|
["DEBUG"] = 7
|
|
}
|
|
|
|
-- smtp : send SMTP message
|
|
--
|
|
-- Copyright 2018 Thierry Fournier
|
|
--
|
|
-- This function is compliant with HAProxy cosockets
|
|
-- EHLO was replaced with HELO for better compatibility with
|
|
-- basic mail server implementations
|
|
--
|
|
function smtp_send_email(server, port, domain, from, to, data)
|
|
local ret
|
|
local reason
|
|
local tcp = core.tcp()
|
|
local smtp_wait_code = function(tcp, code)
|
|
local ret
|
|
-- Read headers until we reac a 2.. code.
|
|
while true do
|
|
-- read line
|
|
ret = tcp:receive("*l")
|
|
if ret == nil then
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
-- expected code
|
|
if string.match(ret, code) ~= nil then
|
|
return true, nil
|
|
end
|
|
-- other code
|
|
if string.match(ret, '^%d%d%d ') ~= nil then
|
|
return false, ret
|
|
end
|
|
-- other informational message, wait.
|
|
end
|
|
end
|
|
|
|
if tcp:connect(server, port) == nil then
|
|
return false, "Can't connect to \""..server..":"..port.."\""
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^220 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send("HELO " .. domain .. "\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^250 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send("MAIL FROM: <" .. from .. ">\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^250 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send("RCPT TO: <" .. to .. ">\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^250 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send("DATA\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^354 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send(data .. "\r\n.\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^250 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
if tcp:send("QUIT\r\n") == nil then
|
|
tcp:close()
|
|
return false, "Connection unexpectly closed"
|
|
end
|
|
|
|
ret, reason = smtp_wait_code(tcp, '^221 ')
|
|
if ret == false then
|
|
tcp:close()
|
|
return false, reason
|
|
end
|
|
|
|
tcp:close()
|
|
return true, nil
|
|
end
|
|
|
|
local function send_email_alert(srv, level, message, when)
|
|
local mailers = srv:get_proxy():get_mailers()
|
|
|
|
if mailers == nil then
|
|
return -- nothing to do
|
|
end
|
|
|
|
if level > mailers.log_level then
|
|
return
|
|
end
|
|
|
|
-- email sending is performed asynchronously
|
|
core.register_task(function(mailers, msg, when)
|
|
for name, mailsrv in pairs(mailers.mailservers) do
|
|
local mailsrv_ip, mailsrv_port = string.match(mailsrv, "([^:]+):([^:]+)")
|
|
local date = os.date("%a, %d %b %Y %T %z (%Z)", when)
|
|
local c = core.concat()
|
|
|
|
c:add(string.format("From: %s\r\n", mailers.smtp_from))
|
|
c:add(string.format("To: %s\r\n", mailers.smtp_to))
|
|
c:add(string.format("Date: %s\r\n", date))
|
|
c:add(string.format("Subject: [HAProxy Alert] %s\r\n", msg))
|
|
c:add("\r\n")
|
|
c:add(string.format("%s\r\n", msg))
|
|
|
|
local ret, reason = smtp_send_email(mailsrv_ip, mailsrv_port,
|
|
mailers.smtp_hostname, mailers.smtp_from,
|
|
mailers.smtp_to, c:dump())
|
|
if ret == false then
|
|
core.Warning("Can't send email: " .. reason)
|
|
end
|
|
end
|
|
end, mailers, message, when)
|
|
end
|
|
|
|
local function srv_get_check_details(check)
|
|
local c = core.concat()
|
|
|
|
c:add(", ")
|
|
c:add(string.format("reason: %s", check.reason.desc))
|
|
if check.reason.code ~= nil
|
|
then
|
|
c:add(string.format(", code: %d", check.reason.code))
|
|
end
|
|
if check.duration >= 0
|
|
then
|
|
c:add(string.format(", check duration: %dms", check.duration))
|
|
end
|
|
|
|
return c:dump()
|
|
end
|
|
|
|
local function srv_get_status_details(srv, requeued)
|
|
local c = core.concat()
|
|
|
|
c:add(string.format("%d active and %d backup servers left.",
|
|
srv:get_proxy():get_srv_act(),
|
|
srv:get_proxy():get_srv_bck()))
|
|
c:add(" ")
|
|
c:add(string.format("%d sessions active, %d requeued, %d remaining in queue",
|
|
srv:get_cur_sess(),
|
|
requeued,
|
|
srv:get_pend_conn()))
|
|
return c:dump()
|
|
end
|
|
|
|
local function srv_state_handler(event, data, when)
|
|
local server = data.reference
|
|
local state = data.state
|
|
local c = core.concat()
|
|
local log_level = SYSLOG_LEVEL["ALERT"]
|
|
local message
|
|
|
|
if server == nil then
|
|
-- server already removed, can't do much
|
|
return
|
|
end
|
|
|
|
if state.admin then
|
|
-- don't report if is related to an administrative change and not
|
|
-- directly due to an operational change
|
|
return
|
|
end
|
|
|
|
-- we don't send an alert if the server was previously stopping
|
|
if state.old_state == "STOPPING" or server:is_draining() then
|
|
log_level = SYSLOG_LEVEL["NOTICE"]
|
|
end
|
|
|
|
-- prepare the message
|
|
c:add(string.format("Server %s/%s is %s",
|
|
server:get_proxy():get_name(),
|
|
server:get_name(),
|
|
state.new_state == "RUNNING" and "UP" or "DOWN"))
|
|
|
|
if server:tracking()
|
|
then
|
|
-- server is tracking another server, it means that the operational
|
|
-- state change is inherited
|
|
c:add(string.format(" via %s/%s",
|
|
server:tracking():get_proxy():get_name(),
|
|
server:tracking():get_name()))
|
|
end
|
|
|
|
if state.check ~= nil
|
|
then
|
|
c:add(srv_get_check_details(state.check))
|
|
else
|
|
c:add(state.cause)
|
|
end
|
|
|
|
c:add(". ")
|
|
c:add(srv_get_status_details(server, state.requeued))
|
|
send_email_alert(server, log_level, c:dump(), when)
|
|
end
|
|
|
|
local function srv_admin_handler(event, data, when)
|
|
local server = data.reference
|
|
local admin = data.admin
|
|
local c = core.concat()
|
|
|
|
if server == nil then
|
|
-- server already removed, can't do much
|
|
return
|
|
end
|
|
|
|
-- only send an email when server is entering drain state and not under maint
|
|
if not (not admin.old_admin["DRAIN"] and
|
|
admin.new_admin["DRAIN"] and
|
|
not admin.new_admin["MAINT"]) then
|
|
return
|
|
end
|
|
|
|
-- prepare the message
|
|
c:add(string.format("Server %s/%s enters drain state",
|
|
server:get_proxy():get_name(),
|
|
server:get_name()))
|
|
|
|
if server:tracking() and admin.new_admin["IDRAIN"]
|
|
then
|
|
-- server is tracking another server and IDRAIN is set, it means
|
|
-- that the admin state change is inherited
|
|
c:add(string.format(" via %s/%s",
|
|
server:tracking():get_proxy():get_name(),
|
|
server:tracking():get_name()))
|
|
end
|
|
|
|
c:add(". ")
|
|
c:add(srv_get_status_details(server, admin.requeued))
|
|
send_email_alert(server, SYSLOG_LEVEL["NOTICE"], c:dump(), when)
|
|
end
|
|
|
|
local function srv_check_handler(event, data, when)
|
|
local server = data.reference
|
|
local check = data.check
|
|
local c = core.concat()
|
|
|
|
if server == nil then
|
|
-- server already removed, can't do much
|
|
return
|
|
end
|
|
|
|
-- we will always send an email, prepare the message
|
|
c:add(string.format("%s check for %sserver %s/%s ",
|
|
check.agent and "Agent" or "Health",
|
|
server:is_backup() and "backup " or "",
|
|
server:get_proxy():get_name(),
|
|
server:get_name()))
|
|
|
|
if check.result == "CONDPASS" then
|
|
c:add("conditionally succeeded")
|
|
elseif check.result == "PASSED" then
|
|
c:add("succeeded")
|
|
else
|
|
c:add("failed")
|
|
end
|
|
|
|
c:add(srv_get_check_details(check))
|
|
|
|
c:add(", status: ")
|
|
if check.health.cur >= check.health.rise then
|
|
-- good
|
|
c:add(string.format("%d/%d %s",
|
|
check.health.cur - check.health.rise + 1,
|
|
check.health.fall,
|
|
server:get_weight() and "UP" or "DRAIN"))
|
|
else
|
|
-- bad
|
|
c:add(string.format("%d/%d DOWN",
|
|
check.health.cur,
|
|
check.health.rise))
|
|
end
|
|
|
|
send_email_alert(server, SYSLOG_LEVEL["INFO"], c:dump(), when)
|
|
end
|
|
|
|
-- single function for multiple event types since all events come
|
|
-- from the same subscription to reduce memory footprint
|
|
local function srv_event_dispatch(event, data, mgmt, when)
|
|
if event == "SERVER_STATE" then srv_state_handler(event, data, when) end
|
|
if event == "SERVER_ADMIN" then srv_admin_handler(event, data, when) end
|
|
if event == "SERVER_CHECK" then srv_check_handler(event, data, when) end
|
|
end
|
|
|
|
local function mailers_track_server_events(srv)
|
|
local mailer_conf = srv:get_proxy():get_mailers()
|
|
|
|
-- don't track server events if the parent proxy did not enable email alerts
|
|
if mailer_conf == nil
|
|
then return
|
|
end
|
|
|
|
-- email alerts are enabled, track server state and admin changes
|
|
local subscriptions = {"SERVER_STATE", "SERVER_ADMIN"}
|
|
|
|
if mailer_conf.track_server_health
|
|
then
|
|
-- track server check events as well (this event source is expensive)
|
|
table.insert(subscriptions, "SERVER_CHECK")
|
|
end
|
|
|
|
-- perform the event subscription from the server
|
|
srv:event_sub(subscriptions, srv_event_dispatch)
|
|
end
|
|
|
|
local function srv_event_add(event, data)
|
|
-- do nothing if the server was already removed
|
|
if data.reference == nil
|
|
then return
|
|
end
|
|
|
|
-- server still exists, check if it can be tracked for email alerts
|
|
mailers_track_server_events(data.reference)
|
|
end
|
|
|
|
|
|
-- disable legacy email-alerts since email-alerts will be sent from lua directly
|
|
core.disable_legacy_mailers()
|
|
|
|
-- event subscription are purposely performed in a task to prevent email
|
|
-- alerts from being generated too early (when process is starting up)
|
|
core.register_task(function()
|
|
|
|
-- do nothing if not on primary thread
|
|
-- this prevents emails from being sent multiple times when
|
|
-- lua-load-per-thread is used to load the script since the task
|
|
-- will be started on each haproxy thread
|
|
if core.thread > 1 then core.done() end
|
|
|
|
-- subscribe to SERVER_ADD to be notified when new servers are added
|
|
core.event_sub({"SERVER_ADD"}, srv_event_add)
|
|
|
|
-- loop through existing backends to detect existing servers
|
|
for backend_name, backend in pairs(core.backends) do
|
|
for srv_name, srv in pairs(backend.servers) do
|
|
mailers_track_server_events(srv)
|
|
end
|
|
end
|
|
|
|
end)
|