From 958c6bc0bece4fafb1dd78e4532b1d0f31bc1c29 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Sun, 23 Oct 2022 16:37:53 +0200 Subject: [PATCH] updated haproxy state --- states/haproxy/config.sls | 11 - states/haproxy/defaults.yaml | 39 +- states/haproxy/install.sls | 28 +- states/haproxy/map.jinja | 2 - states/haproxy/scripts/collector.lua | 36 + states/haproxy/scripts/http.lua | 830 +++++++++++++++++++ states/haproxy/scripts/json.lua | 388 +++++++++ states/haproxy/scripts/weight_by_latency.lua | 43 + states/haproxy/templates/haproxy.cfg.j2 | 111 ++- 9 files changed, 1422 insertions(+), 66 deletions(-) create mode 100644 states/haproxy/scripts/collector.lua create mode 100644 states/haproxy/scripts/http.lua create mode 100644 states/haproxy/scripts/json.lua create mode 100644 states/haproxy/scripts/weight_by_latency.lua diff --git a/states/haproxy/config.sls b/states/haproxy/config.sls index d038f37..5e0b7d2 100644 --- a/states/haproxy/config.sls +++ b/states/haproxy/config.sls @@ -10,14 +10,3 @@ haproxy-config: - template: jinja - watch_in: - service: haproxy-service - -haproxy-config-access: - file.managed: - - name: {{ haproxy.config.dir }}/{{ haproxy.config.accessfile }} - - source: salt://haproxy/templates/access.j2 - - user: {{ haproxy.config.user }} - - group: {{ haproxy.config.group }} - - mode: "0600" - - template: jinja - - watch_in: - - service: haproxy-service diff --git a/states/haproxy/defaults.yaml b/states/haproxy/defaults.yaml index 6d0096b..59ba14e 100644 --- a/states/haproxy/defaults.yaml +++ b/states/haproxy/defaults.yaml @@ -4,29 +4,42 @@ haproxy: packages: - haproxy scripts: - - hello_world.lua + - name: scripts/http.lua + lib: true + - name: scripts/json.lua + lib: true + - name: scripts/collector.lua + lib: false + - name: scripts/weight_by_latency.lua + lib: false + maps: + - access config: + servername: "PaulBSD WebServer 1.0" + http2: true defaults: - #log global: - #log 127.0.0.1 local0: - log stdout format raw daemon info: - mode http: - option httplog: - option forwardfor: - retries 2: - timeout client 30m: - timeout connect 4s: - timeout server 30m: - timeout check 5s: + #log: global + #log: 127.0.0.1 local0 + log: stdout format raw daemon info + retries: 2 + timeout client: 30m + timeout connect: 4s + timeout server: 30m + timeout check: 5s dir: /etc/haproxy configfile: haproxy.cfg - accessfile: access user: haproxy group: haproxy http_port: 80 https_port: 443 admin: false + api: + enable: true + filesocket: /var/run/hap-lb.sock + tcpsocket: ipv4@127.0.0.1:9990 acme_dir: /etc/acme + acme_fullchains_dir: /etc/acme/fullchains + acme_dh_dir: /etc/acme/dh ssl_ciphers: - "ECDH+AESGCM" - "DH+AESGCM" diff --git a/states/haproxy/install.sls b/states/haproxy/install.sls index 97b8ac0..757f258 100644 --- a/states/haproxy/install.sls +++ b/states/haproxy/install.sls @@ -4,6 +4,13 @@ haproxy-pkg: pkg.installed: - pkgs: {{ haproxy.packages }} +haproxy-maps-dir: + file.directory: + - name: {{ haproxy.config.dir }}/maps + - user: {{ haproxy.config.user }} + - group: {{ haproxy.config.group }} + - mode: "0700" + haproxy-script-dir: file.directory: - name: {{ haproxy.config.dir }}/scripts @@ -11,14 +18,27 @@ haproxy-script-dir: - group: {{ haproxy.config.group }} - mode: "0700" -{% for filename in haproxy.scripts %} -haprox-script-{{ filename }}: +{% for file in haproxy.scripts %} +haproxy-script-{{ file.name }}: file.managed: - - name: {{ haproxy.config.dir }}/scripts/{{ filename }} - - source: salt://haproxy/scripts/{{ filename }} + - name: {{ haproxy.config.dir }}/{{ file.name }} + - source: salt://haproxy/{{ file.name }} - user: {{ haproxy.config.user }} - group: {{ haproxy.config.group }} - mode: "0700" - watch_in: - service: haproxy-service +{% endfor %} + +{% for filename in haproxy.maps %} +haproxy-maps-{{ filename }}: + file.managed: + - name: {{ haproxy.config.dir }}/maps/{{ filename }} + - source: salt://haproxy/templates/{{ filename }}.j2 + - user: {{ haproxy.config.user }} + - group: {{ haproxy.config.group }} + - mode: "0600" + - template: jinja + - watch_in: + - service: haproxy-service {% endfor %} \ No newline at end of file diff --git a/states/haproxy/map.jinja b/states/haproxy/map.jinja index b34f9d1..6595cc5 100644 --- a/states/haproxy/map.jinja +++ b/states/haproxy/map.jinja @@ -2,7 +2,5 @@ {%- set haproxy = salt['pillar.get']('haproxy', default=defaults.haproxy, merge=True) -%} -{%- set certs = salt['file.find'](haproxy.config.acme_dir, type='f', name='*.full')%} - {%- set users = salt['pillar.get']('htpasswds') -%} {%- set net = salt['pillar.get']('net') -%} \ No newline at end of file diff --git a/states/haproxy/scripts/collector.lua b/states/haproxy/scripts/collector.lua new file mode 100644 index 0000000..80c402a --- /dev/null +++ b/states/haproxy/scripts/collector.lua @@ -0,0 +1,36 @@ +--require = GLOBAL.require +--require("http.request") +local request = require("http") +local json = require("json") + +core.register_action("test", { "http-req" }, function(txn) + local info = {} + local headers = {} + + local reqbody = txn.sf:req_body() + local reqclientip = txn.f:src() + local reqheaders = txn.http:req_get_headers() + local reqmethod = txn.f:method() + local reqpath = txn.f:path() + + info["body"] = reqbody + info["clientip"] = reqclientip + local headers = {} + for k,v in pairs(reqheaders) do + headers[k] = {} + for z,y in pairs(v) do + table.insert(headers[k],y) + end + end + info["headers"] = headers + info["method"] = reqmethod + info["path"] = reqpath + + local infojson = json.encode(info) + --local req = request.new_from_uri("https://ipbl.paulbsd.com") + --local headers, stream = req:go() + --local body = assert(stream:get_body_as_string()) + + txn.Info(txn, infojson) +end +) diff --git a/states/haproxy/scripts/http.lua b/states/haproxy/scripts/http.lua new file mode 100644 index 0000000..947a761 --- /dev/null +++ b/states/haproxy/scripts/http.lua @@ -0,0 +1,830 @@ +-- +-- HTTP 1.1 library for HAProxy Lua modules +-- +-- The library is loosely modeled after Python's Requests Library +-- using the same field names and very similar calling conventions for +-- "HTTP verb" methods (where we use Lua specific named parameter support) +-- +-- In addition to client side, the library also supports server side request +-- parsing, where we utilize HAProxy Lua API for all heavy lifting. +-- +-- +-- Copyright (c) 2017-2020. Adis Nezirović +-- Copyright (c) 2017-2020. HAProxy Technologies, LLC. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +local _author = "Adis Nezirovic " +local _copyright = "Copyright 2017-2020. HAProxy Technologies, LLC." +local _version = "1.0.0" + +local json = require "json" + +-- Utility functions + +-- HTTP headers fetch helper +-- +-- Returns a header value(s) according to strategy (fold by default): +-- - single/string value for "fold", "first" and "last" strategies +-- - table for "all" strategy (for single value, a table with single element) +-- +-- @param hdrs table Headers table as received by http.get and friends +-- @param name string Header name +-- @param strategy string "multiple header values" handling strategy +-- @return header value (string or table) or nil +local function get_header(hdrs, name, strategy) + if hdrs == nil or name == nil then return nil end + + local v = hdrs[name:lower()] + if type(v) ~= "table" and strategy ~= "all" then return v end + + if strategy == nil or strategy == "fold" then + return table.concat(v, ",") + elseif strategy == "first" then + return v[1] + elseif strategy == "last" then + return v[#v] + elseif strategy == "all" then + if type(v) ~= "table" then + return {v} + else + return v + end + end +end + + + +-- HTTP headers iterator helper +-- +-- Returns key/value pairs for all header, making sure that returned values +-- are always of string type (if necessary, it folds multiple headers with +-- the same name) +-- +-- @param hdrs table Headers table as received by http.get and friends +-- @return header name/value iterator (suitable for use in "for" loops) +local function get_headers_folded(hdrs) + if hdrs == nil then + return function() end + end + + local function iter(t, k) + local v + k, v = next(t, k) + + if v ~= nil then + if type(v) ~= "table" then + return k, v + else + return k, table.concat(v, ",") + end + end + end + + return iter, hdrs, nil +end + +-- HTTP headers iterator +-- +-- Returns key/value pairs for all headers, for multiple headers with same name +-- it will return every name/value pair +-- (i.e. you can safely use it to process responses with 'Set-Cookie' header) +-- +-- @param hdrs table Headers table as received by http.get and friends +-- @return header name/value iterator (suitable for use in "for" loops) +local function get_headers_flattened(hdrs) + if hdrs == nil then + return function() end + end + + local k -- top level key (string) + local k_sub = 0 -- sub table key (integer), 0 if item not a table, + -- nil after last sub table iteration + local v_sub -- sub table + + return function () + local v + if k_sub == 0 then + k, v = next(hdrs, k) + if k == nil then return end + else + k_sub, v = next(v_sub, k_sub) + + if k_sub == nil then + k_sub = 0 + k, v = next(hdrs, k) + end + end + + if k == nil then return end + + if type(v) ~= "table" then + return k, v + else + v_sub = v + k_sub = k_sub + 1 + return k, v[k_sub] + end + end +end + + +--- Parse key/value pairs from a string +-- +-- @param s Lua string with (multiple) key/value pairs (separated by 'sep') +-- +-- @return Table with parsed keys and values or nil +local function parse_kv(s, sep) + if s == nil then return nil end + idx = 1 + result = {} + + while idx < s:len() do + i, j = s:find(sep, idx) + + if i == nil then + k, v = string.match(s:sub(idx), "^(.-)=(.*)$") + if k then result[k] = v end + break + end + + k, v = string.match(s:sub(idx, i-1), "^(.-)=(.*)$") + if k then result[k] = v end + idx = j + 1 + end + + if next(result) == nil then + return nil + else + return result + end +end + + +--- Make deep copy of table and it's values +-- +-- Use only for simple tables (it handles nested table values), but not for +-- Lua objects or similar, or very big tables (this uses recursion). +-- +-- @param t Cloned Lua table or nil +-- +-- @return Cloned table or nil +local function copyTable(t) + if type(t) ~= "table" then + return nil + end + + local r = {} + + for k, v in pairs(t) do + if type(v) == "table" then + r[k] = copyTable(v) + else + r[k] = v + end + end + + return r +end + + +--- Namespace object which hosts HTTP verb methods and request/response classes +local M = {} + + +--- HTTP response class +M.response = {} +M.response.__index = M.response + +local _reason = { + [200] = "OK", + [201] = "Created", + [204] = "No Content", + [301] = "Moved Permanently", + [302] = "Found", + [400] = "Bad Request", + [403] = "Forbidden", + [404] = "Not Found", + [405] = "Method Not Allowed", + [408] = "Request Timeout", + [413] = "Payload Too Large", + [429] = "Too many requests", + [500] = "Internal Server Error", + [501] = "Not Implemented", + [502] = "Bad Gateway", + [503] = "Service Unavailable", + [504] = "Gateway Timeout" +} + +--- Creates HTTP response from scratch +-- +-- @param status_code HTTP status code +-- @param reason HTTP status code text (e.g. "OK" for 200 response) +-- @param headers HTTP response headers +-- @param request The HTTP request which triggered the response +-- @param encoding Default encoding for response or conversions +-- +-- @return response object +function M.response.create(t) + local self = setmetatable({}, M.response) + + if not t then + t = {} + end + + self.status_code = t.status_code or nil + self.reason = t.reason or _reason[self.status_code] or "" + self.headers = copyTable(t.headers) or {} + self.content = t.content or "" + self.request = t.request or nil + self.encoding = t.encoding or "utf-8" + + return self +end + +function M.response.send(self, applet) + applet:set_status(tonumber(self.status_code), self.reason) + + for k, v in pairs(self.headers) do + if type(v) == "table" then + for _, hdr_val in pairs(v) do + applet:add_header(k, hdr_val) + end + else + applet:add_header(k, v) + end + end + + if not self.headers["content-type"] then + if type(self.content) == "table" then + applet:add_header("content-type", "application/json; charset=" .. + self.encoding) + if next(self.content) == nil then + -- Return empty JSON object for empty Lua tables + -- (that makes more sense then returning []) + self.content = "{}" + else + self.content = json.encode(self.content) + end + else + applet:add_header("content-type", "text/plain; charset=" .. + self.encoding) + end + end + + if not self.headers["content-length"] then + applet:add_header("content-length", #tostring(self.content)) + end + + applet:start_response() + applet:send(tostring(self.content)) +end + +--- Convert response content to JSON +-- +-- @return Lua table (decoded json) +function M.response.json(self) + return json.decode(self.content) +end + +-- Response headers getter +-- +-- Returns a header value(s) according to strategy (fold by default): +-- - single/string value for "fold", "first" and "last" strategies +-- - table for "all" strategy (for single value, a table with single element) +-- +-- @param name string Header name +-- @param strategy string "multiple header values" handling strategy +-- @return header value (string or table) or nil +function M.response.get_header(self, name, strategy) + return get_header(self.headers, name, strategy) +end + +-- Response headers iterator +-- +-- Yields key/value pairs for all headers, making sure that returned values +-- are always of string type +-- +-- @param folded boolean Specifies whether to fold headers with same name +-- @return header name/value iterator (suitable for use in "for" loops) +function M.response.get_headers(self, folded) + if folded == true then + return get_headers_folded(self.headers) + else + return get_headers_flattened(self.headers) + end +end + + +--- HTTP request class (client or server side, depending on the constructor) +M.request = {} +M.request.__index = M.request + +--- HTTP request constructor +-- +-- Parses client HTTP request (as forwarded by HAProxy) +-- +-- @param applet HAProxy AppletHTTP Lua object +-- +-- @return Request object +function M.request.parse(applet) + local self = setmetatable({}, M.request) + self.method = applet.method + + if (applet.method == "POST" or applet.method == "PUT") and + applet.length > 0 then + self.data = applet:receive() + if self.data == "" then self.data = nil end + end + + self.headers = {} + for k, v in pairs(applet.headers) do + if (v[1]) then -- (non folded header with multiple values) + self.headers[k] = {} + for _, val in pairs(v) do + table.insert(self.headers[k], val) + end + else + self.headers[k] = v[0] + end + end + + if not self.headers["host"] then + return nil, "Bad request, no Host header specified" + end + + self.cookies = parse_kv(self.headers["cookie"], "; ") + + -- TODO: Patch ApletHTTP and add schema of request + local schema = applet.schema or "http" + local url = {schema, "://", self.headers["host"], applet.path} + + self.params = {} + if applet.qs:len() > 0 then + for _, arg in ipairs(core.tokenize(applet.qs, "&", true)) do + kv = core.tokenize(arg, "=", true) + self.params[kv[1]] = kv[2] + end + url[#url+1] = "?" + url[#url+1] = applet.qs + end + + self.url = table.concat(url) + + return self +end + +--- Escape Lua pattern chars in HTTP multipart boundary +-- +-- This function escapes only minimal number of characters, which can be +-- observed in multipart boundaries, namely: -, +, ? and . +-- +-- @param s string Data to escape +-- +-- @return escaped data (string) +local function escape_pattern(s) + return s:gsub("%-", "%%-"):gsub("%+", "%%+"):gsub("%?", "%%?"):gsub("%.", "%%.") +end + +--- Parse HTTP POST data +-- +-- @return Table with submitted form data +function M.request.parse_multipart(self) + local ct = self.headers['content-type'] + if ct == nil then + return nil, 'Content-Type header not present' + end + + if self.data == nil then + return nil, 'Empty body' + end + local body = self.data + local result ={} + + if ct:find('^multipart/form[-]data;') then + local boundary = ct:match('^multipart/form[-]data; boundary=["]?(.+)["]?$') + if boundary == nil then + return nil, 'Could not parse boundary from Content-Type' + end + + -- per RFC2046, CLRF is treated as a part of boundary + -- but first one does not have it, so we're going pretend + -- it is part of the content and ignore it there (in the pattern) + boundary = string.format('%%-%%-%s.-\r\n', escape_pattern(boundary)) + + local i = 1 + local j + local old_i + + while true do + i, j = body:find(boundary, i) + + if i == nil then break end + + if old_i then + local part = body:sub(old_i, i - 1) + local k, fn, t, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"; filename="(.+)"\r\n[cC]ontent[-][tT]ype: (.+)\r\n\r\n(.+)\r\n$') + + if k then + result[k] = { + filename = fn, + content_type = t, + data = v + } + else + k, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"\r\n\r\n(.+)\r\n$') + + if k then + result[k] = v + end + end + + end + + i = j + 1 + old_i = i + end + elseif ct == 'application/x-www-form-urlencoded' then + result = parse_kv(body, '&') + else + return nil, 'Unsupported Content-Type: ' .. ct + end + + if result == nil or not next(result) then + return nil, 'Could not parse form data' + end + + return result +end + +--- Reads (all) chunks from a HTTP response +-- +-- @param socket socket object (with already established tcp connection) +-- @param get_all boolean (true by default), collect all chunks at once +-- or yield every chunk separately. +-- +-- @return Full response payload or nil and an error message +function M.receive_chunked(socket, get_all) + if socket == nil then + return nil, "http.receive_chunked: Socket is nil" + end + local data = {} + + while true do + local chunk, err = socket:receive("*l") + + if chunk == nil then + return nil, "http.receive_chunked(): Receive error (chunk length): " .. tostring(err) + end + + local chunk_len = tonumber(chunk, 16) + if chunk_len == nil then + return nil, "http.receive_chunked(): Could not parse chunk length" + end + + if chunk_len == 0 then + -- TODO: support trailers + break + end + + -- Consume next chunk (including the \r\n) + chunk, err = socket:receive(chunk_len+2) + if chunk == nil then + return nil, "http.receive_chunked(): Receive error (chunk data): " .. tostring(err) + end + + -- Strip the \r\n before collection + local chunk_data = string.sub(chunk, 1, -3) + + if get_all == false then + return chunk_data + end + + table.insert(data, chunk_data) + end + + return table.concat(data) +end + + +-- Request headers getter +-- +-- Returns a header value(s) according to strategy (fold by default): +-- - single/string value for "fold", "first" and "last" strategies +-- - table for "all" strategy (for single value, a table with single element) +-- +-- @param name string Header name +-- @param strategy string "multiple header values" handling strategy +-- @return header value (string or table) or nil +function M.request.get_header(self, name, strategy) + return get_header(self.headers, name, strategy) +end + +-- Request headers iterator +-- +-- Yields key/value pairs for all headers, making sure that returned values +-- are always of string type +-- +-- @param hdrs table Headers table as received by http.get and friends +-- @param folded boolean Specifies whether to fold headers with same name +-- @return header name/value iterator (suitable for use in "for" loops) +function M.request.get_headers(self, folded) + if folded == true then + return get_headers_folded(self.headers) + else + return get_headers_flattened(self.headers) + end +end + +--- Creates HTTP request from scratch +-- +-- @param method HTTP method +-- @param url Valid HTTP url +-- @param headers Lua table with request headers +-- @param data Request content +-- @param params Lua table with request url arguments +-- @param auth (username, password) tuple for HTTP auth +-- +-- @return request object +function M.request.create(t) + local self = setmetatable({}, M.request) + + if t.method then + self.method = t.method:lower() + else + self.method = "get" + end + self.url = t.url or nil + self.headers = copyTable(t.headers) or {} + self.data = t.data or nil + self.params = copyTable(t.params) or {} + self.auth = copyTable(t.auth) or {} + + return self +end + +--- HTTP HEAD request +function M.head(t) + return M.send("HEAD", t) +end + +--- HTTP GET request +function M.get(t) + return M.send("GET", t) +end + +--- HTTP PUT request +function M.put(t) + return M.send("PUT", t) +end + +--- HTTP POST request +function M.post(t) + return M.send("POST", t) +end + +--- HTTP DELETE request +function M.delete(t) + return M.send("DELETE", t) +end + + +--- Send HTTP request +-- +-- @param method HTTP method +-- @param url Valid HTTP url (mandatory) +-- @param headers Lua table with request headers +-- @param data Request content +-- @param params Lua table with request url arguments +-- @param auth (username, password) tuple for HTTP auth +-- @param timeout Optional timeout for socket operations (5s by default) +-- +-- @return Response object or tuple (nil, msg) on errors + +-- Note that the prefered way to call this method is via Lua +-- "keyword arguments" convention, e.g. +-- http.get{uri="http://example.net"} +function M.send(method, t) + if type(t) ~= "table" then + return nil, "http." .. method:lower() .. + ": expecting Request object for named parameters" + end + + if type(t.url) ~= "string" then + return nil, "http." .. method:lower() .. ": 'url' parameter missing" + end + + local socket = core.tcp() + socket:settimeout(t.timeout or 5) + local connect + if t.url:sub(1, 7) ~= "http://" and t.url:sub(1, 8) ~= "https://" then + t.url = "http://" .. t.url + end + local schema, host, req_uri = t.url:match("^(.*)://(.-)(/.*)$") + + if not schema then + -- maybe path (request uri) is missing + schema, host = t.url:match("^(.*)://(.-)$") + if not schema then + return nil, "http." .. method:lower() .. ": Could not parse URL: " .. t.url + end + req_uri = "/" + end + + local addr, port = host:match("(.*):(%d+)") + + if schema == "http" then + connect = socket.connect + if not port then + addr = host + port = 80 + end + elseif schema == "https" then + connect = socket.connect_ssl + if not port then + addr = host + port = 443 + end + else + return nil, "http." .. method:lower() .. ": Invalid URL schema " .. tostring(schema) + end + + local c, err = connect(socket, addr, port) + + if c then + local req = {} + local hdr_tbl = {} + + if t.headers then + for k, v in pairs(t.headers) do + if type(v) == "table" then + table.insert(hdr_tbl, k .. ": " .. table.concat(v, ",")) + else + table.insert(hdr_tbl, k .. ": " .. tostring(v)) + end + end + else + t.headers = {} -- dummy table + end + + if not t.headers.host then + -- 'Host' header must be provided for HTTP/1.1 + table.insert(hdr_tbl, "host: " .. host) + end + + if not t.headers["accept"] then + table.insert(hdr_tbl, "accept: */*") + end + + if not t.headers["user-agent"] then + table.insert(hdr_tbl, "user-agent: haproxy-lua-http/1.0") + end + + if not t.headers.connection then + table.insert(hdr_tbl, "connection: close") + end + + if t.data then + req[4] = t.data + if not t.headers or not t.headers["content-length"] then + table.insert(hdr_tbl, "content-length: " .. tostring(#t.data)) + end + end + + req[1] = method .. " " .. req_uri .. " HTTP/1.1\r\n" + req[2] = table.concat(hdr_tbl, "\r\n") + req[3] = "\r\n\r\n" + + local r, e = socket:send(table.concat(req)) + + if not r then + socket:close() + return nil, "http." .. method:lower() .. ": " .. tostring(e) + end + + local line + r = M.response.create() + + while true do + line, err = socket:receive("*l") + + if not line then + socket:close() + return nil, "http." .. method:lower() .. + ": Receive error (headers): " .. err + end + + if line == "" then break end + + if not r.status_code then + _, r.status_code, r.reason = + line:match("(HTTP/1.[01]) (%d%d%d)(.*)") + if not _ then + socket:close() + return nil, "http." .. method:lower() .. + ": Could not parse request line" + end + r.status_code = tonumber(r.status_code) + else + local sep = line:find(":") + local hdr_name = line:sub(1, sep-1):lower() + local hdr_val = line:sub(sep+1):match("^%s*(.*%S)%s*$") or "" + + if r.headers[hdr_name] == nil then + r.headers[hdr_name] = hdr_val + elseif type(r.headers[hdr_name]) == "table" then + table.insert(r.headers[hdr_name], hdr_val) + else + r.headers[hdr_name] = { + r.headers[hdr_name], + hdr_val + } + end + end + end + + if method:lower() == "head" then + r.content = nil + socket:close() + return r + end + + if r.headers["content-length"] and tonumber(r.headers["content-length"]) > 0 then + r.content, err = socket:receive("*a") + + if not r.content then + socket:close() + return nil, "http." .. method:lower() .. + ": Receive error (content): " .. err + end + end + + if r.headers["transfer-encoding"] and r.headers["transfer-encoding"] == "chunked" then + r.content, err = M.receive_chunked(socket) + if r.content == nil then + socket:close() + return nil, err + end + end + + socket:close() + return r + else + return nil, "http." .. method:lower() .. ": Connection error: " .. tostring(err) + end +end + +M.base64 = {} + +--- URL safe base64 encoder +-- +-- Padding ('=') is omited, as permited per RFC +-- https://tools.ietf.org/html/rfc4648 +-- in order to follow JSON Web Signature RFC +-- https://tools.ietf.org/html/rfc7515 +-- +-- @param s String (can be binary data) to encode +-- @param enc Function which implements base64 encoder (e.g. HAProxy base64 fetch) +-- @return Encoded string +function M.base64.encode(s, enc) + if not s then return nil end + local u = enc(s) + + if not u then + return nil + end + + local pad_len = 2 - ((#s-1) % 3) + + if pad_len > 0 then + return u:sub(1, - pad_len - 1):gsub('[+]', '-'):gsub('[/]', '_') + else + return u:gsub('[+]', '-'):gsub('[/]', '_') + end +end + +--- URLsafe base64 decoder +-- +-- @param s Base64 string to decode +-- @param dec Function which implements base64 decoder (e.g. HAProxy b64dec fetch) +-- @return Decoded string (can be binary data) +function M.base64.decode(s, dec) + if not s then return nil end + + local e = s:gsub('[-]', '+'):gsub('[_]', '/') + return dec(e .. string.rep('=', 3 - ((#s - 1) % 4))) +end + +return M diff --git a/states/haproxy/scripts/json.lua b/states/haproxy/scripts/json.lua new file mode 100644 index 0000000..711ef78 --- /dev/null +++ b/states/haproxy/scripts/json.lua @@ -0,0 +1,388 @@ +-- +-- json.lua +-- +-- Copyright (c) 2020 rxi +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +-- of the Software, and to permit persons to whom the Software is furnished to do +-- so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- + +local json = { _version = "0.1.2" } + +------------------------------------------------------------------------------- +-- Encode +------------------------------------------------------------------------------- + +local encode + +local escape_char_map = { + [ "\\" ] = "\\", + [ "\"" ] = "\"", + [ "\b" ] = "b", + [ "\f" ] = "f", + [ "\n" ] = "n", + [ "\r" ] = "r", + [ "\t" ] = "t", +} + +local escape_char_map_inv = { [ "/" ] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(val) + return "null" +end + + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + -- Encode + for i, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + -- Treat as an object + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + [ "nil" ] = encode_nil, + [ "table" ] = encode_table, + [ "string" ] = encode_string, + [ "number" ] = encode_number, + [ "boolean" ] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + +function json.encode(val) + return ( encode(val) ) +end + + +------------------------------------------------------------------------------- +-- Decode +------------------------------------------------------------------------------- + +local parse + +local function create_set(...) + local res = {} + for i = 1, select("#", ...) do + res[ select(i, ...) ] = true + end + return res +end + +local space_chars = create_set(" ", "\t", "\r", "\n") +local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") +local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u") +local literals = create_set("true", "false", "null") + +local literal_map = { + [ "true" ] = true, + [ "false" ] = false, + [ "null" ] = nil, +} + + +local function next_char(str, idx, set, negate) + for i = idx, #str do + if set[str:sub(i, i)] ~= negate then + return i + end + end + return #str + 1 +end + + +local function decode_error(str, idx, msg) + local line_count = 1 + local col_count = 1 + for i = 1, idx - 1 do + col_count = col_count + 1 + if str:sub(i, i) == "\n" then + line_count = line_count + 1 + col_count = 1 + end + end + error( string.format("%s at line %d col %d", msg, line_count, col_count) ) +end + + +local function codepoint_to_utf8(n) + -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa + local f = math.floor + if n <= 0x7f then + return string.char(n) + elseif n <= 0x7ff then + return string.char(f(n / 64) + 192, n % 64 + 128) + elseif n <= 0xffff then + return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128) + elseif n <= 0x10ffff then + return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, + f(n % 4096 / 64) + 128, n % 64 + 128) + end + error( string.format("invalid unicode codepoint '%x'", n) ) +end + + +local function parse_unicode_escape(s) + local n1 = tonumber( s:sub(1, 4), 16 ) + local n2 = tonumber( s:sub(7, 10), 16 ) + -- Surrogate pair? + if n2 then + return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) + else + return codepoint_to_utf8(n1) + end +end + + +local function parse_string(str, i) + local res = "" + local j = i + 1 + local k = j + + while j <= #str do + local x = str:byte(j) + + if x < 32 then + decode_error(str, j, "control character in string") + + elseif x == 92 then -- `\`: Escape + res = res .. str:sub(k, j - 1) + j = j + 1 + local c = str:sub(j, j) + if c == "u" then + local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1) + or str:match("^%x%x%x%x", j + 1) + or decode_error(str, j - 1, "invalid unicode escape in string") + res = res .. parse_unicode_escape(hex) + j = j + #hex + else + if not escape_chars[c] then + decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string") + end + res = res .. escape_char_map_inv[c] + end + k = j + 1 + + elseif x == 34 then -- `"`: End of string + res = res .. str:sub(k, j - 1) + return res, j + 1 + end + + j = j + 1 + end + + decode_error(str, i, "expected closing quote for string") +end + + +local function parse_number(str, i) + local x = next_char(str, i, delim_chars) + local s = str:sub(i, x - 1) + local n = tonumber(s) + if not n then + decode_error(str, i, "invalid number '" .. s .. "'") + end + return n, x +end + + +local function parse_literal(str, i) + local x = next_char(str, i, delim_chars) + local word = str:sub(i, x - 1) + if not literals[word] then + decode_error(str, i, "invalid literal '" .. word .. "'") + end + return literal_map[word], x +end + + +local function parse_array(str, i) + local res = {} + local n = 1 + i = i + 1 + while 1 do + local x + i = next_char(str, i, space_chars, true) + -- Empty / end of array? + if str:sub(i, i) == "]" then + i = i + 1 + break + end + -- Read token + x, i = parse(str, i) + res[n] = x + n = n + 1 + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "]" then break end + if chr ~= "," then decode_error(str, i, "expected ']' or ','") end + end + return res, i +end + + +local function parse_object(str, i) + local res = {} + i = i + 1 + while 1 do + local key, val + i = next_char(str, i, space_chars, true) + -- Empty / end of object? + if str:sub(i, i) == "}" then + i = i + 1 + break + end + -- Read key + if str:sub(i, i) ~= '"' then + decode_error(str, i, "expected string for key") + end + key, i = parse(str, i) + -- Read ':' delimiter + i = next_char(str, i, space_chars, true) + if str:sub(i, i) ~= ":" then + decode_error(str, i, "expected ':' after key") + end + i = next_char(str, i + 1, space_chars, true) + -- Read value + val, i = parse(str, i) + -- Set + res[key] = val + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "}" then break end + if chr ~= "," then decode_error(str, i, "expected '}' or ','") end + end + return res, i +end + + +local char_func_map = { + [ '"' ] = parse_string, + [ "0" ] = parse_number, + [ "1" ] = parse_number, + [ "2" ] = parse_number, + [ "3" ] = parse_number, + [ "4" ] = parse_number, + [ "5" ] = parse_number, + [ "6" ] = parse_number, + [ "7" ] = parse_number, + [ "8" ] = parse_number, + [ "9" ] = parse_number, + [ "-" ] = parse_number, + [ "t" ] = parse_literal, + [ "f" ] = parse_literal, + [ "n" ] = parse_literal, + [ "[" ] = parse_array, + [ "{" ] = parse_object, +} + + +parse = function(str, idx) + local chr = str:sub(idx, idx) + local f = char_func_map[chr] + if f then + return f(str, idx) + end + decode_error(str, idx, "unexpected character '" .. chr .. "'") +end + + +function json.decode(str) + if type(str) ~= "string" then + error("expected argument of type string, got " .. type(str)) + end + local res, idx = parse(str, next_char(str, 1, space_chars, true)) + idx = next_char(str, idx, space_chars, true) + if idx <= #str then + decode_error(str, idx, "trailing garbage") + end + return res +end + + +return json diff --git a/states/haproxy/scripts/weight_by_latency.lua b/states/haproxy/scripts/weight_by_latency.lua new file mode 100644 index 0000000..0160a95 --- /dev/null +++ b/states/haproxy/scripts/weight_by_latency.lua @@ -0,0 +1,43 @@ +local function getmax(t) + local tmpvalue = 100000 + local svname + local value + + for k,v in pairs(t) do + if v < tmpvalue then + svname = k + value = v + end + tmpvalue = v + end + return svname, value +end + +local function arrange_backends() + local results = {} + while true do + for _, backend in pairs(core.backends) do + results = {} + for n,server in pairs(backend.servers) do + if server:get_stats()["check_status"] == "L4OK" then + local svname = server:get_stats()["svname"] + local latency = server:get_stats()["check_duration"] + results[svname] = latency + end + end + local b,c = getmax(results) + if b ~= nil then + for n,server in pairs(backend.servers) do + if b == server.name then + server:set_weight("10") + else + server:set_weight("1") + end + end + end + end + core.msleep(1000) + end +end + +core.register_task(arrange_backends) diff --git a/states/haproxy/templates/haproxy.cfg.j2 b/states/haproxy/templates/haproxy.cfg.j2 index 9f5e1a5..deee9cc 100644 --- a/states/haproxy/templates/haproxy.cfg.j2 +++ b/states/haproxy/templates/haproxy.cfg.j2 @@ -1,21 +1,35 @@ ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} {%- from "haproxy/map.jinja" import haproxy,certs with context %} -{%- macro internal_access() -%} - acl internal src -f /etc/haproxy/access +{%- macro internal() -%} + acl internal src -f {{ haproxy.config.dir }}/maps/access http-response return status 403 default-errorfiles if ! internal {%- endmacro -%} -{%- macro handle_head() -%} +{%- macro head() -%} http-request return status 200 if { method -i HEAD } {%- endmacro -%} -{%- macro handle_endpoints(endpoints, check, ssl) -%} - {%- for endpoint in endpoints %} - server {{ endpoint.name }} {{ endpoint.name }}:{{ endpoint.port }}{{ " check observe layer7 " if check|default(true) }}{{ " ssl verify none " if ssl|default(false) }} +{%- macro serverheader() -%} + http-response set-header server "{{ haproxy.config.servername }}" +{%- endmacro -%} + +{%- macro endpoints(servers, check, ssl) -%} + {%- for server in servers %} + server {{ server.name }} {{ server.name }}:{{ server.port }}{{ " check observe layer7 inter 1s fall 5 rise 5 " if check|default(true) }}{{ " ssl verify none " if ssl|default(false) }} {%- endfor %} {%- endmacro -%} +{%- macro cache() -%} + http-request cache-use static if { path_end .css .js .png .jpg } + http-response cache-store static +{%- endmacro -%} + +{%- macro compression() -%} + compression algo gzip + compression type text/html text/plain text/css text/javascript application/javascript +{%- endmacro -%} + {%- macro admin() -%} listen stats mode http @@ -26,32 +40,32 @@ listen stats {%- endmacro -%} {%- macro api() -%} -listen stats - mode http - bind *:7000 v4v6 - stats enable - stats refresh 5s - stats uri / -{%- endmacro %} - -global -{%- for filename in haproxy.scripts %} - lua-load {{ haproxy.config.dir }}/scripts/{{ filename }} -{%- endfor %} - maxconn 1000 stats socket ipv4@127.0.0.1:9990 level admin stats socket /var/run/hap-lb.sock mode 666 level admin stats timeout 2m +{%- endmacro %} + +global + lua-prepend-path {{ haproxy.config.dir }}/scripts/?.lua +{%- for file in haproxy.scripts %} + {%- if not file.lib %} + lua-load {{ haproxy.config.dir }}/{{ file.name }} + {%- endif %} +{%- endfor %} + maxconn 1000 +{%- if haproxy.config.api.enable %} + {{ api() }} +{%- endif %} ssl-default-bind-ciphers {{ haproxy.config.ssl_ciphers|join(":") }} ssl-default-bind-options {{ haproxy.config.ssl_options|join(" ") }} ssl-default-server-ciphers {{ haproxy.config.ssl_ciphers|join(":") }} ssl-default-server-options {{ haproxy.config.ssl_options|join(" ") }} - crt-base {{ haproxy.config.acme_dir }}/certs - ssl-dh-param-file {{ haproxy.config.acme_dir }}/dh/dh.pem + crt-base {{ haproxy.config.acme_fullchains_dir }} + ssl-dh-param-file {{ haproxy.config.acme_dh_dir }}/dh.pem defaults -{%- for default in haproxy.config.defaults.keys() %} - {{ default }} +{%- for key, value in haproxy.config.defaults.items() %} + {{ key }} {{ value }} {%- endfor %} {%- if haproxy.config.admin %} @@ -59,10 +73,13 @@ defaults {%- endif %} cache static - total-max-size 4095 + total-max-size 256 max-object-size 50000 max-age 120 +backend per_ip_rates + stick-table type string size 1m expire 10s store http_req_rate(10s) + frontend http bind *:80,:::80 v4v6 mode http @@ -70,37 +87,59 @@ frontend http http-request redirect scheme https if http frontend https - bind *:443,:::443 v4v6 {% for cert in certs %}{{ " ssl crt " + cert + " " }}{% endfor %} + bind *:443,:::443 v4v6 ssl crt {{ haproxy.config.acme_fullchains_dir }}{% if haproxy.config.http2 %} alpn h2,http/1.1{% endif %} + mode http + option httplog {%- for name, values in haproxy.config.vhosts.items() %} use_backend {{ name }} if { hdr(Host) -i {{ values.host }} } {%- endfor %} + http-request track-sc0 src table per_ip_rates + http-request capture req.hdr(User-Agent) len 200 + http-request capture req.hdr(Content-Type) len 200 + http-request capture req.hdr(Referer) len 200 + http-request capture sc_http_req_rate(0) len 4 + http-request deny deny_status 429 if { sc_http_req_rate(0) gt 100 } + http-request set-header x-proxy-id "{{ salt["grains.get"]("host") }}" + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" default_backend nginx {% for name, values in haproxy.config.vhosts.items() %} backend {{ name }} balance {{ values.balance|default("roundrobin") }} -{%- if values.handle_head|default(false) %} - {{ handle_head() }} + mode http + option forwardfor +{%- if values.head|default(false) %} + {{ head() }} {%- endif %} + +{%- if values.compression|default(true) %} + {{ compression() }} +{%- endif %} + {%- if values.usecache|default(true) %} - http-request cache-use static if { path_end .css .js .png .jpg } - http-response cache-store static + {{ cache() }} {%- endif %} -{%- if values.internal_access|default(false) %} - {{ internal_access() }} + +{%- if values.serverheader|default(true) %} + {{ serverheader() }} {%- endif %} - {{- handle_endpoints(values.endpoints, values.check, values.ssl) }} -{% endfor %} + +{%- if values.internal|default(false) %} + {{ internal() }} +{%- endif %} + {{- endpoints(values.servers, values.check, values.ssl) }} +{% endfor -%} {% for name, values in haproxy.config.services.items() %} listen {{ name }} bind :::{{ values.port }} v4v6 mode tcp + option tcplog {%- if values.type == "postgres" %} option pgsql-check user postgres {%- endif %} default-server inter 3s fall 3 - {%- for endpoint in values.endpoints %} - server {{ endpoint.name }} {{ endpoint.name }}:{{ endpoint.port }} check {{ "backup" if endpoint.backup|default(false) }} port {{ endpoint.port }} + {%- for server in values.servers %} + server {{ server.name }} {{ server.name }}:{{ server.port }} check {{ "backup" if server.backup|default(false) }} port {{ server.port }} {%- endfor %} -{% endfor %} +{% endfor -%}