From 14800c11d13fdb7364b0ef3bc0b3cbd670a457c3 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Thu, 13 Apr 2023 23:13:34 +0200 Subject: [PATCH] updated haproxy state --- states/haproxy/defaults.yaml | 27 +- states/haproxy/install.sls | 17 +- states/haproxy/map.jinja | 2 + states/haproxy/mods/haproxy.c | 201 +++++ states/haproxy/scripts/compile.lua | 25 + states/haproxy/scripts/geoip.lua | 19 + states/haproxy/scripts/http.lua | 830 --------------------- states/haproxy/templates/haproxy.cfg.j2 | 51 +- states/haproxy/templates/maps/countries.j2 | 5 + 9 files changed, 327 insertions(+), 850 deletions(-) create mode 100644 states/haproxy/mods/haproxy.c create mode 100644 states/haproxy/scripts/compile.lua create mode 100644 states/haproxy/scripts/geoip.lua delete mode 100644 states/haproxy/scripts/http.lua create mode 100644 states/haproxy/templates/maps/countries.j2 diff --git a/states/haproxy/defaults.yaml b/states/haproxy/defaults.yaml index ec638ce..703b191 100644 --- a/states/haproxy/defaults.yaml +++ b/states/haproxy/defaults.yaml @@ -3,11 +3,14 @@ haproxy: enabled: true packages: - haproxy + - liblua5.3-dev + - lua-filesystem - libcurl4-openssl-dev - libmaxminddb-dev - libjansson-dev maps: - access + - countries - domains - redirects - vhosts @@ -19,6 +22,15 @@ haproxy: config: dir: /etc/haproxy configfile: haproxy.cfg + syscontact: haproxy@example.com + geoip: + enabled: true + countries: + FR: OK + dbs: + - name: geoip/GeoLite2-City.mmdb + url: https://git.paulbsd.com/paulbsd/GeoLite.mmdb/releases/download/2023.03.26/GeoLite2-City.mmdb + lua_max_mem: 1024 peers: hosts: [] port: 4096 @@ -28,7 +40,16 @@ haproxy: - scripts - mods - errors + geoip_dbs: scripts: + - name: mods/haproxy.c + lib: true + - name: scripts/compile.lua + lib: true + - name: scripts/geoip.lua + lib: false + args: + - /etc/haproxy/geoip/GeoLite2-City.mmdb - name: scripts/json.lua lib: true - name: scripts/collector.lua @@ -38,7 +59,7 @@ haproxy: namespace: paulbsd user: haproxy group: haproxy - servername: High-performance Web Server + servername: "High-performance Web Server 1.0" http2: true defaults: #log: global @@ -82,11 +103,13 @@ haproxy: - .js - .png - .jpg + - .svg + - .webp ddos: timeperiod: 10s maxrequests: 200 size: 1m - domains: {} + domains: [] vhosts: {} services: {} spoe: {} diff --git a/states/haproxy/install.sls b/states/haproxy/install.sls index 54509c5..745405f 100644 --- a/states/haproxy/install.sls +++ b/states/haproxy/install.sls @@ -32,7 +32,22 @@ haproxy-script-{{ file.name }}: - source: salt://haproxy/{{ file.name }} - user: {{ haproxy.config.user }} - group: {{ haproxy.config.group }} - - mode: "0700" + - mode: "0600" + - watch_in: + - service: haproxy-service +{% endfor %} + +{% for file in haproxy.config.geoip.dbs %} +haproxy-geoip-{{ file.name }}: + file.managed: + - name: {{ haproxy.config.dir }}/{{ file.name }} + - source: {{ file.url }} + - skip_verify: True + - user: {{ haproxy.config.user }} + - group: {{ haproxy.config.group }} + - mode: "0600" + - require: + - file: haproxy-config-geoip-dir - watch_in: - service: haproxy-service {% endfor %} diff --git a/states/haproxy/map.jinja b/states/haproxy/map.jinja index b219565..adf975d 100644 --- a/states/haproxy/map.jinja +++ b/states/haproxy/map.jinja @@ -18,3 +18,5 @@ {%- endfor %} {% do haproxy.config.peers.update({"hosts": peers_ip }) %} + +{% do haproxy.config.update({"syscontact": salt['pillar.get']('syscontact',default='anonymous@example.com')}) %} diff --git a/states/haproxy/mods/haproxy.c b/states/haproxy/mods/haproxy.c new file mode 100644 index 0000000..0b7d540 --- /dev/null +++ b/states/haproxy/mods/haproxy.c @@ -0,0 +1,201 @@ +/* +#!/usr/bin/env zsh + +for i in $(ls *.c) +do + basename=$(echo ${i}|awk -F '.' '{ print $1}') + sudo -u haproxy cc -I/usr/include/ -I/usr/include/lua5.3/ -fPIC -shared -o ${basename}.so ${i} -lcurl -ljansson -lmaxminddb +done +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static MMDB_s mmdb; + +const int STR_MAX_LEN = 255; + +static int load_geoip(lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + if (status != 0) { + printf("Error loading geoip database\n"); + exit(1); + } else { + printf("Loaded geoip database at: %s\n", filename); + } + return 0; +} + +int get_geoip_info(const char* ip_address, MMDB_lookup_result_s* result) { + int gai_error, mmdb_error; + (*result) = MMDB_lookup_string(&mmdb, ip_address, &gai_error, &mmdb_error); + + if (0 != gai_error) { + fprintf(stderr, "\n Error from getaddrinfo for %s - %s\n\n", ip_address, gai_strerror(gai_error)); + exit(2); + } + + if (MMDB_SUCCESS != mmdb_error) { + fprintf(stderr, "\n Got an error from libmaxminddb: %s\n\n", MMDB_strerror(mmdb_error)); + exit(3); + } + return 0; +} + +int filter_value(MMDB_lookup_result_s* result, + MMDB_entry_data_list_s** entry_data_list, + MMDB_entry_data_s* entry_data, + const char* query) { + int status = MMDB_get_entry_data_list(&result->entry, entry_data_list); + + if (result->found_entry) { + if (strcmp(query, "country") == 0) { + status = MMDB_get_value(&result->entry, entry_data, + "country", + "iso_code", + NULL); + return status; + } else if (strcmp(query, "city") == 0){ + status = MMDB_get_value(&result->entry, entry_data, + "city", + "names", + "en", + NULL); + return status; + } + } + return 1; +} + +static int get_country(lua_State *L) { + const char *ip_address = luaL_checkstring(L, 1); + char country[STR_MAX_LEN]; + + MMDB_entry_data_list_s *entry_data_list = NULL; + MMDB_entry_data_s entry_data; + MMDB_lookup_result_s result; + + get_geoip_info(ip_address, &result); + + int status = filter_value(&result, &entry_data_list, &entry_data, "country"); + + if (status == 0) { + sprintf(country, "%.*s", entry_data.data_size,entry_data.utf8_string); + } else { + strcpy(country, "not found"); + } + + lua_pushstring(L, country); + return 1; +} + +static int get_city(lua_State *L) { + const char *ip_address = luaL_checkstring(L, 1); + char city[STR_MAX_LEN]; + + MMDB_entry_data_list_s *entry_data_list = NULL; + MMDB_entry_data_s entry_data; + MMDB_lookup_result_s result; + + get_geoip_info(ip_address, &result); + + int status = filter_value(&result, &entry_data_list, &entry_data, "city"); + + if (status == 0) { + sprintf(city, "%.*s", entry_data.data_size, entry_data.utf8_string); + } else { + strcpy(city, "not found"); + } + + lua_pushstring(L, city); + return 1; +} + +static int download_file() { + return 0; +} + +static int request(lua_State *L) { + const char *url = luaL_checkstring(L, 1); + CURL *curl = curl_easy_init(); + if (curl) { + CURLcode res; + curl_easy_setopt(curl, CURLOPT_URL, url); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + } + return 0; +} + +/* +static void stackDump(lua_State *L) +{ + int i; + int top = lua_gettop(L); + + for(i = 1; i <= top; i++) { + int t = lua_type(L, i); + switch(t) { + case LUA_TNIL: + printf("nil"); + break; + case LUA_TBOOLEAN: + printf(lua_toboolean(L, i) ? "true" : "false"); + break; + case LUA_TNUMBER: + printf("%g", lua_tonumber(L, i)); + break; + case LUA_TSTRING: + printf("%s", lua_tostring(L, i)); + break; + + default: + printf("%s", lua_typename(L, t)); + } + printf(" "); + } + printf("\n\n"); +}*/ + + +int get_info(lua_State *L) { + lua_getglobal(L, "core"); + lua_getfield(L, -1, "get_info"); + + int err = lua_pcall(L, 0, 1, 0); + + if (err == 0) { + lua_getfield(L, -1, "Build info"); + const char *version = lua_tostring(L, -1); + printf("HAProxy version: %s\n",version); + } else { + printf("%d %s\n", err, lua_tostring(L, -1)); + } + return 0; +} + +int luaopen_haproxy(lua_State *L) { + printf("Loading C lua module\n"); + + lua_pushcfunction(L, load_geoip); + lua_setglobal(L, "load_geoip"); + + lua_pushcfunction(L, get_country); + lua_setglobal(L, "get_country"); + + lua_pushcfunction(L, get_city); + lua_setglobal(L, "get_city"); + + lua_pushcfunction(L, request); + lua_setglobal(L, "request"); + return 0; +} diff --git a/states/haproxy/scripts/compile.lua b/states/haproxy/scripts/compile.lua new file mode 100644 index 0000000..35bb7ee --- /dev/null +++ b/states/haproxy/scripts/compile.lua @@ -0,0 +1,25 @@ +lfs = require("lfs") + +modpath = "/etc/haproxy/mods" + +local compile = {} +local user = "haproxy" +local group = "haproxy" +local libs = {"-lcurl", "-ljansson", "-lmaxminddb"} + +function compile.check(module) + local sourcepath = modpath.."/"..module..".c" + local binpath = modpath.."/"..module..".so" + + local binexists = io.open(binpath) + + if not binexists or (binexists and lfs.attributes(sourcepath).change > lfs.attributes(binpath).change ) then + local cmd = "cc -I/usr/include/ -I/usr/include/lua5.3/ -fPIC -shared -o " .. binpath .. " " .. sourcepath .. " " ..table.concat(libs," ") + local res = io.popen(cmd) + local aa = res:read("a*") + io.popen("chown "..user..":"..group.." "..binpath) + io.popen("chmod 600 "..binpath) + end +end + +return compile diff --git a/states/haproxy/scripts/geoip.lua b/states/haproxy/scripts/geoip.lua new file mode 100644 index 0000000..2877c22 --- /dev/null +++ b/states/haproxy/scripts/geoip.lua @@ -0,0 +1,19 @@ +local compile = require("compile") +compile.check("haproxy") +local haproxy = require("haproxy") + +local args = table.pack(...) +load_geoip(args[1]) + +local function country(txn, var1) + local src = txn:get_var(var1) + return get_country(src) +end + +local function city(txn, var1) + local src = txn:get_var(var1) + return get_city(src) +end + +core.register_fetches("country", country) +core.register_fetches("city", city) diff --git a/states/haproxy/scripts/http.lua b/states/haproxy/scripts/http.lua deleted file mode 100644 index 947a761..0000000 --- a/states/haproxy/scripts/http.lua +++ /dev/null @@ -1,830 +0,0 @@ --- --- HTTP 1.1 library for HAProxy Lua modules --- --- The library is loosely modeled after Python's Requests Library --- using the same field names and very similar calling conventions for --- "HTTP verb" methods (where we use Lua specific named parameter support) --- --- In addition to client side, the library also supports server side request --- parsing, where we utilize HAProxy Lua API for all heavy lifting. --- --- --- Copyright (c) 2017-2020. Adis Nezirović --- Copyright (c) 2017-2020. HAProxy Technologies, LLC. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- SPDX-License-Identifier: Apache-2.0 - -local _author = "Adis Nezirovic " -local _copyright = "Copyright 2017-2020. HAProxy Technologies, LLC." -local _version = "1.0.0" - -local json = require "json" - --- Utility functions - --- HTTP headers fetch helper --- --- Returns a header value(s) according to strategy (fold by default): --- - single/string value for "fold", "first" and "last" strategies --- - table for "all" strategy (for single value, a table with single element) --- --- @param hdrs table Headers table as received by http.get and friends --- @param name string Header name --- @param strategy string "multiple header values" handling strategy --- @return header value (string or table) or nil -local function get_header(hdrs, name, strategy) - if hdrs == nil or name == nil then return nil end - - local v = hdrs[name:lower()] - if type(v) ~= "table" and strategy ~= "all" then return v end - - if strategy == nil or strategy == "fold" then - return table.concat(v, ",") - elseif strategy == "first" then - return v[1] - elseif strategy == "last" then - return v[#v] - elseif strategy == "all" then - if type(v) ~= "table" then - return {v} - else - return v - end - end -end - - - --- HTTP headers iterator helper --- --- Returns key/value pairs for all header, making sure that returned values --- are always of string type (if necessary, it folds multiple headers with --- the same name) --- --- @param hdrs table Headers table as received by http.get and friends --- @return header name/value iterator (suitable for use in "for" loops) -local function get_headers_folded(hdrs) - if hdrs == nil then - return function() end - end - - local function iter(t, k) - local v - k, v = next(t, k) - - if v ~= nil then - if type(v) ~= "table" then - return k, v - else - return k, table.concat(v, ",") - end - end - end - - return iter, hdrs, nil -end - --- HTTP headers iterator --- --- Returns key/value pairs for all headers, for multiple headers with same name --- it will return every name/value pair --- (i.e. you can safely use it to process responses with 'Set-Cookie' header) --- --- @param hdrs table Headers table as received by http.get and friends --- @return header name/value iterator (suitable for use in "for" loops) -local function get_headers_flattened(hdrs) - if hdrs == nil then - return function() end - end - - local k -- top level key (string) - local k_sub = 0 -- sub table key (integer), 0 if item not a table, - -- nil after last sub table iteration - local v_sub -- sub table - - return function () - local v - if k_sub == 0 then - k, v = next(hdrs, k) - if k == nil then return end - else - k_sub, v = next(v_sub, k_sub) - - if k_sub == nil then - k_sub = 0 - k, v = next(hdrs, k) - end - end - - if k == nil then return end - - if type(v) ~= "table" then - return k, v - else - v_sub = v - k_sub = k_sub + 1 - return k, v[k_sub] - end - end -end - - ---- Parse key/value pairs from a string --- --- @param s Lua string with (multiple) key/value pairs (separated by 'sep') --- --- @return Table with parsed keys and values or nil -local function parse_kv(s, sep) - if s == nil then return nil end - idx = 1 - result = {} - - while idx < s:len() do - i, j = s:find(sep, idx) - - if i == nil then - k, v = string.match(s:sub(idx), "^(.-)=(.*)$") - if k then result[k] = v end - break - end - - k, v = string.match(s:sub(idx, i-1), "^(.-)=(.*)$") - if k then result[k] = v end - idx = j + 1 - end - - if next(result) == nil then - return nil - else - return result - end -end - - ---- Make deep copy of table and it's values --- --- Use only for simple tables (it handles nested table values), but not for --- Lua objects or similar, or very big tables (this uses recursion). --- --- @param t Cloned Lua table or nil --- --- @return Cloned table or nil -local function copyTable(t) - if type(t) ~= "table" then - return nil - end - - local r = {} - - for k, v in pairs(t) do - if type(v) == "table" then - r[k] = copyTable(v) - else - r[k] = v - end - end - - return r -end - - ---- Namespace object which hosts HTTP verb methods and request/response classes -local M = {} - - ---- HTTP response class -M.response = {} -M.response.__index = M.response - -local _reason = { - [200] = "OK", - [201] = "Created", - [204] = "No Content", - [301] = "Moved Permanently", - [302] = "Found", - [400] = "Bad Request", - [403] = "Forbidden", - [404] = "Not Found", - [405] = "Method Not Allowed", - [408] = "Request Timeout", - [413] = "Payload Too Large", - [429] = "Too many requests", - [500] = "Internal Server Error", - [501] = "Not Implemented", - [502] = "Bad Gateway", - [503] = "Service Unavailable", - [504] = "Gateway Timeout" -} - ---- Creates HTTP response from scratch --- --- @param status_code HTTP status code --- @param reason HTTP status code text (e.g. "OK" for 200 response) --- @param headers HTTP response headers --- @param request The HTTP request which triggered the response --- @param encoding Default encoding for response or conversions --- --- @return response object -function M.response.create(t) - local self = setmetatable({}, M.response) - - if not t then - t = {} - end - - self.status_code = t.status_code or nil - self.reason = t.reason or _reason[self.status_code] or "" - self.headers = copyTable(t.headers) or {} - self.content = t.content or "" - self.request = t.request or nil - self.encoding = t.encoding or "utf-8" - - return self -end - -function M.response.send(self, applet) - applet:set_status(tonumber(self.status_code), self.reason) - - for k, v in pairs(self.headers) do - if type(v) == "table" then - for _, hdr_val in pairs(v) do - applet:add_header(k, hdr_val) - end - else - applet:add_header(k, v) - end - end - - if not self.headers["content-type"] then - if type(self.content) == "table" then - applet:add_header("content-type", "application/json; charset=" .. - self.encoding) - if next(self.content) == nil then - -- Return empty JSON object for empty Lua tables - -- (that makes more sense then returning []) - self.content = "{}" - else - self.content = json.encode(self.content) - end - else - applet:add_header("content-type", "text/plain; charset=" .. - self.encoding) - end - end - - if not self.headers["content-length"] then - applet:add_header("content-length", #tostring(self.content)) - end - - applet:start_response() - applet:send(tostring(self.content)) -end - ---- Convert response content to JSON --- --- @return Lua table (decoded json) -function M.response.json(self) - return json.decode(self.content) -end - --- Response headers getter --- --- Returns a header value(s) according to strategy (fold by default): --- - single/string value for "fold", "first" and "last" strategies --- - table for "all" strategy (for single value, a table with single element) --- --- @param name string Header name --- @param strategy string "multiple header values" handling strategy --- @return header value (string or table) or nil -function M.response.get_header(self, name, strategy) - return get_header(self.headers, name, strategy) -end - --- Response headers iterator --- --- Yields key/value pairs for all headers, making sure that returned values --- are always of string type --- --- @param folded boolean Specifies whether to fold headers with same name --- @return header name/value iterator (suitable for use in "for" loops) -function M.response.get_headers(self, folded) - if folded == true then - return get_headers_folded(self.headers) - else - return get_headers_flattened(self.headers) - end -end - - ---- HTTP request class (client or server side, depending on the constructor) -M.request = {} -M.request.__index = M.request - ---- HTTP request constructor --- --- Parses client HTTP request (as forwarded by HAProxy) --- --- @param applet HAProxy AppletHTTP Lua object --- --- @return Request object -function M.request.parse(applet) - local self = setmetatable({}, M.request) - self.method = applet.method - - if (applet.method == "POST" or applet.method == "PUT") and - applet.length > 0 then - self.data = applet:receive() - if self.data == "" then self.data = nil end - end - - self.headers = {} - for k, v in pairs(applet.headers) do - if (v[1]) then -- (non folded header with multiple values) - self.headers[k] = {} - for _, val in pairs(v) do - table.insert(self.headers[k], val) - end - else - self.headers[k] = v[0] - end - end - - if not self.headers["host"] then - return nil, "Bad request, no Host header specified" - end - - self.cookies = parse_kv(self.headers["cookie"], "; ") - - -- TODO: Patch ApletHTTP and add schema of request - local schema = applet.schema or "http" - local url = {schema, "://", self.headers["host"], applet.path} - - self.params = {} - if applet.qs:len() > 0 then - for _, arg in ipairs(core.tokenize(applet.qs, "&", true)) do - kv = core.tokenize(arg, "=", true) - self.params[kv[1]] = kv[2] - end - url[#url+1] = "?" - url[#url+1] = applet.qs - end - - self.url = table.concat(url) - - return self -end - ---- Escape Lua pattern chars in HTTP multipart boundary --- --- This function escapes only minimal number of characters, which can be --- observed in multipart boundaries, namely: -, +, ? and . --- --- @param s string Data to escape --- --- @return escaped data (string) -local function escape_pattern(s) - return s:gsub("%-", "%%-"):gsub("%+", "%%+"):gsub("%?", "%%?"):gsub("%.", "%%.") -end - ---- Parse HTTP POST data --- --- @return Table with submitted form data -function M.request.parse_multipart(self) - local ct = self.headers['content-type'] - if ct == nil then - return nil, 'Content-Type header not present' - end - - if self.data == nil then - return nil, 'Empty body' - end - local body = self.data - local result ={} - - if ct:find('^multipart/form[-]data;') then - local boundary = ct:match('^multipart/form[-]data; boundary=["]?(.+)["]?$') - if boundary == nil then - return nil, 'Could not parse boundary from Content-Type' - end - - -- per RFC2046, CLRF is treated as a part of boundary - -- but first one does not have it, so we're going pretend - -- it is part of the content and ignore it there (in the pattern) - boundary = string.format('%%-%%-%s.-\r\n', escape_pattern(boundary)) - - local i = 1 - local j - local old_i - - while true do - i, j = body:find(boundary, i) - - if i == nil then break end - - if old_i then - local part = body:sub(old_i, i - 1) - local k, fn, t, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"; filename="(.+)"\r\n[cC]ontent[-][tT]ype: (.+)\r\n\r\n(.+)\r\n$') - - if k then - result[k] = { - filename = fn, - content_type = t, - data = v - } - else - k, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"\r\n\r\n(.+)\r\n$') - - if k then - result[k] = v - end - end - - end - - i = j + 1 - old_i = i - end - elseif ct == 'application/x-www-form-urlencoded' then - result = parse_kv(body, '&') - else - return nil, 'Unsupported Content-Type: ' .. ct - end - - if result == nil or not next(result) then - return nil, 'Could not parse form data' - end - - return result -end - ---- Reads (all) chunks from a HTTP response --- --- @param socket socket object (with already established tcp connection) --- @param get_all boolean (true by default), collect all chunks at once --- or yield every chunk separately. --- --- @return Full response payload or nil and an error message -function M.receive_chunked(socket, get_all) - if socket == nil then - return nil, "http.receive_chunked: Socket is nil" - end - local data = {} - - while true do - local chunk, err = socket:receive("*l") - - if chunk == nil then - return nil, "http.receive_chunked(): Receive error (chunk length): " .. tostring(err) - end - - local chunk_len = tonumber(chunk, 16) - if chunk_len == nil then - return nil, "http.receive_chunked(): Could not parse chunk length" - end - - if chunk_len == 0 then - -- TODO: support trailers - break - end - - -- Consume next chunk (including the \r\n) - chunk, err = socket:receive(chunk_len+2) - if chunk == nil then - return nil, "http.receive_chunked(): Receive error (chunk data): " .. tostring(err) - end - - -- Strip the \r\n before collection - local chunk_data = string.sub(chunk, 1, -3) - - if get_all == false then - return chunk_data - end - - table.insert(data, chunk_data) - end - - return table.concat(data) -end - - --- Request headers getter --- --- Returns a header value(s) according to strategy (fold by default): --- - single/string value for "fold", "first" and "last" strategies --- - table for "all" strategy (for single value, a table with single element) --- --- @param name string Header name --- @param strategy string "multiple header values" handling strategy --- @return header value (string or table) or nil -function M.request.get_header(self, name, strategy) - return get_header(self.headers, name, strategy) -end - --- Request headers iterator --- --- Yields key/value pairs for all headers, making sure that returned values --- are always of string type --- --- @param hdrs table Headers table as received by http.get and friends --- @param folded boolean Specifies whether to fold headers with same name --- @return header name/value iterator (suitable for use in "for" loops) -function M.request.get_headers(self, folded) - if folded == true then - return get_headers_folded(self.headers) - else - return get_headers_flattened(self.headers) - end -end - ---- Creates HTTP request from scratch --- --- @param method HTTP method --- @param url Valid HTTP url --- @param headers Lua table with request headers --- @param data Request content --- @param params Lua table with request url arguments --- @param auth (username, password) tuple for HTTP auth --- --- @return request object -function M.request.create(t) - local self = setmetatable({}, M.request) - - if t.method then - self.method = t.method:lower() - else - self.method = "get" - end - self.url = t.url or nil - self.headers = copyTable(t.headers) or {} - self.data = t.data or nil - self.params = copyTable(t.params) or {} - self.auth = copyTable(t.auth) or {} - - return self -end - ---- HTTP HEAD request -function M.head(t) - return M.send("HEAD", t) -end - ---- HTTP GET request -function M.get(t) - return M.send("GET", t) -end - ---- HTTP PUT request -function M.put(t) - return M.send("PUT", t) -end - ---- HTTP POST request -function M.post(t) - return M.send("POST", t) -end - ---- HTTP DELETE request -function M.delete(t) - return M.send("DELETE", t) -end - - ---- Send HTTP request --- --- @param method HTTP method --- @param url Valid HTTP url (mandatory) --- @param headers Lua table with request headers --- @param data Request content --- @param params Lua table with request url arguments --- @param auth (username, password) tuple for HTTP auth --- @param timeout Optional timeout for socket operations (5s by default) --- --- @return Response object or tuple (nil, msg) on errors - --- Note that the prefered way to call this method is via Lua --- "keyword arguments" convention, e.g. --- http.get{uri="http://example.net"} -function M.send(method, t) - if type(t) ~= "table" then - return nil, "http." .. method:lower() .. - ": expecting Request object for named parameters" - end - - if type(t.url) ~= "string" then - return nil, "http." .. method:lower() .. ": 'url' parameter missing" - end - - local socket = core.tcp() - socket:settimeout(t.timeout or 5) - local connect - if t.url:sub(1, 7) ~= "http://" and t.url:sub(1, 8) ~= "https://" then - t.url = "http://" .. t.url - end - local schema, host, req_uri = t.url:match("^(.*)://(.-)(/.*)$") - - if not schema then - -- maybe path (request uri) is missing - schema, host = t.url:match("^(.*)://(.-)$") - if not schema then - return nil, "http." .. method:lower() .. ": Could not parse URL: " .. t.url - end - req_uri = "/" - end - - local addr, port = host:match("(.*):(%d+)") - - if schema == "http" then - connect = socket.connect - if not port then - addr = host - port = 80 - end - elseif schema == "https" then - connect = socket.connect_ssl - if not port then - addr = host - port = 443 - end - else - return nil, "http." .. method:lower() .. ": Invalid URL schema " .. tostring(schema) - end - - local c, err = connect(socket, addr, port) - - if c then - local req = {} - local hdr_tbl = {} - - if t.headers then - for k, v in pairs(t.headers) do - if type(v) == "table" then - table.insert(hdr_tbl, k .. ": " .. table.concat(v, ",")) - else - table.insert(hdr_tbl, k .. ": " .. tostring(v)) - end - end - else - t.headers = {} -- dummy table - end - - if not t.headers.host then - -- 'Host' header must be provided for HTTP/1.1 - table.insert(hdr_tbl, "host: " .. host) - end - - if not t.headers["accept"] then - table.insert(hdr_tbl, "accept: */*") - end - - if not t.headers["user-agent"] then - table.insert(hdr_tbl, "user-agent: haproxy-lua-http/1.0") - end - - if not t.headers.connection then - table.insert(hdr_tbl, "connection: close") - end - - if t.data then - req[4] = t.data - if not t.headers or not t.headers["content-length"] then - table.insert(hdr_tbl, "content-length: " .. tostring(#t.data)) - end - end - - req[1] = method .. " " .. req_uri .. " HTTP/1.1\r\n" - req[2] = table.concat(hdr_tbl, "\r\n") - req[3] = "\r\n\r\n" - - local r, e = socket:send(table.concat(req)) - - if not r then - socket:close() - return nil, "http." .. method:lower() .. ": " .. tostring(e) - end - - local line - r = M.response.create() - - while true do - line, err = socket:receive("*l") - - if not line then - socket:close() - return nil, "http." .. method:lower() .. - ": Receive error (headers): " .. err - end - - if line == "" then break end - - if not r.status_code then - _, r.status_code, r.reason = - line:match("(HTTP/1.[01]) (%d%d%d)(.*)") - if not _ then - socket:close() - return nil, "http." .. method:lower() .. - ": Could not parse request line" - end - r.status_code = tonumber(r.status_code) - else - local sep = line:find(":") - local hdr_name = line:sub(1, sep-1):lower() - local hdr_val = line:sub(sep+1):match("^%s*(.*%S)%s*$") or "" - - if r.headers[hdr_name] == nil then - r.headers[hdr_name] = hdr_val - elseif type(r.headers[hdr_name]) == "table" then - table.insert(r.headers[hdr_name], hdr_val) - else - r.headers[hdr_name] = { - r.headers[hdr_name], - hdr_val - } - end - end - end - - if method:lower() == "head" then - r.content = nil - socket:close() - return r - end - - if r.headers["content-length"] and tonumber(r.headers["content-length"]) > 0 then - r.content, err = socket:receive("*a") - - if not r.content then - socket:close() - return nil, "http." .. method:lower() .. - ": Receive error (content): " .. err - end - end - - if r.headers["transfer-encoding"] and r.headers["transfer-encoding"] == "chunked" then - r.content, err = M.receive_chunked(socket) - if r.content == nil then - socket:close() - return nil, err - end - end - - socket:close() - return r - else - return nil, "http." .. method:lower() .. ": Connection error: " .. tostring(err) - end -end - -M.base64 = {} - ---- URL safe base64 encoder --- --- Padding ('=') is omited, as permited per RFC --- https://tools.ietf.org/html/rfc4648 --- in order to follow JSON Web Signature RFC --- https://tools.ietf.org/html/rfc7515 --- --- @param s String (can be binary data) to encode --- @param enc Function which implements base64 encoder (e.g. HAProxy base64 fetch) --- @return Encoded string -function M.base64.encode(s, enc) - if not s then return nil end - local u = enc(s) - - if not u then - return nil - end - - local pad_len = 2 - ((#s-1) % 3) - - if pad_len > 0 then - return u:sub(1, - pad_len - 1):gsub('[+]', '-'):gsub('[/]', '_') - else - return u:gsub('[+]', '-'):gsub('[/]', '_') - end -end - ---- URLsafe base64 decoder --- --- @param s Base64 string to decode --- @param dec Function which implements base64 decoder (e.g. HAProxy b64dec fetch) --- @return Decoded string (can be binary data) -function M.base64.decode(s, dec) - if not s then return nil end - - local e = s:gsub('[-]', '+'):gsub('[_]', '/') - return dec(e .. string.rep('=', 3 - ((#s - 1) % 4))) -end - -return M diff --git a/states/haproxy/templates/haproxy.cfg.j2 b/states/haproxy/templates/haproxy.cfg.j2 index 05ec94d..ff9d0b8 100644 --- a/states/haproxy/templates/haproxy.cfg.j2 +++ b/states/haproxy/templates/haproxy.cfg.j2 @@ -22,9 +22,9 @@ {%- macro httpsslrules() -%}ssl verify none{%- endmacro -%} -{%- macro httpendpoints(servers=[], check=True, ssl=False, disabled=False) -%} +{%- macro httpendpoints(servers=[], check=True, disabled=False) -%} {%- for server in servers %} - server {{ server.name }} {{ server.name }}:{{ server.port }}{{ " " + httpcheckrules(inter=server.inter|default("2s"), fall=server.fall|default(5), rise=server.rise|default(5)) if check }}{{ " " + httpsslrules() if ssl }}{{ " disabled" if server.disabled|default(False) }}{{ " send-proxy" if server.proxy|default(False) }} + server {{ server.name }} {{ server.name }}:{{ server.port }}{{ " " + httpcheckrules(inter=server.inter|default("2s"), fall=server.fall|default(5), rise=server.rise|default(5)) if check }}{{ " " + httpsslrules() if server.ssl|default(False) }}{{ " disabled" if server.disabled|default(False) }}{{ " send-proxy" if server.proxy|default(False) }} {%- endfor %} {%- endmacro -%} @@ -64,23 +64,29 @@ backend admin from {{ haproxy.config.namespace }} # Global config global + maxconn 1000 lua-prepend-path {{ haproxy.config.dir }}/mods/?.so cpath lua-prepend-path {{ haproxy.config.dir }}/scripts/?.lua {%- for file in haproxy.config.scripts %} {%- if not file.lib %} - lua-load {{ haproxy.config.dir }}/{{ file.name }} + lua-load {{ haproxy.config.dir }}/{{ file.name }} {% if "args" in file.keys() %}{{ file.args|join(" ")}}{% endif %} {%- endif %} {%- endfor %} {%- if haproxy.config.api.enable %} {{ api() }} {%- endif %} - maxconn 1000 + + crt-base {{ haproxy.config.acme_fullchains_dir }} + ssl-dh-param-file {{ haproxy.config.acme_dh_dir }}/dh.pem + ssl-default-bind-ciphers {{ haproxy.config.ssl_ciphers|join(":") }} ssl-default-bind-options {{ haproxy.config.ssl_options|join(" ") }} ssl-default-server-ciphers {{ haproxy.config.ssl_ciphers|join(":") }} ssl-default-server-options {{ haproxy.config.ssl_options|join(" ") }} - crt-base {{ haproxy.config.acme_fullchains_dir }} - ssl-dh-param-file {{ haproxy.config.acme_dh_dir }}/dh.pem + + tune.lua.maxmem {{ haproxy.config.lua_max_mem }} + + expose-experimental-directives # Defaults values defaults {{ haproxy.config.namespace }} @@ -122,11 +128,12 @@ frontend http from {{ haproxy.config.namespace }} acl path_root path / acl path_host path_dir /host acl path_date path_dir /date - acl path_srchash path -m dir /srchash + acl path_srchash path_dir /srchash ## Basic rules http-request set-var(txn.srchash) src,crc32,mod(100) http-request set-var(txn.httpdate) date,http_date() + http-request return status 200 content-type text/html lf-string "%H\n" if self_host path_host http-request return status 200 content-type text/html lf-string "%[var(txn.httpdate)]\n" if self_host path_date http-request return status 200 content-type text/html lf-string "%[var(txn.srchash)]\n" if self_host path_srchash @@ -147,7 +154,7 @@ frontend https from {{ haproxy.config.namespace }} ## ACLs acl internal src -f {{ haproxy.config.dir }}/maps/access - acl domains req.hdr(Host),map_dom({{ haproxy.config.dir }}/maps/domains) -m found req.hdr(host) -m str %H + acl domains req.hdr(Host),map_dom({{ haproxy.config.dir }}/maps/domains) -m found acl security_txt path /.well-known/security.txt acl robots_txt path /robots.txt acl max_req_rate sc_http_req_rate(0) gt {{ haproxy.config.ddos.maxrequests|default(200) }} @@ -156,6 +163,7 @@ frontend https from {{ haproxy.config.namespace }} acl path_host path_dir /host acl path_date path_dir /date acl path_srchash path /srchash + acl path_country path /country ## Basic rules http-request set-var(txn.random) rand,mul(5) @@ -164,19 +172,27 @@ frontend https from {{ haproxy.config.namespace }} http-request set-var(req.src) src http-request set-var(req.host) req.hdr(Host) http-request set-var(req.accesshash) str(),concat(,req.src,),concat(-,req.host,) + http-request track-sc0 var(req.accesshash) table per_ip_rates - http-request capture req.hdr(User-Agent) len 200 - http-request capture req.hdr(Content-Type) len 200 - http-request capture req.hdr(Referer) len 200 + http-request capture req.hdr(User-Agent) len 50 + http-request capture req.hdr(Content-Type) len 50 http-request capture sc_http_req_rate(0) len 4 ## DDoS http-request deny deny_status 429 if max_req_rate !internal +{%- if haproxy.config.geoip.enabled %} + # GeoIP + http-request set-var(txn.country) lua.country(req.src) + http-request return content-type text/plain lf-string "%[var(txn.country)]" if path_country + acl allowed_country var(txn.country),map_str(/etc/haproxy/maps/countries,OK) OK + http-response deny deny_status 451 content-type text/plain string "Denied" if !allowed_country !internal +{%- endif %} + ## Returns http-request return status 200 content-type text/plain string "User-agent: *\r\nAllow: /" if robots_txt - http-request return status 200 content-type text/plain string "Contact: mailto:paul@paulbsd.com" if security_txt + http-request return status 200 content-type text/plain string "Contact: mailto:{{ haproxy.config.syscontact }}" if security_txt http-request return status 200 content-type text/html lf-string "%H\n" if self_host path_host http-request return status 200 content-type text/html lf-string "%[var(txn.httpdate)]\n" if self_host path_date http-request return status 200 content-type text/html lf-string "%[var(txn.srchash)]\n" if self_host path_srchash @@ -189,17 +205,18 @@ frontend https from {{ haproxy.config.namespace }} http-response set-header Server "{{ haproxy.config.servername }}" http-response set-header X-Random "%[var(txn.random)]" - log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" - http-request redirect location %[req.hdr(Host),map_dom({{ haproxy.config.dir }}/maps/redirects)] code 301 if { req.hdr(Host),map_dom({{ haproxy.config.dir }}/maps/redirects) -m found } http-request deny deny_status 404 unless domains {%- if haproxy.config.admin %} use_backend admin if self_host internal {%- endif %} use_backend %[req.hdr(Host),lower,map({{ haproxy.config.dir }}/maps/vhosts,nginx)] - monitor-uri /dead_or_alive default_backend {{ ns.default_backend }} + monitor-uri /dead_or_alive + + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" + # HTTP Backends {%- for name, values in haproxy.config.vhosts.items() %} @@ -234,7 +251,7 @@ backend {{ name }} from {{ haproxy.config.namespace }} {%- if values.internal|default(False) %} {{ internal() }} {%- endif %} - {{- httpendpoints(servers=values.servers, check=values.check|default(haproxy.config.check), ssl=values.ssl|default(False)) }} + {{- httpendpoints(servers=values.servers, check=values.check|default(haproxy.config.check)) }} {%- endif %} {% endfor %} @@ -268,5 +285,5 @@ backend {{ name }} from {{ haproxy.config.namespace }} {% endfor %} {%- if haproxy.config.admin %} -{{ admin() }} +{{ admin() if haproxy.config.admin }} {%- endif %} diff --git a/states/haproxy/templates/maps/countries.j2 b/states/haproxy/templates/maps/countries.j2 new file mode 100644 index 0000000..dcabd5a --- /dev/null +++ b/states/haproxy/templates/maps/countries.j2 @@ -0,0 +1,5 @@ +## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} +{%- from "haproxy/map.jinja" import haproxy with context %} +{%- for name, status in haproxy.config.geoip.countries.items() %} +{{ name }} {{ status }} +{%- endfor %}