updated haproxy state

states/haproxy/config.sls

@@ -10,14 +10,3 @@ haproxy-config:
     - template: jinja
     - watch_in:
       - service: haproxy-service
-
-haproxy-config-access:
-  file.managed:
-    - name: {{ haproxy.config.dir }}/{{ haproxy.config.accessfile }}
-    - source: salt://haproxy/templates/access.j2
-    - user: {{ haproxy.config.user }}
-    - group: {{ haproxy.config.group }}
-    - mode: "0600"
-    - template: jinja
-    - watch_in:
-      - service: haproxy-service

states/haproxy/defaults.yaml

@@ -4,29 +4,42 @@ haproxy:
   packages:
     - haproxy
   scripts:
-    - hello_world.lua
+    - name: scripts/http.lua
+      lib: true
+    - name: scripts/json.lua
+      lib: true
+    - name: scripts/collector.lua
+      lib: false
+    - name: scripts/weight_by_latency.lua
+      lib: false
+  maps:
+    - access
   config:
+    servername: "PaulBSD WebServer 1.0"
+    http2: true
     defaults:
-      #log global:
+      #log: global
-      #log 127.0.0.1 local0:
+      #log: 127.0.0.1 local0
-      log stdout format raw daemon info:
+      log: stdout format raw daemon info
-      mode http:
+      retries: 2
-      option httplog:
+      timeout client: 30m
-      option forwardfor:
+      timeout connect: 4s
-      retries 2:
+      timeout server: 30m
-      timeout client 30m:
+      timeout check: 5s
-      timeout connect 4s:
-      timeout server 30m:
-      timeout check 5s:
     dir: /etc/haproxy
     configfile: haproxy.cfg
-    accessfile: access
     user: haproxy
     group: haproxy
     http_port: 80
     https_port: 443
     admin: false
+    api:
+      enable: true
+      filesocket: /var/run/hap-lb.sock
+      tcpsocket: ipv4@127.0.0.1:9990
     acme_dir: /etc/acme
+    acme_fullchains_dir: /etc/acme/fullchains
+    acme_dh_dir: /etc/acme/dh
     ssl_ciphers:
       - "ECDH+AESGCM"
       - "DH+AESGCM"

states/haproxy/install.sls

@@ -4,6 +4,13 @@ haproxy-pkg:
   pkg.installed:
     - pkgs: {{ haproxy.packages }}
 
+haproxy-maps-dir:
+  file.directory:
+    - name: {{ haproxy.config.dir }}/maps
+    - user: {{ haproxy.config.user }}
+    - group: {{ haproxy.config.group }}
+    - mode: "0700"
+
 haproxy-script-dir:
   file.directory:
     - name: {{ haproxy.config.dir }}/scripts
@@ -11,14 +18,27 @@ haproxy-script-dir:
     - group: {{ haproxy.config.group }}
     - mode: "0700"
 
-{% for filename in haproxy.scripts %}
+{% for file in haproxy.scripts %}
-haprox-script-{{ filename }}:
+haproxy-script-{{ file.name }}:
   file.managed:
-    - name: {{ haproxy.config.dir }}/scripts/{{ filename }}
+    - name: {{ haproxy.config.dir }}/{{ file.name }}
-    - source: salt://haproxy/scripts/{{ filename }}
+    - source: salt://haproxy/{{ file.name }}
     - user: {{ haproxy.config.user }}
     - group: {{ haproxy.config.group }}
     - mode: "0700"
     - watch_in:
       - service: haproxy-service
+{% endfor %}
+
+{% for filename in haproxy.maps %}
+haproxy-maps-{{ filename }}:
+  file.managed:
+    - name: {{ haproxy.config.dir }}/maps/{{ filename }}
+    - source: salt://haproxy/templates/{{ filename }}.j2
+    - user: {{ haproxy.config.user }}
+    - group: {{ haproxy.config.group }}
+    - mode: "0600"
+    - template: jinja
+    - watch_in:
+      - service: haproxy-service
 {% endfor %}

states/haproxy/map.jinja

@@ -2,7 +2,5 @@
 
 {%- set haproxy = salt['pillar.get']('haproxy', default=defaults.haproxy, merge=True) -%}
 
-{%- set certs = salt['file.find'](haproxy.config.acme_dir, type='f', name='*.full')%}
-
 {%- set users = salt['pillar.get']('htpasswds') -%}
 {%- set net = salt['pillar.get']('net') -%}

states/haproxy/scripts/collector.lua

@@ -0,0 +1,36 @@
+--require = GLOBAL.require
+--require("http.request")
+local request = require("http")
+local json = require("json")
+
+core.register_action("test", { "http-req" }, function(txn)
+  local info = {}
+  local headers = {}
+
+  local reqbody = txn.sf:req_body()
+  local reqclientip = txn.f:src()
+  local reqheaders = txn.http:req_get_headers()
+  local reqmethod = txn.f:method()
+  local reqpath = txn.f:path()
+
+  info["body"] = reqbody
+  info["clientip"] = reqclientip
+  local headers = {}
+  for k,v in pairs(reqheaders) do
+    headers[k] = {}
+    for z,y in pairs(v) do
+      table.insert(headers[k],y)
+    end
+  end
+  info["headers"] = headers
+  info["method"] = reqmethod
+  info["path"] = reqpath
+
+  local infojson = json.encode(info)
+  --local req = request.new_from_uri("https://ipbl.paulbsd.com")
+  --local headers, stream = req:go()
+  --local body = assert(stream:get_body_as_string())
+
+  txn.Info(txn, infojson)
+end
+)

states/haproxy/scripts/http.lua

@@ -0,0 +1,830 @@
+--
+-- HTTP 1.1 library for HAProxy Lua modules
+--
+-- The library is loosely modeled after Python's Requests Library
+-- using the same field names and very similar calling conventions for
+-- "HTTP verb" methods (where we use Lua specific named parameter support)
+--
+-- In addition to client side, the library also supports server side request
+-- parsing, where we utilize HAProxy Lua API for all heavy lifting.
+--
+--
+-- Copyright (c) 2017-2020. Adis Nezirović <anezirovic@haproxy.com>
+-- Copyright (c) 2017-2020. HAProxy Technologies, LLC.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--    http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- SPDX-License-Identifier: Apache-2.0
+
+local _author = "Adis Nezirovic <anezirovic@haproxy.com>"
+local _copyright = "Copyright 2017-2020. HAProxy Technologies, LLC."
+local _version = "1.0.0"
+
+local json = require "json"
+
+-- Utility functions
+
+-- HTTP headers fetch helper
+--
+-- Returns a header value(s) according to strategy (fold by default):
+--  - single/string value for "fold", "first" and "last" strategies
+--  - table for "all" strategy (for single value, a table with single element)
+--
+-- @param hdrs table Headers table as received by http.get and friends
+-- @param name string Header name
+-- @param strategy string "multiple header values" handling strategy
+-- @return header value (string or table) or nil
+local function get_header(hdrs, name, strategy)
+    if hdrs == nil or name == nil then return nil end
+
+    local v = hdrs[name:lower()]
+    if type(v) ~= "table" and strategy ~= "all" then return v end
+
+    if strategy == nil or strategy == "fold" then
+        return table.concat(v, ",")
+    elseif strategy == "first" then
+        return v[1]
+    elseif strategy == "last" then
+        return v[#v]
+    elseif strategy == "all" then
+        if type(v) ~= "table" then
+            return {v}
+        else
+            return v
+        end
+    end
+end
+
+
+
+-- HTTP headers iterator helper
+--
+-- Returns key/value pairs for all header, making sure that returned values
+-- are always of string type (if necessary, it folds multiple headers with
+-- the same name)
+--
+-- @param hdrs table Headers table as received by http.get and friends
+-- @return header name/value iterator (suitable for use in "for" loops)
+local function get_headers_folded(hdrs)
+    if hdrs == nil then
+        return function() end
+    end
+
+    local function iter(t, k)
+        local v
+        k, v = next(t, k)
+
+        if v ~= nil then
+            if type(v) ~= "table" then
+                return k, v
+            else
+                return k, table.concat(v, ",")
+            end
+        end
+    end
+
+    return iter, hdrs, nil
+end
+
+-- HTTP headers iterator
+--
+-- Returns key/value pairs for all headers, for multiple headers with same name
+-- it will return every name/value pair
+-- (i.e. you can safely use it to process responses with 'Set-Cookie' header)
+--
+-- @param hdrs table Headers table as received by http.get and friends
+-- @return header name/value iterator (suitable for use in "for" loops)
+local function get_headers_flattened(hdrs)
+    if hdrs == nil then
+        return function() end
+    end
+
+    local k  -- top level key (string)
+    local k_sub = 0  -- sub table key (integer), 0 if item not a table,
+                     -- nil after last sub table iteration
+    local v_sub  -- sub table
+
+    return function ()
+        local v
+        if k_sub == 0 then
+            k, v = next(hdrs, k)
+            if k == nil then return end
+        else
+            k_sub, v = next(v_sub, k_sub)
+
+            if k_sub == nil then
+                k_sub = 0
+                k, v = next(hdrs, k)
+            end
+        end
+
+        if k == nil then return end
+
+        if type(v) ~= "table" then
+            return k, v
+        else
+            v_sub = v
+            k_sub = k_sub + 1
+            return k, v[k_sub]
+        end
+    end
+end
+
+
+--- Parse key/value pairs from a string
+--
+-- @param s Lua string with (multiple) key/value pairs (separated by 'sep')
+--
+-- @return Table with parsed keys and values or nil
+local function parse_kv(s, sep)
+    if s == nil then return nil end
+    idx = 1
+    result = {}
+
+    while idx < s:len() do
+        i, j = s:find(sep, idx)
+
+        if i == nil then
+            k, v = string.match(s:sub(idx), "^(.-)=(.*)$")
+            if k then result[k] = v end
+            break
+        end
+
+        k, v = string.match(s:sub(idx, i-1), "^(.-)=(.*)$")
+        if k then result[k] = v end
+        idx = j + 1
+    end
+
+    if next(result) == nil then
+        return nil
+    else
+        return result
+    end
+end
+
+
+--- Make deep copy of table and it's values
+--
+-- Use only for simple tables (it handles nested table values), but  not for
+-- Lua objects or similar, or very big tables (this uses recursion).
+--
+-- @param t Cloned Lua table or nil
+--
+-- @return Cloned table or nil
+local function copyTable(t)
+    if type(t) ~= "table" then
+        return nil
+    end
+
+    local r = {}
+
+    for k, v in pairs(t) do
+        if type(v) == "table" then
+            r[k] = copyTable(v)
+        else
+            r[k] = v
+        end
+    end
+
+    return r
+end
+
+
+--- Namespace object which hosts HTTP verb methods and request/response classes
+local M = {}
+
+
+--- HTTP response class
+M.response = {}
+M.response.__index = M.response
+
+local _reason = {
+    [200] = "OK",
+    [201] = "Created",
+    [204] = "No Content",
+    [301] = "Moved Permanently",
+    [302] = "Found",
+    [400] = "Bad Request",
+    [403] = "Forbidden",
+    [404] = "Not Found",
+    [405] = "Method Not Allowed",
+    [408] = "Request Timeout",
+    [413] = "Payload Too Large",
+    [429] = "Too many requests",
+    [500] = "Internal Server Error",
+    [501] = "Not Implemented",
+    [502] = "Bad Gateway",
+    [503] = "Service Unavailable",
+    [504] = "Gateway Timeout"
+}
+
+--- Creates HTTP response from scratch
+--
+-- @param status_code HTTP status code
+-- @param reason HTTP status code text (e.g. "OK" for 200 response)
+-- @param headers HTTP response headers
+-- @param request The HTTP request which triggered the response
+-- @param encoding Default encoding for response or conversions
+--
+-- @return response object
+function M.response.create(t)
+    local self = setmetatable({}, M.response)
+
+    if not t then
+        t =  {}
+    end
+
+    self.status_code = t.status_code or nil
+    self.reason = t.reason or _reason[self.status_code] or ""
+    self.headers = copyTable(t.headers) or {}
+    self.content = t.content or ""
+    self.request = t.request or nil
+    self.encoding = t.encoding or "utf-8"
+
+    return self
+end
+
+function M.response.send(self, applet)
+    applet:set_status(tonumber(self.status_code), self.reason)
+
+    for k, v in pairs(self.headers) do
+        if type(v) == "table" then
+            for _, hdr_val in pairs(v) do
+                applet:add_header(k, hdr_val)
+            end
+        else
+            applet:add_header(k, v)
+        end
+    end
+
+    if not self.headers["content-type"] then
+        if type(self.content) == "table" then
+            applet:add_header("content-type", "application/json; charset=" ..
+                              self.encoding)
+            if next(self.content) == nil then
+                -- Return empty JSON object for empty Lua tables
+                -- (that makes more sense then returning [])
+                self.content = "{}"
+            else
+                self.content = json.encode(self.content)
+            end
+        else
+            applet:add_header("content-type", "text/plain; charset=" ..
+                              self.encoding)
+        end
+    end
+
+    if not self.headers["content-length"] then
+        applet:add_header("content-length", #tostring(self.content))
+    end
+
+    applet:start_response()
+    applet:send(tostring(self.content))
+end
+
+--- Convert response content to JSON
+--
+-- @return Lua table (decoded json)
+function M.response.json(self)
+    return json.decode(self.content)
+end
+
+-- Response headers getter
+--
+-- Returns a header value(s) according to strategy (fold by default):
+--  - single/string value for "fold", "first" and "last" strategies
+--  - table for "all" strategy (for single value, a table with single element)
+--
+-- @param name string Header name
+-- @param strategy string "multiple header values" handling strategy
+-- @return header value (string or table) or nil
+function M.response.get_header(self, name, strategy)
+    return get_header(self.headers, name, strategy)
+end
+
+-- Response headers iterator
+--
+-- Yields key/value pairs for all headers, making sure that returned values
+-- are always of string type
+--
+-- @param folded boolean Specifies whether to fold headers with same name
+-- @return header name/value iterator (suitable for use in "for" loops)
+function M.response.get_headers(self, folded)
+    if folded == true then
+        return get_headers_folded(self.headers)
+    else
+        return get_headers_flattened(self.headers)
+    end
+end
+
+
+--- HTTP request class (client or server side, depending on the constructor)
+M.request = {}
+M.request.__index = M.request
+
+--- HTTP request constructor
+--
+-- Parses client HTTP request (as forwarded by HAProxy)
+--
+-- @param applet HAProxy AppletHTTP Lua object
+--
+-- @return Request object
+function M.request.parse(applet)
+    local self = setmetatable({}, M.request)
+    self.method = applet.method
+
+    if (applet.method == "POST" or applet.method == "PUT") and
+            applet.length > 0 then
+        self.data = applet:receive()
+        if self.data == "" then self.data = nil end
+    end
+
+    self.headers = {}
+    for k, v in pairs(applet.headers) do
+        if (v[1]) then  -- (non folded header with multiple values)
+            self.headers[k] = {}
+            for _, val in pairs(v) do
+                table.insert(self.headers[k], val)
+            end
+        else
+            self.headers[k] = v[0]
+        end
+    end
+
+    if not self.headers["host"] then
+        return nil, "Bad request, no Host header specified"
+    end
+
+    self.cookies = parse_kv(self.headers["cookie"], "; ")
+
+    -- TODO: Patch ApletHTTP and add schema of request
+    local schema = applet.schema or "http"
+    local url = {schema, "://", self.headers["host"], applet.path}
+
+    self.params = {}
+    if applet.qs:len() > 0 then
+        for _, arg in ipairs(core.tokenize(applet.qs, "&", true)) do
+            kv = core.tokenize(arg, "=", true)
+            self.params[kv[1]] = kv[2]
+        end
+        url[#url+1] = "?"
+        url[#url+1] = applet.qs
+    end
+
+    self.url = table.concat(url)
+
+    return self
+end
+
+--- Escape Lua pattern chars in HTTP multipart boundary
+--
+-- This function escapes only minimal number of characters, which can be
+-- observed in multipart boundaries, namely: -, +, ? and .
+--
+-- @param s string Data to escape
+--
+-- @return escaped data (string)
+local function escape_pattern(s)
+    return s:gsub("%-", "%%-"):gsub("%+", "%%+"):gsub("%?", "%%?"):gsub("%.", "%%.")
+end
+
+--- Parse HTTP POST data
+--
+-- @return Table with submitted form data
+function M.request.parse_multipart(self)
+    local ct = self.headers['content-type']
+    if ct == nil then
+        return nil, 'Content-Type header not present'
+    end
+
+    if self.data == nil then
+        return nil, 'Empty body'
+    end
+    local body = self.data
+    local result ={}
+
+    if ct:find('^multipart/form[-]data;') then
+        local boundary = ct:match('^multipart/form[-]data; boundary=["]?(.+)["]?$')
+        if boundary == nil then
+            return nil, 'Could not parse boundary from Content-Type'
+        end
+
+        -- per RFC2046, CLRF is treated as a part of boundary
+        -- but first one does not have it, so we're going pretend
+        -- it is part of the content and ignore it there (in the pattern)
+        boundary = string.format('%%-%%-%s.-\r\n', escape_pattern(boundary))
+
+        local i = 1
+        local j
+        local old_i
+
+        while true do
+            i, j = body:find(boundary, i)
+
+            if i == nil then break end
+
+            if old_i then
+                local part = body:sub(old_i, i - 1)
+                local k, fn, t, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"; filename="(.+)"\r\n[cC]ontent[-][tT]ype: (.+)\r\n\r\n(.+)\r\n$')
+
+                if k then
+                    result[k] = {
+                        filename = fn,
+                        content_type = t,
+                        data = v
+                    }
+                else
+                    k, v = part:match('^[cC]ontent[-][dD]isposition: form[-]data; name[=]"(.+)"\r\n\r\n(.+)\r\n$')
+
+                    if k then
+                        result[k] = v
+                    end
+                end
+
+            end
+
+            i = j + 1
+            old_i = i
+        end
+    elseif ct == 'application/x-www-form-urlencoded' then
+        result = parse_kv(body, '&')
+    else
+        return nil, 'Unsupported Content-Type: ' .. ct
+    end
+
+    if result == nil or not next(result) then
+        return nil, 'Could not parse form data'
+    end
+
+    return result
+end
+
+--- Reads (all) chunks from a HTTP response
+--
+-- @param socket socket object (with already established tcp connection)
+-- @param get_all boolean (true by default), collect all chunks at once
+--                or yield every chunk separately.
+--
+-- @return Full response payload or nil and an error message
+function M.receive_chunked(socket, get_all)
+    if socket == nil then
+        return nil, "http.receive_chunked: Socket is nil"
+    end
+    local data = {}
+
+    while true do
+        local chunk, err = socket:receive("*l")
+
+        if chunk == nil then
+            return nil, "http.receive_chunked(): Receive error (chunk length): " .. tostring(err)
+        end
+
+        local chunk_len = tonumber(chunk, 16)
+        if chunk_len == nil then
+            return nil, "http.receive_chunked(): Could not parse chunk length"
+        end
+
+        if chunk_len == 0 then
+            -- TODO: support trailers
+            break
+        end
+
+        -- Consume next chunk (including the \r\n)
+        chunk, err = socket:receive(chunk_len+2)
+        if chunk == nil then
+            return nil, "http.receive_chunked(): Receive error (chunk data): " .. tostring(err)
+        end
+
+        -- Strip the \r\n before collection
+        local chunk_data = string.sub(chunk, 1, -3)
+
+        if get_all == false then
+            return chunk_data
+        end
+
+        table.insert(data, chunk_data)
+    end
+
+    return table.concat(data)
+end
+
+
+-- Request headers getter
+--
+-- Returns a header value(s) according to strategy (fold by default):
+--  - single/string value for "fold", "first" and "last" strategies
+--  - table for "all" strategy (for single value, a table with single element)
+--
+-- @param name string Header name
+-- @param strategy string "multiple header values" handling strategy
+-- @return header value (string or table) or nil
+function M.request.get_header(self, name, strategy)
+    return get_header(self.headers, name, strategy)
+end
+
+-- Request headers iterator
+--
+-- Yields key/value pairs for all headers, making sure that returned values
+-- are always of string type
+--
+-- @param hdrs table Headers table as received by http.get and friends
+-- @param folded boolean Specifies whether to fold headers with same name
+-- @return header name/value iterator (suitable for use in "for" loops)
+function M.request.get_headers(self, folded)
+    if folded == true then
+        return get_headers_folded(self.headers)
+    else
+        return get_headers_flattened(self.headers)
+    end
+end
+
+--- Creates HTTP request from scratch
+--
+-- @param method HTTP method
+-- @param url Valid HTTP url
+-- @param headers Lua table with request headers
+-- @param data Request content
+-- @param params Lua table with request url arguments
+-- @param auth (username, password) tuple for HTTP auth
+--
+-- @return request object
+function M.request.create(t)
+    local self = setmetatable({}, M.request)
+
+    if t.method then
+        self.method = t.method:lower()
+    else
+        self.method = "get"
+    end
+    self.url = t.url or nil
+    self.headers = copyTable(t.headers) or {}
+    self.data = t.data or nil
+    self.params = copyTable(t.params) or {}
+    self.auth = copyTable(t.auth) or {}
+
+    return self
+end
+
+--- HTTP HEAD request
+function M.head(t)
+    return M.send("HEAD", t)
+end
+
+--- HTTP GET request
+function M.get(t)
+    return M.send("GET", t)
+end
+
+--- HTTP PUT request
+function M.put(t)
+    return M.send("PUT", t)
+end
+
+--- HTTP POST request
+function M.post(t)
+    return M.send("POST", t)
+end
+
+--- HTTP DELETE request
+function M.delete(t)
+    return M.send("DELETE", t)
+end
+
+
+--- Send HTTP request
+--
+-- @param method HTTP method
+-- @param url Valid HTTP url (mandatory)
+-- @param headers Lua table with request headers
+-- @param data Request content
+-- @param params Lua table with request url arguments
+-- @param auth (username, password) tuple for HTTP auth
+-- @param timeout Optional timeout for socket operations (5s by default)
+--
+-- @return Response object or tuple (nil, msg) on errors
+
+-- Note that the prefered way to call this method is via Lua
+-- "keyword arguments" convention, e.g.
+--   http.get{uri="http://example.net"}
+function M.send(method, t)
+    if type(t) ~= "table" then
+        return nil, "http." .. method:lower() ..
+            ": expecting Request object for named parameters"
+    end
+
+    if type(t.url) ~= "string" then
+        return nil, "http." .. method:lower() .. ": 'url' parameter missing"
+    end
+
+    local socket = core.tcp()
+    socket:settimeout(t.timeout or 5)
+    local connect
+    if t.url:sub(1, 7) ~= "http://" and t.url:sub(1, 8) ~= "https://" then
+        t.url = "http://" .. t.url
+    end
+    local schema, host, req_uri = t.url:match("^(.*)://(.-)(/.*)$")
+
+    if not schema then
+        -- maybe path (request uri) is missing
+        schema, host = t.url:match("^(.*)://(.-)$")
+        if not schema then
+            return nil, "http." .. method:lower() .. ": Could not parse URL: " .. t.url
+        end
+        req_uri = "/"
+    end
+
+    local addr, port = host:match("(.*):(%d+)")
+
+    if schema == "http" then
+        connect = socket.connect
+        if not port then
+            addr = host
+            port = 80
+        end
+    elseif schema == "https" then
+        connect = socket.connect_ssl
+        if not port then
+            addr = host
+            port = 443
+        end
+    else
+        return nil, "http." .. method:lower() .. ": Invalid URL schema " .. tostring(schema)
+    end
+
+    local c, err = connect(socket, addr, port)
+
+    if c then
+        local req = {}
+        local hdr_tbl = {}
+
+        if t.headers then
+            for k, v in pairs(t.headers) do
+                if type(v) == "table" then
+                    table.insert(hdr_tbl, k .. ": " .. table.concat(v, ","))
+                else
+                    table.insert(hdr_tbl, k .. ": " .. tostring(v))
+                end
+            end
+        else
+            t.headers = {}  -- dummy table
+        end
+
+        if not t.headers.host then
+            -- 'Host' header must be provided for HTTP/1.1
+            table.insert(hdr_tbl, "host: " .. host)
+        end
+
+        if not t.headers["accept"] then
+            table.insert(hdr_tbl, "accept: */*")
+        end
+
+        if not t.headers["user-agent"] then
+            table.insert(hdr_tbl, "user-agent: haproxy-lua-http/1.0")
+        end
+
+        if not t.headers.connection then
+            table.insert(hdr_tbl, "connection: close")
+        end
+
+        if t.data then
+            req[4] = t.data
+            if not t.headers or not t.headers["content-length"] then
+                table.insert(hdr_tbl, "content-length: " .. tostring(#t.data))
+            end
+        end
+
+        req[1] = method .. " " .. req_uri .. " HTTP/1.1\r\n"
+        req[2] = table.concat(hdr_tbl, "\r\n")
+        req[3] = "\r\n\r\n"
+
+        local r, e = socket:send(table.concat(req))
+
+        if not r then
+            socket:close()
+            return nil, "http." .. method:lower() .. ": " .. tostring(e)
+        end
+
+        local line
+        r = M.response.create()
+
+        while true do
+            line, err = socket:receive("*l")
+
+            if not line then
+                socket:close()
+                return nil, "http." .. method:lower() ..
+                       ": Receive error (headers): "  .. err
+            end
+
+            if line == "" then break end
+
+            if not r.status_code then
+                _, r.status_code, r.reason =
+                    line:match("(HTTP/1.[01]) (%d%d%d)(.*)")
+                if not _ then
+                    socket:close()
+                    return nil, "http." .. method:lower() ..
+                        ": Could not parse request line"
+                end
+                r.status_code = tonumber(r.status_code)
+            else
+                local sep = line:find(":")
+                local hdr_name = line:sub(1, sep-1):lower()
+                local hdr_val = line:sub(sep+1):match("^%s*(.*%S)%s*$") or ""
+
+                if r.headers[hdr_name] == nil then
+                    r.headers[hdr_name] = hdr_val
+                elseif type(r.headers[hdr_name]) == "table" then
+                    table.insert(r.headers[hdr_name], hdr_val)
+                else
+                    r.headers[hdr_name] = {
+                        r.headers[hdr_name],
+                        hdr_val
+                    }
+                end
+            end
+        end
+
+        if method:lower() == "head" then
+            r.content = nil
+            socket:close()
+            return r
+        end
+
+        if r.headers["content-length"] and tonumber(r.headers["content-length"]) > 0 then
+            r.content, err = socket:receive("*a")
+
+            if not r.content then
+                socket:close()
+                return nil, "http." .. method:lower() ..
+                       ": Receive error (content): " .. err
+            end
+        end
+
+        if r.headers["transfer-encoding"] and r.headers["transfer-encoding"] == "chunked" then
+            r.content, err = M.receive_chunked(socket)
+            if r.content == nil then
+                socket:close()
+                return nil, err
+            end
+        end
+
+        socket:close()
+        return r
+    else
+        return nil, "http." .. method:lower() .. ": Connection error: " .. tostring(err)
+    end
+end
+
+M.base64 = {}
+
+--- URL safe base64 encoder
+--
+-- Padding ('=') is omited, as permited per RFC
+--   https://tools.ietf.org/html/rfc4648
+-- in order to follow JSON Web Signature RFC
+--   https://tools.ietf.org/html/rfc7515
+--
+-- @param s String (can be binary data) to encode
+-- @param enc Function which implements base64 encoder (e.g. HAProxy base64 fetch)
+-- @return Encoded string
+function M.base64.encode(s, enc)
+    if not s then return nil end
+    local u = enc(s)
+
+    if not u then
+        return nil
+    end
+
+    local pad_len = 2 - ((#s-1) % 3)
+
+    if pad_len > 0 then
+        return u:sub(1, - pad_len - 1):gsub('[+]', '-'):gsub('[/]', '_')
+    else
+        return u:gsub('[+]', '-'):gsub('[/]', '_')
+    end
+end
+
+--- URLsafe base64 decoder
+--
+-- @param s Base64 string to decode
+-- @param dec Function which implements base64 decoder (e.g. HAProxy b64dec fetch)
+-- @return Decoded string (can be binary data)
+function M.base64.decode(s, dec)
+    if not s then return nil end
+
+    local e = s:gsub('[-]', '+'):gsub('[_]', '/')
+    return dec(e .. string.rep('=', 3 - ((#s - 1) % 4)))
+end
+
+return M

states/haproxy/scripts/json.lua

@@ -0,0 +1,388 @@
+--
+-- json.lua
+--
+-- Copyright (c) 2020 rxi
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a copy of
+-- this software and associated documentation files (the "Software"), to deal in
+-- the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+-- of the Software, and to permit persons to whom the Software is furnished to do
+-- so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+-- SOFTWARE.
+--
+
+local json = { _version = "0.1.2" }
+
+-------------------------------------------------------------------------------
+-- Encode
+-------------------------------------------------------------------------------
+
+local encode
+
+local escape_char_map = {
+  [ "\\" ] = "\\",
+  [ "\"" ] = "\"",
+  [ "\b" ] = "b",
+  [ "\f" ] = "f",
+  [ "\n" ] = "n",
+  [ "\r" ] = "r",
+  [ "\t" ] = "t",
+}
+
+local escape_char_map_inv = { [ "/" ] = "/" }
+for k, v in pairs(escape_char_map) do
+  escape_char_map_inv[v] = k
+end
+
+
+local function escape_char(c)
+  return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte()))
+end
+
+
+local function encode_nil(val)
+  return "null"
+end
+
+
+local function encode_table(val, stack)
+  local res = {}
+  stack = stack or {}
+
+  -- Circular reference?
+  if stack[val] then error("circular reference") end
+
+  stack[val] = true
+
+  if rawget(val, 1) ~= nil or next(val) == nil then
+    -- Treat as array -- check keys are valid and it is not sparse
+    local n = 0
+    for k in pairs(val) do
+      if type(k) ~= "number" then
+        error("invalid table: mixed or invalid key types")
+      end
+      n = n + 1
+    end
+    if n ~= #val then
+      error("invalid table: sparse array")
+    end
+    -- Encode
+    for i, v in ipairs(val) do
+      table.insert(res, encode(v, stack))
+    end
+    stack[val] = nil
+    return "[" .. table.concat(res, ",") .. "]"
+
+  else
+    -- Treat as an object
+    for k, v in pairs(val) do
+      if type(k) ~= "string" then
+        error("invalid table: mixed or invalid key types")
+      end
+      table.insert(res, encode(k, stack) .. ":" .. encode(v, stack))
+    end
+    stack[val] = nil
+    return "{" .. table.concat(res, ",") .. "}"
+  end
+end
+
+
+local function encode_string(val)
+  return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"'
+end
+
+
+local function encode_number(val)
+  -- Check for NaN, -inf and inf
+  if val ~= val or val <= -math.huge or val >= math.huge then
+    error("unexpected number value '" .. tostring(val) .. "'")
+  end
+  return string.format("%.14g", val)
+end
+
+
+local type_func_map = {
+  [ "nil"     ] = encode_nil,
+  [ "table"   ] = encode_table,
+  [ "string"  ] = encode_string,
+  [ "number"  ] = encode_number,
+  [ "boolean" ] = tostring,
+}
+
+
+encode = function(val, stack)
+  local t = type(val)
+  local f = type_func_map[t]
+  if f then
+    return f(val, stack)
+  end
+  error("unexpected type '" .. t .. "'")
+end
+
+
+function json.encode(val)
+  return ( encode(val) )
+end
+
+
+-------------------------------------------------------------------------------
+-- Decode
+-------------------------------------------------------------------------------
+
+local parse
+
+local function create_set(...)
+  local res = {}
+  for i = 1, select("#", ...) do
+    res[ select(i, ...) ] = true
+  end
+  return res
+end
+
+local space_chars   = create_set(" ", "\t", "\r", "\n")
+local delim_chars   = create_set(" ", "\t", "\r", "\n", "]", "}", ",")
+local escape_chars  = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u")
+local literals      = create_set("true", "false", "null")
+
+local literal_map = {
+  [ "true"  ] = true,
+  [ "false" ] = false,
+  [ "null"  ] = nil,
+}
+
+
+local function next_char(str, idx, set, negate)
+  for i = idx, #str do
+    if set[str:sub(i, i)] ~= negate then
+      return i
+    end
+  end
+  return #str + 1
+end
+
+
+local function decode_error(str, idx, msg)
+  local line_count = 1
+  local col_count = 1
+  for i = 1, idx - 1 do
+    col_count = col_count + 1
+    if str:sub(i, i) == "\n" then
+      line_count = line_count + 1
+      col_count = 1
+    end
+  end
+  error( string.format("%s at line %d col %d", msg, line_count, col_count) )
+end
+
+
+local function codepoint_to_utf8(n)
+  -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa
+  local f = math.floor
+  if n <= 0x7f then
+    return string.char(n)
+  elseif n <= 0x7ff then
+    return string.char(f(n / 64) + 192, n % 64 + 128)
+  elseif n <= 0xffff then
+    return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128)
+  elseif n <= 0x10ffff then
+    return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128,
+                       f(n % 4096 / 64) + 128, n % 64 + 128)
+  end
+  error( string.format("invalid unicode codepoint '%x'", n) )
+end
+
+
+local function parse_unicode_escape(s)
+  local n1 = tonumber( s:sub(1, 4),  16 )
+  local n2 = tonumber( s:sub(7, 10), 16 )
+   -- Surrogate pair?
+  if n2 then
+    return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000)
+  else
+    return codepoint_to_utf8(n1)
+  end
+end
+
+
+local function parse_string(str, i)
+  local res = ""
+  local j = i + 1
+  local k = j
+
+  while j <= #str do
+    local x = str:byte(j)
+
+    if x < 32 then
+      decode_error(str, j, "control character in string")
+
+    elseif x == 92 then -- `\`: Escape
+      res = res .. str:sub(k, j - 1)
+      j = j + 1
+      local c = str:sub(j, j)
+      if c == "u" then
+        local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1)
+                 or str:match("^%x%x%x%x", j + 1)
+                 or decode_error(str, j - 1, "invalid unicode escape in string")
+        res = res .. parse_unicode_escape(hex)
+        j = j + #hex
+      else
+        if not escape_chars[c] then
+          decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string")
+        end
+        res = res .. escape_char_map_inv[c]
+      end
+      k = j + 1
+
+    elseif x == 34 then -- `"`: End of string
+      res = res .. str:sub(k, j - 1)
+      return res, j + 1
+    end
+
+    j = j + 1
+  end
+
+  decode_error(str, i, "expected closing quote for string")
+end
+
+
+local function parse_number(str, i)
+  local x = next_char(str, i, delim_chars)
+  local s = str:sub(i, x - 1)
+  local n = tonumber(s)
+  if not n then
+    decode_error(str, i, "invalid number '" .. s .. "'")
+  end
+  return n, x
+end
+
+
+local function parse_literal(str, i)
+  local x = next_char(str, i, delim_chars)
+  local word = str:sub(i, x - 1)
+  if not literals[word] then
+    decode_error(str, i, "invalid literal '" .. word .. "'")
+  end
+  return literal_map[word], x
+end
+
+
+local function parse_array(str, i)
+  local res = {}
+  local n = 1
+  i = i + 1
+  while 1 do
+    local x
+    i = next_char(str, i, space_chars, true)
+    -- Empty / end of array?
+    if str:sub(i, i) == "]" then
+      i = i + 1
+      break
+    end
+    -- Read token
+    x, i = parse(str, i)
+    res[n] = x
+    n = n + 1
+    -- Next token
+    i = next_char(str, i, space_chars, true)
+    local chr = str:sub(i, i)
+    i = i + 1
+    if chr == "]" then break end
+    if chr ~= "," then decode_error(str, i, "expected ']' or ','") end
+  end
+  return res, i
+end
+
+
+local function parse_object(str, i)
+  local res = {}
+  i = i + 1
+  while 1 do
+    local key, val
+    i = next_char(str, i, space_chars, true)
+    -- Empty / end of object?
+    if str:sub(i, i) == "}" then
+      i = i + 1
+      break
+    end
+    -- Read key
+    if str:sub(i, i) ~= '"' then
+      decode_error(str, i, "expected string for key")
+    end
+    key, i = parse(str, i)
+    -- Read ':' delimiter
+    i = next_char(str, i, space_chars, true)
+    if str:sub(i, i) ~= ":" then
+      decode_error(str, i, "expected ':' after key")
+    end
+    i = next_char(str, i + 1, space_chars, true)
+    -- Read value
+    val, i = parse(str, i)
+    -- Set
+    res[key] = val
+    -- Next token
+    i = next_char(str, i, space_chars, true)
+    local chr = str:sub(i, i)
+    i = i + 1
+    if chr == "}" then break end
+    if chr ~= "," then decode_error(str, i, "expected '}' or ','") end
+  end
+  return res, i
+end
+
+
+local char_func_map = {
+  [ '"' ] = parse_string,
+  [ "0" ] = parse_number,
+  [ "1" ] = parse_number,
+  [ "2" ] = parse_number,
+  [ "3" ] = parse_number,
+  [ "4" ] = parse_number,
+  [ "5" ] = parse_number,
+  [ "6" ] = parse_number,
+  [ "7" ] = parse_number,
+  [ "8" ] = parse_number,
+  [ "9" ] = parse_number,
+  [ "-" ] = parse_number,
+  [ "t" ] = parse_literal,
+  [ "f" ] = parse_literal,
+  [ "n" ] = parse_literal,
+  [ "[" ] = parse_array,
+  [ "{" ] = parse_object,
+}
+
+
+parse = function(str, idx)
+  local chr = str:sub(idx, idx)
+  local f = char_func_map[chr]
+  if f then
+    return f(str, idx)
+  end
+  decode_error(str, idx, "unexpected character '" .. chr .. "'")
+end
+
+
+function json.decode(str)
+  if type(str) ~= "string" then
+    error("expected argument of type string, got " .. type(str))
+  end
+  local res, idx = parse(str, next_char(str, 1, space_chars, true))
+  idx = next_char(str, idx, space_chars, true)
+  if idx <= #str then
+    decode_error(str, idx, "trailing garbage")
+  end
+  return res
+end
+
+
+return json

states/haproxy/scripts/weight_by_latency.lua

@@ -0,0 +1,43 @@
+local function getmax(t)
+  local tmpvalue = 100000
+  local svname
+  local value
+
+  for k,v in pairs(t) do
+    if v < tmpvalue then
+      svname = k
+      value = v
+    end
+    tmpvalue = v
+  end
+  return svname, value
+end
+
+local function arrange_backends()
+  local results = {}
+  while true do
+    for _, backend in pairs(core.backends) do
+      results = {}
+      for n,server in pairs(backend.servers) do
+        if server:get_stats()["check_status"] == "L4OK" then
+          local svname = server:get_stats()["svname"]
+          local latency = server:get_stats()["check_duration"]
+          results[svname] = latency
+        end
+      end
+      local b,c = getmax(results)
+      if b ~= nil then
+        for n,server in pairs(backend.servers) do
+          if b == server.name then
+            server:set_weight("10")
+          else
+            server:set_weight("1")
+          end
+        end
+      end
+    end
+    core.msleep(1000)
+  end
+end
+
+core.register_task(arrange_backends)

states/haproxy/templates/haproxy.cfg.j2

@@ -1,21 +1,35 @@
 ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }}
 {%- from "haproxy/map.jinja" import haproxy,certs with context %}
 
-{%- macro internal_access() -%}
+{%- macro internal() -%}
-  acl internal src -f /etc/haproxy/access
+  acl internal src -f {{ haproxy.config.dir }}/maps/access
   http-response return status 403 default-errorfiles if ! internal
 {%- endmacro -%}
 
-{%- macro handle_head() -%}
+{%- macro head() -%}
   http-request return status 200 if { method -i HEAD }
 {%- endmacro -%}
 
-{%- macro handle_endpoints(endpoints, check, ssl) -%}
+{%- macro serverheader() -%}
-  {%- for endpoint in endpoints %}
+  http-response set-header server "{{ haproxy.config.servername }}"
-  server {{ endpoint.name }} {{ endpoint.name }}:{{ endpoint.port }}{{ " check observe layer7 " if check|default(true) }}{{ " ssl verify none " if ssl|default(false) }}
+{%- endmacro -%}
+
+{%- macro endpoints(servers, check, ssl) -%}
+  {%- for server in servers %}
+  server {{ server.name }} {{ server.name }}:{{ server.port }}{{ " check observe layer7 inter 1s fall 5 rise 5 " if check|default(true) }}{{ " ssl verify none " if ssl|default(false) }}
   {%- endfor %}
 {%- endmacro -%}
 
+{%- macro cache() -%}
+  http-request cache-use static if { path_end .css .js .png .jpg }
+  http-response cache-store static 
+{%- endmacro -%}
+
+{%- macro compression() -%}
+  compression algo gzip
+  compression type text/html text/plain text/css text/javascript application/javascript
+{%- endmacro -%}
+
 {%- macro admin() -%}
 listen stats
   mode http
@@ -26,32 +40,32 @@ listen stats
 {%- endmacro -%}
 
 {%- macro api() -%}
-listen stats
-  mode http
-  bind *:7000 v4v6
-  stats enable
-  stats refresh 5s
-  stats uri /
-{%- endmacro %}
-
-global
-{%- for filename in haproxy.scripts %}
-  lua-load {{ haproxy.config.dir }}/scripts/{{ filename }}
-{%- endfor %}
-  maxconn 1000
   stats socket ipv4@127.0.0.1:9990 level admin
   stats socket /var/run/hap-lb.sock mode 666 level admin
   stats timeout 2m
+{%- endmacro %}
+
+global
+  lua-prepend-path {{ haproxy.config.dir }}/scripts/?.lua
+{%- for file in haproxy.scripts %}
+  {%- if not file.lib %}
+  lua-load {{ haproxy.config.dir }}/{{ file.name }}
+  {%- endif %}
+{%- endfor %}
+  maxconn 1000
+{%- if haproxy.config.api.enable %}
+  {{ api() }}
+{%- endif %}
   ssl-default-bind-ciphers {{ haproxy.config.ssl_ciphers|join(":") }}
   ssl-default-bind-options {{ haproxy.config.ssl_options|join(" ") }}
   ssl-default-server-ciphers {{ haproxy.config.ssl_ciphers|join(":") }}
   ssl-default-server-options {{ haproxy.config.ssl_options|join(" ") }}
-  crt-base {{ haproxy.config.acme_dir }}/certs
+  crt-base {{ haproxy.config.acme_fullchains_dir }}
-  ssl-dh-param-file {{ haproxy.config.acme_dir }}/dh/dh.pem
+  ssl-dh-param-file {{ haproxy.config.acme_dh_dir }}/dh.pem
 
 defaults
-{%- for default in haproxy.config.defaults.keys() %}
+{%- for key, value in haproxy.config.defaults.items() %}
-  {{ default }}
+  {{ key }} {{ value }}
 {%- endfor %}
 
 {%- if haproxy.config.admin %}
@@ -59,10 +73,13 @@ defaults
 {%- endif %}
 
 cache static
-  total-max-size 4095
+  total-max-size 256
   max-object-size 50000
   max-age 120
 
+backend per_ip_rates
+  stick-table type string size 1m expire 10s store http_req_rate(10s)
+
 frontend http
   bind *:80,:::80 v4v6
   mode http
@@ -70,37 +87,59 @@ frontend http
   http-request redirect scheme https if http
 
 frontend https
-  bind *:443,:::443 v4v6 {% for cert in certs %}{{ " ssl crt " + cert + " " }}{% endfor %}
+  bind *:443,:::443 v4v6 ssl crt {{ haproxy.config.acme_fullchains_dir }}{% if haproxy.config.http2 %} alpn h2,http/1.1{% endif %}
+  mode http
+  option httplog
   {%- for name, values in haproxy.config.vhosts.items() %}
   use_backend {{ name }} if { hdr(Host) -i {{ values.host }} }
   {%- endfor %}
+  http-request track-sc0 src table per_ip_rates
+  http-request capture req.hdr(User-Agent) len 200
+  http-request capture req.hdr(Content-Type) len 200
+  http-request capture req.hdr(Referer) len 200
+  http-request capture sc_http_req_rate(0) len 4
+  http-request deny deny_status 429 if { sc_http_req_rate(0) gt 100 }
+  http-request set-header x-proxy-id "{{ salt["grains.get"]("host") }}"
+  log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
   default_backend nginx
 
 {% for name, values in haproxy.config.vhosts.items() %}
 backend {{ name }}
   balance {{ values.balance|default("roundrobin") }}
-{%- if values.handle_head|default(false) %}
+  mode http
-  {{ handle_head() }}
+  option forwardfor
+{%- if values.head|default(false) %}
+  {{ head() }}
 {%- endif %}
+
+{%- if values.compression|default(true) %}
+  {{ compression() }}
+{%- endif %}
+
 {%- if values.usecache|default(true) %}
-  http-request cache-use static if { path_end .css .js .png .jpg }
+  {{ cache() }}
-  http-response cache-store static 
 {%- endif %}
-{%- if values.internal_access|default(false) %}
+
-  {{ internal_access() }}
+{%- if values.serverheader|default(true) %}
+  {{ serverheader() }}
 {%- endif %}
-  {{- handle_endpoints(values.endpoints, values.check, values.ssl) }}
+
-{% endfor %}
+{%- if values.internal|default(false) %}
+  {{ internal() }}
+{%- endif %}
+  {{- endpoints(values.servers, values.check, values.ssl) }}
+{% endfor -%}
 
 {% for name, values in haproxy.config.services.items() %}
 listen {{ name }}
   bind :::{{ values.port }} v4v6
   mode tcp
+  option tcplog
 {%- if values.type == "postgres" %}
   option pgsql-check user postgres
 {%- endif %}
   default-server inter 3s fall 3
-  {%- for endpoint in values.endpoints %}
+  {%- for server in values.servers %}
-  server {{ endpoint.name }} {{ endpoint.name }}:{{ endpoint.port }} check {{ "backup" if endpoint.backup|default(false) }} port {{ endpoint.port }}
+  server {{ server.name }} {{ server.name }}:{{ server.port }} check {{ "backup" if server.backup|default(false) }} port {{ server.port }}
   {%- endfor %}
-{% endfor %}
+{% endfor -%}