Auto Failover for Master Node
The new config allows for one node to always act as master. Should the master node fail the failover node will kick in. During normal operations the failover node's recon traffic is routed to the master node and whilst the master is down recon is routed to external peers.
Haproxy tcp routing over NAT
Previously I was also NATing traffic via the lb node to external peers, i've now consolidated this requirement into haproxy.
During Normal Running
Outbound recon traffic from the failover is routed to master1
During Failover
Outbound traffic from the failover node is routed to the external peers
New HaProxy Config
global
log 127.0.0.1 local2 debug
maxconn 4096
chroot /usr/share/haproxy
uid 99
gid 99
daemon
tune.ssl.default-dh-param 4096
ssl-default-bind-options no-sslv3 no-tls-tickets
stats socket /var/run/haproxy.stat mode 600 level operator
stats timeout 5s
ssl-default-bind-ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:ECDH-RSA-AES256-GCM-SHA384:ECDH-ECDSA-AES256-GCM-SHA384:ECDH-RSA-AES256-SHA384:ECDH-ECDSA-AES256-SHA384:ECDH-RSA-AES256-SHA:ECDH-ECDSA-AES256-SHA:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:EDH-DSS-DES-CBC3-SHA:ECDH-RSA-AES128-GCM-SHA256:ECDH-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-DES-CBC3-SHA"
tune.ssl.maxrecord 2859
tune.ssl.cachesize 100000
tune.ssl.lifetime 600
defaults
mode http
option dontlognull
retries 3
maxconn 2000
timeout connect 5000
timeout client 50000
timeout server 50000
option abortonclose
retries 2
##################################################
# frontends
##################################################
frontend http-sks
mode http
#log global
option forwardfor # set the client's IP in X-Forwarded-For.
option httplog
option httpclose
rspadd Via: lb1.mj2.uk
rsprep ^Server.* Server: Unix
bind 178.32.66.144:80
bind 178.32.66.144:11371
bind 2001:41d0:2:a8b4::10:80
bind 2001:41d0:2:a8b4::10:11371
bind 10.0.0.19:11371
bind 10.0.0.21:11371
bind 10.0.0.22:11371
bind 10.0.0.23:11371
bind 10.0.0.24:11371
bind 10.0.0.25:11371
bind 10.0.0.36:11371
# Security
option http-buffer-request
timeout http-request 60s
# SKS
acl apac_url path_beg /.well
acl mast_url path_beg /pks/add /pks/hashquery
acl mast_param urlp(op) stats
acl sks_n2 urlp(go) n2
acl sks_n3 urlp(go) n3
acl sks_n4 urlp(go) n4
acl sks_n5 urlp(go) n5
acl sks_m1 urlp(go) m1
acl sks_m2 urlp(go) m2
use_backend http-sks-n3 if mast_param sks_n3
use_backend http-sks-n4 if mast_param sks_n4
use_backend http-sks-n5 if mast_param sks_n5
use_backend http-sks-m1 if mast_param sks_m1
use_backend http-sks-m2 if mast_param sks_m2
#peer2
acl host_p2 hdr_end(host) -i sks.srv.dumain.com
acl ip_p2 dst 10.0.0.24
use_backend http-sks-recon-peer2 if host_p2 or ip_p2
#peer3
acl host_p3 hdr_end(host) -i keyserver.opensuse.org
acl ip_p3 dst 10.0.0.21
use_backend http-sks-recon-peer3 if host_p3 or ip_p3
#peer4
acl host_p4 hdr_end(host) -i whippet.andrewg.com
acl ip_p4 dst 10.0.0.22
use_backend http-sks-recon-peer4 if host_p4 or ip_p4
#peer5
acl host_p5 hdr_end(host) -i skspub.ward.ie
acl ip_p5 dst 10.0.0.23
use_backend http-sks-recon-peer5 if host_p5 or ip_p5
#peer6
acl host_p6 hdr_end(host) -i keys.bonus-communis.eu
acl ip_p6 dst 10.0.0.25
use_backend http-sks-recon-peer6 if host_p6 or ip_p6
#peer7
acl host_p7 hdr_end(host) -i keyserver.garden-lan.com
acl ip_p7 dst 10.0.0.36
use_backend http-sks-recon-peer7 if host_p7 or ip_p7
#peer8
#acl host_p8 hdr_end(host) -i www.example.com
#use_backend http-sks-recon-peer8 if host_p8 or ip_p8
#peer9
#acl host_p9 hdr_end(host) -i www.example.com
#use_backend http-sks-recon-peer9 if host_p9 or ip_p9
# No peer mast req
use_backend http-sks-master-apache if apac_url
use_backend http-sks-master if mast_url
use_backend http-sks-master if mast_param
# Default
default_backend http-sks
frontend tcp-sks-recon
mode tcp
#log global
option tcplog
bind 178.32.66.144:11370
#bind 2001:41d0:2:a8b4::10:11370
default_backend tcp-sks-recon
frontend tcp-sks-recon-outbound
mode tcp
#log global
option tcplog
bind 10.0.0.19:11370
bind 10.0.0.21:11370
bind 10.0.0.22:11370
bind 10.0.0.23:11370
bind 10.0.0.24:11370
bind 10.0.0.25:11370
bind 10.0.0.36:11370
acl p2 dst 10.0.0.24
acl p3 dst 10.0.0.21
acl p4 dst 10.0.0.22
acl p5 dst 10.0.0.23
acl p6 dst 10.0.0.25
acl p7 dst 10.0.0.36
#acl p8 dst ?
#acl p9 dst ?
acl master_alive nbsrv(tcp-sks-recon) gt 0
acl failover_src src 10.0.0.42
use_backend tcp-sks-recon if master_alive failover_src
use_backend tcp-sks-recon-peer2 if p2
use_backend tcp-sks-recon-peer3 if p3
use_backend tcp-sks-recon-peer4 if p4
use_backend tcp-sks-recon-peer5 if p5
use_backend tcp-sks-recon-peer6 if p6
use_backend tcp-sks-recon-peer7 if p7
#use_backend tcp-sks-recon-peer8 if p8
#use_backend tcp-sks-recon-peer9 if p9
default_backend tcp-sks-recon
frontend https-sks
mode http
option forwardfor # set the client's IP in X-Forwarded-For.
option httplog
option httpclose
rspadd Via: lb1.mj2.uk
rsprep ^Server.* Server: Unix
bind 178.32.66.144:443 ssl crt /etc/haproxy/ssl/sks.mj2.uk.pem crt /etc/haproxy/ssl/pgp.mj2.uk.pem # strict-sni
bind 2001:41d0:2:a8b4::10:443 ssl crt /etc/haproxy/ssl/sks.mj2.uk.pem crt /etc/haproxy/ssl/pgp.mj2.uk.pem # strict-sni
http-request set-header X-Forwarded-Port %[dst_port]
http-request add-header X-Forwarded-Proto https #if { ssl_fc }
acl mast_url path_beg /pks/add /pks/hashquery
acl mast_param urlp(op) stats
use_backend https-sks-master if mast_url
use_backend https-sks-master if mast_param
use_backend https-sks if { ssl_fc_sni cert1 } # content switching based on SNI
default_backend https-sks
# Security
option http-buffer-request
timeout http-request 60s
##################################################
# Backends
##################################################
backend http-sks-master-apache
mode http
option httpchk GET /test.html HTTP/1.1rnHost: sks.mj2.uk
http-check expect string OK
option forwardfor
server m1 10.0.0.43:11371 weight 1 maxconn 1000 check port 11371 inter 30s rise 1 fall 2
server m2 10.0.0.42:11371 weight 1 maxconn 1000 check port 11371 inter 30s rise 1 fall 2 backup
backend http-sks-master
mode http
option httpchk GET /test.html HTTP/1.1rnHost: sks.mj2.uk
http-check expect string OK
option forwardfor
server m1 10.0.0.43:11371 maxconn 1000 check port 11371 inter 30s rise 1 fall 2
server m2 10.0.0.42:11371 maxconn 1000 check port 11371 inter 30s rise 1 fall 2 backup
backend http-sks-n3
mode http
option forwardfor
server node3 10.0.0.39:11371 maxconn 1000
backend http-sks-n4
mode http
option forwardfor
server node4 10.0.0.40:11371 maxconn 1000
backend http-sks-n5
mode http
option forwardfor
server node5 10.0.0.41:11371 maxconn 1000
backend http-sks-m1
mode http
option forwardfor
server node5 10.0.0.43:11371 maxconn 1000
backend http-sks-m2
mode http
option forwardfor
server node5 10.0.0.42:11371 maxconn 1000
backend http-sks
mode http
option httpchk GET /test.html HTTP/1.1rnHost: sks.mj2.uk
http-check expect string OK
balance leastconn
hash-type consistent
option forwardfor
# Stick Table
stick-table type ip size 1m expire 1m
# Sticky Session
stick on src
# No Cookie Version
server node3 10.0.0.39:11371 weight 20 maxconn 1000 check port 11371 inter 20s rise 2 fall 3
server node4 10.0.0.40:11371 weight 20 maxconn 1000 check port 11371 inter 20s rise 2 fall 3
server node5 10.0.0.41:11371 weight 20 maxconn 1000 check port 11371 inter 20s rise 2 fall 3
backend tcp-sks-recon
mode tcp
option tcp-check
tcp-check connect
tcp-check expect string bitquantum
balance leastconn
server m1 10.0.0.43:11370 weight 1 maxconn 1000 check inter 60s rise 1 fall 2
server m2 10.0.0.42:11370 weight 1 maxconn 1000 check inter 60s rise 1 fall 2 backup
backend tcp-sks-recon-peer2
mode tcp
#server remote1 sks.srv.dumain.com:11370 maxconn 100
server remote1 85.119.82.209:11370 maxconn 100
backend http-sks-recon-peer2
mode http
#server remote1 sks.srv.dumain.com:11370 maxconn 100
server remote1 85.119.82.209:11371 maxconn 100
backend tcp-sks-recon-peer3
mode tcp
server remote1 keyserver.opensuse.org:11370 maxconn 100
backend http-sks-recon-peer3
mode http
server remote1 keyserver.opensuse.org:11371 maxconn 100
backend tcp-sks-recon-peer4
mode tcp
server remote1 whippet.andrewg.com:11370 maxconn 100
backend http-sks-recon-peer4
mode http
server remote1 whippet.andrewg.com:11371 maxconn 100
backend tcp-sks-recon-peer5
mode tcp
server remote1 skspub.ward.ie:11370 maxconn 100
backend http-sks-recon-peer5
mode http
server remote1 skspub.ward.ie:11371 maxconn 100
backend tcp-sks-recon-peer6
mode tcp
server remote1 keys.bonus-communis.eu:11370 maxconn 100
backend http-sks-recon-peer6
mode http
server remote1 keys.bonus-communis.eu:11371 maxconn 100
backend tcp-sks-recon-peer7
mode tcp
server remote1 keyserver.garden-lan.com:11370 maxconn 100
backend http-sks-recon-peer7
mode http
server remote1 keyserver.garden-lan.com:11371 maxconn 100
#backend tcp-sks-recon-peer8
# mode tcp
# server remote1 www.example.com:11370 maxconn 100
#backend http-sks-recon-peer8
# mode http
# server remote1 www.example.com:11371 maxconn 100
#backend tcp-sks-recon-peer9
# mode tcp
# server remote1 www.example.com:11370 maxconn 100
#backend http-sks-recon-peer9
# mode http
# server remote1 www.example.com:11371 maxconn 100
backend https-sks-master
mode http
option httpchk GET /test.html HTTP/1.1rnHost: sks.mj2.uk
http-check expect string OK
option forwardfor
# HSTS
# http://blog.haproxy.com/2015/06/09/haproxy-and-http-strict-transport-security-hsts-header-in-http-redirects/
#http-response set-header Strict-Transport-Security "max-age=16000000; includeSubDomains; preload;"
#server node1 10.0.0.9:11371 weight 1 maxconn 1000 check port 80 inter 30s rise 1 fall 2
server m1 10.0.0.43:11371 weight 1 maxconn 1000 check port 11371 inter 30s rise 1 fall 2
server m2 10.0.0.42:11371 weight 1 maxconn 1000 check port 11371 inter 30s rise 1 fall 2 backup
backend https-sks
mode http
option httpchk GET /test.html HTTP/1.1rnHost: sks.mj2.uk
http-check expect string OK
balance leastconn
#option httpclose
option forwardfor
# HSTS
# http://blog.haproxy.com/2015/06/09/haproxy-and-http-strict-transport-security-hsts-header-in-http-redirects/
#http-response set-header Strict-Transport-Security "max-age=16000000; includeSubDomains; preload;"
# Stick Table
stick-table type ip size 1m expire 1m
# Sticky Session
stick on src
# No Cookie Version
server node3 10.0.0.39:11371 weight 20 maxconn 1000 check port 11371 inter 30s rise 2 fall 3
server node4 10.0.0.40:11371 weight 20 maxconn 1000 check port 11371 inter 30s rise 2 fall 3
server node5 10.0.0.41:11371 weight 20 maxconn 1000 check port 11371 inter 30s rise 2 fall 3
backend capasity-limit-http
mode http
errorfile 503 /etc/haproxy/errors/rate-http.html
backend capasity-limit-https
mode http
errorfile 503 /etc/haproxy/errors/rate-https.html
listen stats
# disabled
bind :8888
stats uri /
mode http
stats enable
stats hide-version
stats realm Haproxy Statistics
stats auth ???:???
Update 27th Sep
I've swapped the health check on "tcp-sks-recon" and "tcp-sks-recon-outbound" from a http check on sks-db to a tcp check on sks-recon. This ensures recon is routed correctly ignoring the state of sks-db.
Update 1st Oct
I've updated the tcp check to check for the string "bitquantum", Changed the recon on external peers back to using individual ip as ports did not work (the recon on port would work but then all requests for missing keys would land on the same ip with the same request hostname).
