Home / server / Questions / 1049158
In Process
Quandary
Asked: 2021-01-09 17:09:31 +0800 CST

Nginx: How to combine ssl_preread_protocol with ssl_preread_server_name (ssh-multiplexing and ssl-sni-passthrough)?

  • 772

I have the nginx.conf file shown below.

I want to run both ssh and a webserver on port 443/SSL.
Also known as SSL-port-multiplexing.
At the same time, I want to use ssl-passthrough with SNI.

For ssh-multiplexing, I use $ssl_preread_protocol.
For SSL-SNI-passthrough, I use $ssl_preread_server_name

If I set proxy_pass $upstream;, then ssh works fine, but the webpage(s) don't work.
If I set proxy_pass $name;, then SSL-SNI-passthrough works, but ssh can't be accessed.

How can I combine the two map instructions ? e.g. something like

if $upstream = ssh 
then proxy_pass $upstream
else proxy_pass $name;
endif

The problem is I need a way to combine the protocol-selection with the server_name-selection.

if(ssh) => forward to port 22
else => forward to port xy depending on server_name

Here's my config file:

stream{

    upstream ssh 
    {
        server 127.0.0.1:22;
    }
    
    upstream https_default_backend 
    {
        server 127.0.0.1:443;
    }
    
    upstream daniel_backend 
    {
        server 127.0.0.1:5005;
    }
    
    
    map $ssl_preread_protocol $upstream 
    {
        default ssh;
        "TLSv1.3" https_default_backend;
        "TLSv1.2" https_default_backend;
        "TLSv1.1" https_default_backend;
        "TLSv1" https_default_backend;
    }
    
    
    map $ssl_preread_server_name $name 
    {
        localhost daniel_backend;
        prodesk daniel_backend;
        daniel-steiger.ch daniel_backend;
        www.daniel-steiger.ch daniel_backend;
        default https_default_backend;
    }
    
    
    # SSH and SSL on the same port
    server {
        listen 443;
        
        ssl_preread on;
        #proxy_protocol on;
        
        # proxy_pass $upstream;
        proxy_pass $name;
    }
    
}
nginx passthrough sni

2 Answers

  1. Already find the solution?

    I also have this problem, and I try this. It seems ok.

    stream {

upstream ssh {
    server 127.0.0.1:22;
}

upstream https_default_backend {
    server 127.0.0.1:443;
}

upstream daniel_backend {
    server 127.0.0.1:5005;
}

map $ssl_preread_protocol $upstream {
    "" ssh;
    default $name;
    "TLSv1.3" $name;
    "TLSv1.2" $name;
    "TLSv1.1" $name;
    "TLSv1" $name;
}
    
map $ssl_preread_server_name $name {
    localhost daniel_backend;
    prodesk daniel_backend;
    daniel-steiger.ch daniel_backend;
    www.daniel-steiger.ch daniel_backend;
    default https_default_backend;
}

server {
    listen 443;
    ssl_preread on;
    proxy_pass $upstream;
}
}
    • 0

  2. Necromancing.
    Answering my own question for the benefit of others.
    This is NOT possible with nginx (AFAIK).
    You can, however, achive the goal with HAproxy.
    The configuration is not quite that easy, so see below the hack that works for me.

    Note that I've changed all values with search-and replace (there might be errors) in notepad.
    This configuration assumes the following:

    If you want to use the proxy-protocol (proxy v2 is newest), uncomment # send-proxy-v2 e.g. the line

    server web0 127.0.0.1:8005 # send-proxy-v2

    becomes

    server web0 127.0.0.1:8005 send-proxy-v2

    Note that sni-passthrough reverses the proxy-order.
    In nginx, the order is
    -> request -> decrypt -> proxy headering decrypted request -> re-encrypt request -> forward
    In haproxy SNI-passthough, the order becomes
    -> request -> proxy headering encrypted request -> forward

    Thus the middleware processing order in your http servers (on port 8000+x)
    using nginx is -> SSL-decrypt -> unheader -> process
    while using HAproxy, it is -> unheader -> SSL-decrypt -> process

    This is due to using sni-passthrough on HAproxy, and using the SSL-keys in nginx (no passthrough). This nastly little fact caused me a lot of head-scratching.

    Also note, I set up example.int, foo.int and bar.int in the hosts file resolving to 10.0.0.2 (internal-network IP address of the machine with HAproxy) in the local network for testing purposes. You still see these entries in this haproxy.cfg file

    # /etc/haproxy/haproxy.cfg

# Validate: 
# haproxy -c -V -f /etc/haproxy/haproxy.cfg
# Another way is to 
# sudo service haproxy configtest

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

    # Default SSL material locations
    ca-base /etc/ssl/certs
    crt-base /etc/ssl/private

    # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
        ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
        ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_comACHA20_POLY1305_SHA256
        ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets

defaults
    log global
    mode    http
    option  httplog
    option  dontlognull
        timeout connect 5000
        timeout client  50000
        timeout server  50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

    

frontend http
    bind *:80
    mode http
    option forwardfor
    # option httpchk /check.cfm
    # use-server  server1 if { hdr(host) -i server1.domain.net }
    # use-server  server2 if { hdr(host) -i server2.domain.net }
    # server server1 localhost:22201 check
    # server server2 localhost:22202 check
    # default_backend nodes
    # redirect scheme https code 301 if !{ ssl_fc }
    
    
    
    # http://10.0.0.2/.well-known/acme-challenge/token.txt
    # http://44.33.22.11/.well-known/acme-challenge/token.txt
    # http://firstname-lastname.com/.well-known/acme-challenge/token.txt
    # http://forename-familyname.com/.well-known/acme-challenge/token.txt
    
    # https://www.haproxy.com/documentation/aloha/12-5/traffic-management/lb-layer7/acls/  
    # For ACLs sharing the same name, the following rules apply:
    # It is possible to use the same <aclname> for many ACLs, even if they do not have the same matching criterion
    # A logical OR applies between all of them
    
    # acl firstname_lastname_com dst 10.0.0.2
    # acl firstname_lastname_com dst 44.33.22.11
    
    acl firstname_lastname_com  hdr(host)     -i 44.33.22.11
    acl firstname_lastname_com  hdr(host)     -i 10.0.0.2
    
    acl firstname_lastname_com  hdr(host)     -i firstname-lastname.com
    acl firstname_lastname_com  hdr(host)     -m end .firstname-lastname.com
    
    acl forename_familyname_com  hdr(host)     -i forename-familyname.com
    acl forename_familyname_com  hdr(host)     -m end .forename-familyname.com
    
    
    #use_backend http_firstname_lastname_com if { hdr(host) -i firstname-lastname.com }
    #use_backend http_firstname_lastname_com if { hdr(host) -m end .firstname-lastname.com }
    
    use_backend http_firstname_lastname_com if firstname_lastname_com 
    use_backend http_forename_familyname_com if forename_familyname_com 
    
    
    
    

    
backend http_firstname_lastname_com
    mode http
    balance roundrobin
    server web0 127.0.0.1:8006
    
    
    
backend http_forename_familyname_com
    mode http
    balance roundrobin
    server web0 127.0.0.1:8008
    
    
    
#backend nodes
#    mode http
#    balance roundrobin
#    option forwardfor
#    reqirep ^Host: Host:\ node1.myapp.mycompany.com
#    server web01 node1.myapp.mycompany.com:80
    
# sudo systemctl stop nginx
# sudo systemctl disable nginx

# sudo systemctl enable haproxy
# service haproxy start
# sudo haproxy -c -V -f /etc/haproxy/haproxy.cfg
# service haproxy restart

    
frontend https
    bind *:443
    mode tcp
    option tcplog
    tcp-request inspect-delay 5s
    tcp-request content accept if { req.ssl_hello_type 1 }
    #tcp-request content accept if { req_ssl_hello_type 1 }

    # https://datamakes.com/2018/02/17/high-intensity-port-sharing-with-haproxy/
    # systemctl restart sshd
    # systemctl disable sshd
    # systemctl enable sshd
    # sudo apt-get install openssh-server
    # sudo systemctl status ssh
    # sudo ufw allow ssh
    # sudo ufw enable
    # sudo ufw status
    # ufw allow 443/tcp
    # ufw allow 8443/tcp
    # /etc/ssh/sshd_config  ==> PermitRootLogin yes  + PasswordAuthentication no + ChallengeResponseAuthentication no  ~/.ssh/id_rsa.pub ==> ~/.ssh/authorized_keys
    acl ssh_payload payload(0,7) -m bin 5353482d322e30
    

    
    
    
    
    # /mnt/sshfs/var/www/.dotnet/corefx/cryptography/crls/
    # sudo apt-get install exfat-utils exfat-fuse

    
    # https://10.0.0.2/.well-known/acme-challenge/token.txt
    # https://44.33.22.11/.well-known/acme-challenge/token.txt
    # http://firstname-lastname.com/.well-known/acme-challenge/token.txt
    # http://forename-familyname.com/.well-known/acme-challenge/token.txt
  
    # https://www.haproxy.com/documentation/aloha/12-5/traffic-management/lb-layer7/acls/  
    # For ACLs sharing the same name, the following rules apply:
    # It is possible to use the same <aclname> for many ACLs, even if they do not have the same matching criterion
    # A logical OR applies between all of them
  
  
  # sequence matters ! 
    use_backend openssh if ssh_payload
    use_backend openssh if !{ req.ssl_hello_type 1 } { req.len 0 }
  
  
    # having these two lines here blocks ssh if use_backend openssh comes afterwards ...
    # also, this fucks up SNI ...
    # acl firstname_lastname_com dst 10.0.0.2
    # acl firstname_lastname_com dst 44.33.22.11
    
    acl firstname_lastname_com req_ssl_sni -i firstname-lastname.com
    acl firstname_lastname_com req.ssl_sni -m end .firstname-lastname.com
    
    acl forename_familyname_com req_ssl_sni -i forename-familyname.com
    acl forename_familyname_com req.ssl_sni -m end .forename-familyname.com
    
    
    # wildcard
    use_backend https_firstname_lastname_com if firstname_lastname_com
    use_backend https_forename_familyname_com if forename_familyname_com
    
    
    # use_backend example_int if { req_ssl_sni -i example.int }
    # use_backend example_int if { req_ssl_sni -m end .example.int }

    # use_backend example_int if { req_ssl_sni -i example.int }
    # use_backend foo_int if { req_ssl_sni   -i foo.int }
    # use_backend bar_int if { req_ssl_sni -i bar.int }

    
# sudo haproxy -c -V -f /etc/haproxy/haproxy.cfg
    
backend https_firstname_lastname_com
    mode tcp
    balance roundrobin
    server web0 127.0.0.1:8005 # send-proxy-v2
    
backend https_forename_familyname_com
    mode tcp
    balance roundrobin
    server web0 127.0.0.1:8007 # send-proxy-v2

backend foo_int
    balance roundrobin
    server web1 127.0.0.1:8005 send-proxy

backend bar_int 
    balance roundrobin
    server web2 127.0.0.1:8005 ##send-proxy


backend openssh
        mode tcp
        # option tcplog
        # option tcp-check
        # tcp-check expect string SSH-2.0-
        timeout server 3h
        # server openssh 127.0.0.1:22 check 
        server openssh 127.0.0.1:22

    This config forwards all requests for

    ssh [email protected] -p 443

    to 127.0.0.1:22

    and all requests for
    http://firstname-lastname.com to 127.0.0.1:800X where X = 2n (even)
    https://firstname-lastname.com to 127.0.0.1:800X where X = 2n+1 (odd)
    (the better idea would have been to use 800X for http and 900X for https)

    • 0