Proxies timeouts and DNS resolving

packages/nomad/client-proxy.hcl

@@ -26,12 +26,12 @@ variable "session_proxy_service_name" {
   type = string
 }
 
-variable "domain_name" {
+variable "load_balancer_conf" {
   type = string
 }
 
-locals {
-  domain_name_escaped = replace(var.domain_name, ".", "\\.")
+variable "nginx_conf" {
+  type = string
 }
 
 job "client-proxy" {
@@ -73,114 +73,31 @@ job "client-proxy" {
       }
 
       config {
-        // TODO: Fixate versionx
-        image        = "nginx"
+        image        = "nginx:1.27.0"
         network_mode = "host"
         ports        = [var.client_proxy_health_port_name, var.client_proxy_port_name]
         volumes = [
-          "local:/etc/nginx/conf.d",
+          "local:/etc/nginx/",
           "/var/log/client-proxy:/var/log/nginx"
         ]
       }
 
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
-        destination     = "local/load-balancer.conf"
+        data            = var.load_balancer_conf
+        destination     = "local/conf.d/load-balancer.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
-        data            = <<EOF
-map $http_upgrade $conn_upgrade {
-  default     "";
-  "websocket" "Upgrade";
-}
-
-log_format logger-json escape=json
-'{'
-'"source": "client-proxy",'
-'"time": "$time_iso8601",'
-'"resp_body_size": $body_bytes_sent,'
-'"host": "$http_host",'
-'"address": "$remote_addr",'
-'"request_length": $request_length,'
-'"method": "$request_method",'
-'"uri": "$request_uri",'
-'"status": $status,'
-'"user_agent": "$http_user_agent",'
-'"resp_time": $request_time,'
-'"upstream_addr": "$upstream_addr"'
-'}';
-access_log /var/log/nginx/access.log logger-json;
-
-server {
-  listen 3002 default_server;
-
-  server_name _;
-  return 502 "Cannot connect to sandbox";
-}
-[[ range service "session-proxy" ]]
-server {
-  listen 3002;
-  access_log /var/log/nginx/access.log logger-json;
-
-  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${local.domain_name_escaped}$;
-
-  proxy_set_header Host $host;
-  proxy_set_header X-Real-IP $remote_addr;
-
-  proxy_set_header Upgrade $http_upgrade;
-  proxy_set_header Connection $conn_upgrade;
-
-  proxy_hide_header x-frame-options;
-
-  proxy_http_version 1.1;
-
-  client_body_timeout 86400s;
-  client_header_timeout 5s;
-
-  # proxy_read_timeout 600s;
-  proxy_read_timeout 1d;
-  proxy_send_timeout 86400s;
-
-  proxy_cache_bypass 1;
-  proxy_no_cache 1;
-
-  client_max_body_size 1024m;
-
-  proxy_buffering off;
-  proxy_request_buffering off;
-
-  tcp_nodelay on;
-  tcp_nopush on;
-  sendfile on;
-  # send_timeout                600s;
-  # proxy_connect_timeout       30s;
-
-  keepalive_timeout 600s;
-  keepalive_requests 2048;
-  # keepalive_time 86400s;
-
-  # gzip off;
-  location / {
-    proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;
-  }
-}
-[[ end ]]
-server {
-  listen 3001;
-  location /health {
-    access_log off;
-    add_header 'Content-Type' 'application/json';
-    return 200 '{"status":"UP"}';
-  }
+      }
 
-  location /status {
-    access_log off;
-    stub_status;
-    allow all;
-  }
-}
-EOF
+      template {
+        left_delimiter  = "[["
+        right_delimiter = "]]"
+        data            = var.nginx_conf
+        destination     = "local/nginx.conf"
+        change_mode     = "signal"
+        change_signal   = "SIGHUP"
       }
     }
   }

packages/nomad/main.tf

@@ -110,7 +110,10 @@ resource "nomad_job" "client_proxy" {
       client_proxy_health_port_name   = var.client_proxy_health_port.name
       client_proxy_health_port_path   = var.client_proxy_health_port.path
       session_proxy_service_name      = var.session_proxy_service_name
-      domain_name                     = var.domain_name
+      load_balancer_conf              = templatefile("${path.module}/proxies/client.conf", {
+        domain_name_escaped = replace(var.domain_name, ".", "\\.")
+      })
+      nginx_conf                      = file("${path.module}/proxies/nginx.conf")
     }
   }
 }
@@ -125,6 +128,8 @@ resource "nomad_job" "session_proxy" {
       session_proxy_port_number  = var.session_proxy_port.port
       session_proxy_port_name    = var.session_proxy_port.name
       session_proxy_service_name = var.session_proxy_service_name
+      load_balancer_conf         = file("${path.module}/proxies/session.conf")
+      nginx_conf                 = file("${path.module}/proxies/nginx.conf")
     }
   }
 }

packages/nomad/proxies/client.conf

@@ -0,0 +1,89 @@
+map $http_upgrade $conn_upgrade {
+  default     "";
+  "websocket" "Upgrade";
+}
+
+log_format logger-json escape=json
+'{'
+'"source": "client-proxy",'
+'"time": "$time_iso8601",'
+'"resp_body_size": $body_bytes_sent,'
+'"host": "$http_host",'
+'"address": "$remote_addr",'
+'"request_length": $request_length,'
+'"method": "$request_method",'
+'"uri": "$request_uri",'
+'"status": $status,'
+'"user_agent": "$http_user_agent",'
+'"resp_time": $request_time,'
+'"upstream_addr": "$upstream_addr"'
+'}';
+access_log /var/log/nginx/access.log logger-json;
+
+server {
+  listen 3002 default_server;
+
+  server_name _;
+  return 502 "Cannot connect to sandbox";
+}
+[[ range service "session-proxy" ]]
+server {
+  listen 3002;
+  access_log /var/log/nginx/access.log logger-json;
+
+  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${domain_name_escaped}$;
+
+  proxy_set_header Host $host;
+  proxy_set_header X-Real-IP $remote_addr;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $conn_upgrade;
+
+  proxy_hide_header x-frame-options;
+
+  proxy_http_version 1.1;
+
+  client_body_timeout 86400s;
+  client_header_timeout 5s;
+
+  # proxy_read_timeout 600s;
+  proxy_read_timeout 1d;
+  proxy_send_timeout 86400s;
+
+  proxy_cache_bypass 1;
+  proxy_no_cache 1;
+
+  client_max_body_size 1024m;
+
+  proxy_buffering off;
+  proxy_request_buffering off;
+
+  tcp_nodelay on;
+  tcp_nopush on;
+  sendfile on;
+  # send_timeout                600s;
+  # proxy_connect_timeout       30s;
+
+  keepalive_timeout 600s;
+  keepalive_requests 2048;
+
+  # gzip off;
+  location / {
+    proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;
+  }
+}
+[[ end ]]
+server {
+  listen 3001;
+  location /health {
+    access_log off;
+    add_header 'Content-Type' 'application/json';
+    return 200 '{"status":"UP"}';
+  }
+
+  location /status {
+    access_log off;
+    stub_status;
+    allow all;
+  }
+}

packages/nomad/proxies/nginx.conf

@@ -0,0 +1,32 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_time     86400s;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+
+

packages/nomad/proxies/session.conf

@@ -0,0 +1,98 @@
+map $host $dbk_port {
+  default         "";
+  "~^(?<p>\d+)-"  ":$p";
+}
+
+map $host $dbk_session_id {
+  default         "";
+  "~-(?<s>\w+)-"  $s;
+}
+
+map $http_upgrade $conn_upgrade {
+  default     "";
+  "websocket" "Upgrade";
+}
+
+log_format logger-json escape=json
+'{'
+'"source": "session-proxy",'
+'"time": "$time_iso8601",'
+'"resp_body_size": $body_bytes_sent,'
+'"host": "$http_host",'
+'"address": "$remote_addr",'
+'"request_length": $request_length,'
+'"method": "$request_method",'
+'"uri": "$request_uri",'
+'"status": $status,'
+'"user_agent": "$http_user_agent",'
+'"resp_time": $request_time,'
+'"upstream_addr": "$upstream_addr"'
+'}';
+access_log /var/log/nginx/access.log logger-json;
+
+server {
+  listen 3003;
+
+  # DNS server resolved addreses as to <sandbox-id> <ip-address>
+  resolver 127.0.0.1;
+  resolver_timeout 5s;
+
+  proxy_set_header Host $host;
+  proxy_set_header X-Real-IP $remote_addr;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $conn_upgrade;
+
+  proxy_hide_header x-frame-options;
+
+  proxy_http_version 1.1;
+
+  client_body_timeout 86400s;
+  client_header_timeout 5s;
+
+  proxy_read_timeout 600s;
+  proxy_send_timeout 86400s;
+
+  proxy_cache_bypass 1;
+  proxy_no_cache 1;
+
+  client_max_body_size 1024m;
+
+  proxy_buffering off;
+  proxy_request_buffering off;
+
+  tcp_nodelay on;
+  tcp_nopush on;
+  sendfile on;
+
+  # send_timeout                600s;
+
+  proxy_connect_timeout 3s;
+  keepalive_requests 2048;
+  keepalive_timeout 600s;
+  # gzip off;
+
+  location / {
+    if ($dbk_session_id = "") {
+      return 502 "Cannot connect to sandbox";
+    }
+
+    proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri;
+  }
+}
+
+server {
+  listen 3004;
+
+  location /health {
+    access_log off;
+    add_header 'Content-Type' 'application/json';
+    return 200 '{"status":"UP"}';
+  }
+
+  location /status {
+    access_log off;
+    stub_status;
+    allow all;
+  }
+}