diff --git a/packages/nomad/client-proxy.hcl b/packages/nomad/client-proxy.hcl index c1f0e46e6..eeedfc7e4 100644 --- a/packages/nomad/client-proxy.hcl +++ b/packages/nomad/client-proxy.hcl @@ -26,12 +26,12 @@ variable "session_proxy_service_name" { type = string } -variable "domain_name" { +variable "load_balancer_conf" { type = string } -locals { - domain_name_escaped = replace(var.domain_name, ".", "\\.") +variable "nginx_conf" { + type = string } job "client-proxy" { @@ -73,12 +73,11 @@ job "client-proxy" { } config { - // TODO: Fixate versionx - image = "nginx" + image = "nginx:1.27.0" network_mode = "host" ports = [var.client_proxy_health_port_name, var.client_proxy_port_name] volumes = [ - "local:/etc/nginx/conf.d", + "local:/etc/nginx/", "/var/log/client-proxy:/var/log/nginx" ] } @@ -86,101 +85,19 @@ job "client-proxy" { template { left_delimiter = "[[" right_delimiter = "]]" - destination = "local/load-balancer.conf" + data = var.load_balancer_conf + destination = "local/conf.d/load-balancer.conf" change_mode = "signal" change_signal = "SIGHUP" - data = <\d+)-" ":$p"; +} + +map $host $dbk_session_id { + default ""; + "~-(?\w+)-" $s; +} + +map $http_upgrade $conn_upgrade { + default ""; + "websocket" "Upgrade"; +} + +log_format logger-json escape=json +'{' +'"source": "session-proxy",' +'"time": "$time_iso8601",' +'"resp_body_size": $body_bytes_sent,' +'"host": "$http_host",' +'"address": "$remote_addr",' +'"request_length": $request_length,' +'"method": "$request_method",' +'"uri": "$request_uri",' +'"status": $status,' +'"user_agent": "$http_user_agent",' +'"resp_time": $request_time,' +'"upstream_addr": "$upstream_addr"' +'}'; +access_log /var/log/nginx/access.log logger-json; + +server { + listen 3003; + + # DNS server resolved addreses as to + resolver 127.0.0.1; + resolver_timeout 5s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $conn_upgrade; + + proxy_hide_header x-frame-options; + + proxy_http_version 1.1; + + client_body_timeout 86400s; + client_header_timeout 5s; + + proxy_read_timeout 600s; + proxy_send_timeout 86400s; + + proxy_cache_bypass 1; + proxy_no_cache 1; + + client_max_body_size 1024m; + + proxy_buffering off; + proxy_request_buffering off; + + tcp_nodelay on; + tcp_nopush on; + sendfile on; + + # send_timeout 600s; + + proxy_connect_timeout 3s; + keepalive_requests 2048; + keepalive_timeout 600s; + # gzip off; + + location / { + if ($dbk_session_id = "") { + return 502 "Cannot connect to sandbox"; + } + + proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri; + } +} + +server { + listen 3004; + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } + + location /status { + access_log off; + stub_status; + allow all; + } +} \ No newline at end of file diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl index 94965c6b0..4b8331ea6 100644 --- a/packages/nomad/session-proxy.hcl +++ b/packages/nomad/session-proxy.hcl @@ -18,17 +18,20 @@ variable "session_proxy_service_name" { type = string } +variable "load_balancer_conf" { + type = string +} + +variable "nginx_conf" { + type = string +} + job "session-proxy" { type = "system" datacenters = [var.gcp_zone] priority = 80 - // TODO: Removable - constraint { - operator = "distinct_hosts" - value = "true" - } group "session-proxy" { network { @@ -62,12 +65,11 @@ job "session-proxy" { driver = "docker" config { - // TODO: Fixate version - image = "nginx" + image = "nginx:1.27.0" network_mode = "host" ports = [var.session_proxy_port_name, "status"] volumes = [ - "local:/etc/nginx/conf.d", + "local:/etc/nginx/", "/var/log/session-proxy:/var/log/nginx" ] } @@ -82,110 +84,19 @@ job "session-proxy" { template { left_delimiter = "[[" right_delimiter = "]]" - destination = "local/load-balancer.conf" + data = var.load_balancer_conf + destination = "local/conf.d/load-balancer.conf" change_mode = "signal" change_signal = "SIGHUP" - data = <\d+)-" ":$p"; -} - -map $host $dbk_session_id { - default ""; - "~-(?\w+)-" $s; -} - -map $http_upgrade $conn_upgrade { - default ""; - "websocket" "Upgrade"; -} - -log_format logger-json escape=json -'{' -'"source": "session-proxy",' -'"time": "$time_iso8601",' -'"resp_body_size": $body_bytes_sent,' -'"host": "$http_host",' -'"address": "$remote_addr",' -'"request_length": $request_length,' -'"method": "$request_method",' -'"uri": "$request_uri",' -'"status": $status,' -'"user_agent": "$http_user_agent",' -'"resp_time": $request_time,' -'"upstream_addr": "$upstream_addr"' -'}'; -access_log /var/log/nginx/access.log logger-json; - -server { - listen 3003; - - # DNS server resolved addreses as to - resolver 127.0.0.1 valid=2s; - resolver_timeout 5s; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $conn_upgrade; - - proxy_hide_header x-frame-options; - - proxy_http_version 1.1; - - client_body_timeout 86400s; - client_header_timeout 5s; - - proxy_read_timeout 600s; - proxy_send_timeout 86400s; - - proxy_cache_bypass 1; - proxy_no_cache 1; - - client_max_body_size 1024m; - - proxy_buffering off; - proxy_request_buffering off; - - tcp_nodelay on; - tcp_nopush on; - sendfile on; - - # send_timeout 600s; - - # proxy_connect_timeout 30s; - keepalive_requests 2048; - keepalive_timeout 600s; - # keepalive_time 86400s; - # gzip off; - - location / { - if ($dbk_session_id = "") { - return 502 "Cannot connect to sandbox"; - } - - proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri; - } -} - -server { - listen 3004; - - location /health { - access_log off; - add_header 'Content-Type' 'application/json'; - return 200 '{"status":"UP"}'; - } + } - location /status { - access_log off; - stub_status; - allow all; - } -} -EOF + template { + left_delimiter = "[[" + right_delimiter = "]]" + data = var.nginx_conf + destination = "local/nginx.conf" + change_mode = "signal" + change_signal = "SIGHUP" } } } diff --git a/packages/orchestrator/internal/dns/server.go b/packages/orchestrator/internal/dns/server.go index 47f64eff3..29614abb8 100644 --- a/packages/orchestrator/internal/dns/server.go +++ b/packages/orchestrator/internal/dns/server.go @@ -10,7 +10,7 @@ import ( resolver "github.com/miekg/dns" ) -const ttl = 2 +const ttl = 0 type DNS struct { records *smap.Map[string] diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go index 36caae343..8c86481d5 100644 --- a/packages/orchestrator/internal/server/sandboxes.go +++ b/packages/orchestrator/internal/server/sandboxes.go @@ -117,10 +117,13 @@ func (s *server) Delete(ctx context.Context, in *orchestrator.SandboxRequest) (* attribute.String("env.kernel.version", sbx.Sandbox.KernelVersion), ) + // Don't allow connecting to the sandbox anymore. + s.dns.Remove(in.SandboxID) + sbx.Stop(ctx, s.tracer) // Ensure the sandbox is removed from cache. - // Ideally we would rely only on the goroutine defef. + // Ideally we would rely only on the goroutine defer. s.sandboxes.Remove(in.SandboxID) return nil, nil