From 818a71165b1e839c9a1f90b457dc651b8a96fc22 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Mon, 1 Jul 2024 14:14:15 +0200
Subject: [PATCH 01/12] Add keepalive time

---
 packages/nomad/client-proxy.hcl  | 42 +++++++++++++++++++++++++++++
 packages/nomad/session-proxy.hcl | 46 ++++++++++++++++++++++++++++++--
 2 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/packages/nomad/client-proxy.hcl b/packages/nomad/client-proxy.hcl
index c1f0e46e6..efa72c38b 100644
--- a/packages/nomad/client-proxy.hcl
+++ b/packages/nomad/client-proxy.hcl
@@ -182,6 +182,48 @@ server {
 }
 EOF
       }
+
+
+      template {
+        left_delimiter  = "[["
+        right_delimiter = "]]"
+        destination     = "local/nginx.conf"
+        change_mode     = "signal"
+        change_signal   = "SIGHUP"
+        data            = <<EOF
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+    keepalive_time     86400s;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+EOF
     }
   }
 }
diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index 94965c6b0..3a7d8bda9 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -67,7 +67,7 @@ job "session-proxy" {
         network_mode = "host"
         ports        = [var.session_proxy_port_name, "status"]
         volumes = [
-          "local:/etc/nginx/conf.d",
+          "local:/etc/nginx/",
           "/var/log/session-proxy:/var/log/nginx"
         ]
       }
@@ -82,7 +82,7 @@ job "session-proxy" {
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
-        destination     = "local/load-balancer.conf"
+        destination     = "local/conf.d/load-balancer.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
         data            = <<EOF
@@ -185,6 +185,48 @@ server {
     allow all;
   }
 }
+EOF
+      }
+
+      template {
+        left_delimiter  = "[["
+        right_delimiter = "]]"
+        destination     = "local/nginx.conf"
+        change_mode     = "signal"
+        change_signal   = "SIGHUP"
+        data            = <<EOF
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+    keepalive_time     86400s;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
 EOF
       }
     }

From b463d5f50568b9eafb7ef17b2a37c08b915a9848 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 16:04:08 +0200
Subject: [PATCH 02/12] Decrease connect timeout

---
 packages/nomad/session-proxy.hcl | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index 3a7d8bda9..a6b181d6d 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -24,11 +24,6 @@ job "session-proxy" {
 
   priority = 80
 
-  // TODO: Removable
-  constraint {
-    operator = "distinct_hosts"
-    value    = "true"
-  }
 
   group "session-proxy" {
     network {
@@ -62,8 +57,7 @@ job "session-proxy" {
       driver = "docker"
 
       config {
-        // TODO: Fixate version
-        image        = "nginx"
+        image        = "nginx:1.27.0"
         network_mode = "host"
         ports        = [var.session_proxy_port_name, "status"]
         volumes = [
@@ -72,10 +66,9 @@ job "session-proxy" {
         ]
       }
 
-      // TODO: Saner resources
       resources {
-        memory_max = 6000
-        memory = 6000
+        memory_max = 2048
+        memory = 1024
         cpu    = 1024
       }
 
@@ -145,7 +138,7 @@ server {
   proxy_no_cache 1;
 
   client_max_body_size 1024m;
-  
+
   proxy_buffering off;
   proxy_request_buffering off;
 
@@ -153,10 +146,11 @@ server {
   tcp_nopush on;
   sendfile on;
 
-  # send_timeout                600s;
+  send_timeout                600s;
+
+  proxy_connect_timeout       3s;
 
-  # proxy_connect_timeout       30s;
-  keepalive_requests 2048;
+  keepalive_requests 65536;
   keepalive_timeout 600s;
   # keepalive_time 86400s;
   # gzip off;

From b75b70994ec2a8bfdeda8aa961a9cc0afeab1325 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 16:08:57 +0200
Subject: [PATCH 03/12] Remove DNS caching in session proxy

---
 packages/nomad/session-proxy.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index a6b181d6d..ac11c312a 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -115,7 +115,7 @@ server {
   listen 3003;
 
   # DNS server resolved addreses as to <sandbox-id> <ip-address>
-  resolver 127.0.0.1 valid=2s;
+  resolver 127.0.0.1;
   resolver_timeout 5s;
 
   proxy_set_header Host $host;

From c910a9c8783dfe98a3ff5ce4defe8d8c81c76fe8 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 16:09:26 +0200
Subject: [PATCH 04/12] Remove DNS immediately when deleting sandbox

---
 packages/orchestrator/internal/server/sandboxes.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go
index 36caae343..8c86481d5 100644
--- a/packages/orchestrator/internal/server/sandboxes.go
+++ b/packages/orchestrator/internal/server/sandboxes.go
@@ -117,10 +117,13 @@ func (s *server) Delete(ctx context.Context, in *orchestrator.SandboxRequest) (*
 		attribute.String("env.kernel.version", sbx.Sandbox.KernelVersion),
 	)
 
+	// Don't allow connecting to the sandbox anymore.
+	s.dns.Remove(in.SandboxID)
+
 	sbx.Stop(ctx, s.tracer)
 
 	// Ensure the sandbox is removed from cache.
-	// Ideally we would rely only on the goroutine defef.
+	// Ideally we would rely only on the goroutine defer.
 	s.sandboxes.Remove(in.SandboxID)
 
 	return nil, nil

From b2588da15825f1d56c837e27eda22cf912c99b33 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 16:51:46 +0200
Subject: [PATCH 05/12] Decrease ttl of orchestrator DNS to 0

---
 packages/orchestrator/internal/dns/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/orchestrator/internal/dns/server.go b/packages/orchestrator/internal/dns/server.go
index 47f64eff3..29614abb8 100644
--- a/packages/orchestrator/internal/dns/server.go
+++ b/packages/orchestrator/internal/dns/server.go
@@ -10,7 +10,7 @@ import (
 	resolver "github.com/miekg/dns"
 )
 
-const ttl = 2
+const ttl = 0
 
 type DNS struct {
 	records *smap.Map[string]

From 2e40f0530667caec257b650f9fb624d0a8de10d1 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 16:57:19 +0200
Subject: [PATCH 06/12] Refactor nginx configs to files

# Conflicts:
#	packages/nomad/client-proxy.hcl
#	packages/nomad/session-proxy.hcl
---
 packages/nomad/client-proxy.hcl     | 145 +++-------------------------
 packages/nomad/main.tf              |   4 +
 packages/nomad/proxies/client.conf  |  91 +++++++++++++++++
 packages/nomad/proxies/nginx.conf   |  33 +++++++
 packages/nomad/proxies/session.conf | 100 +++++++++++++++++++
 packages/nomad/session-proxy.hcl    |  44 ++-------
 6 files changed, 252 insertions(+), 165 deletions(-)
 create mode 100644 packages/nomad/proxies/client.conf
 create mode 100644 packages/nomad/proxies/nginx.conf
 create mode 100644 packages/nomad/proxies/session.conf

diff --git a/packages/nomad/client-proxy.hcl b/packages/nomad/client-proxy.hcl
index efa72c38b..bd502775f 100644
--- a/packages/nomad/client-proxy.hcl
+++ b/packages/nomad/client-proxy.hcl
@@ -30,6 +30,14 @@ variable "domain_name" {
   type = string
 }
 
+variable "load_balancer_conf" {
+  type = string
+}
+
+variable "nginx_conf" {
+  type = string
+}
+
 locals {
   domain_name_escaped = replace(var.domain_name, ".", "\\.")
 }
@@ -73,12 +81,11 @@ job "client-proxy" {
       }
 
       config {
-        // TODO: Fixate versionx
-        image        = "nginx"
+        image        = "nginx:1.27.0"
         network_mode = "host"
         ports        = [var.client_proxy_health_port_name, var.client_proxy_port_name]
         volumes = [
-          "local:/etc/nginx/conf.d",
+          "local:/etc/nginx",
           "/var/log/client-proxy:/var/log/nginx"
         ]
       }
@@ -86,144 +93,20 @@ job "client-proxy" {
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
-        destination     = "local/load-balancer.conf"
+        data            = var.load_balancer_conf
+        destination     = "local/conf.d/load-balancer.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
-        data            = <<EOF
-map $http_upgrade $conn_upgrade {
-  default     "";
-  "websocket" "Upgrade";
-}
-
-log_format logger-json escape=json
-'{'
-'"source": "client-proxy",'
-'"time": "$time_iso8601",'
-'"resp_body_size": $body_bytes_sent,'
-'"host": "$http_host",'
-'"address": "$remote_addr",'
-'"request_length": $request_length,'
-'"method": "$request_method",'
-'"uri": "$request_uri",'
-'"status": $status,'
-'"user_agent": "$http_user_agent",'
-'"resp_time": $request_time,'
-'"upstream_addr": "$upstream_addr"'
-'}';
-access_log /var/log/nginx/access.log logger-json;
-
-server {
-  listen 3002 default_server;
-
-  server_name _;
-  return 502 "Cannot connect to sandbox";
-}
-[[ range service "session-proxy" ]]
-server {
-  listen 3002;
-  access_log /var/log/nginx/access.log logger-json;
-
-  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${local.domain_name_escaped}$;
-
-  proxy_set_header Host $host;
-  proxy_set_header X-Real-IP $remote_addr;
-
-  proxy_set_header Upgrade $http_upgrade;
-  proxy_set_header Connection $conn_upgrade;
-
-  proxy_hide_header x-frame-options;
-
-  proxy_http_version 1.1;
-
-  client_body_timeout 86400s;
-  client_header_timeout 5s;
-
-  # proxy_read_timeout 600s;
-  proxy_read_timeout 1d;
-  proxy_send_timeout 86400s;
-
-  proxy_cache_bypass 1;
-  proxy_no_cache 1;
-
-  client_max_body_size 1024m;
-
-  proxy_buffering off;
-  proxy_request_buffering off;
-  
-  tcp_nodelay on;
-  tcp_nopush on;
-  sendfile on;
-  # send_timeout                600s;
-  # proxy_connect_timeout       30s;
-
-  keepalive_timeout 600s;
-  keepalive_requests 2048;
-  # keepalive_time 86400s;
-
-  # gzip off;
-  location / {
-    proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;
-  }
-}
-[[ end ]]
-server {
-  listen 3001;
-  location /health {
-    access_log off;
-    add_header 'Content-Type' 'application/json';
-    return 200 '{"status":"UP"}';
-  }
-
-  location /status {
-    access_log off;
-    stub_status;
-    allow all;
-  }
-}
-EOF
       }
 
-
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
+        data            = var.nginx_conf
         destination     = "local/nginx.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
-        data            = <<EOF
-
-user  nginx;
-worker_processes  auto;
-
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
-
-
-events {
-    worker_connections  1024;
-}
-
-
-http {
-    default_type  application/octet-stream;
-
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log  /var/log/nginx/access.log  main;
-
-    sendfile        on;
-    #tcp_nopush     on;
-
-    keepalive_timeout  65;
-    keepalive_time     86400s;
-
-    #gzip  on;
-
-    include /etc/nginx/conf.d/*.conf;
-}
-EOF
+      }
     }
   }
 }
diff --git a/packages/nomad/main.tf b/packages/nomad/main.tf
index a519a30c4..ff6da06b1 100644
--- a/packages/nomad/main.tf
+++ b/packages/nomad/main.tf
@@ -111,6 +111,8 @@ resource "nomad_job" "client_proxy" {
       client_proxy_health_port_path   = var.client_proxy_health_port.path
       session_proxy_service_name      = var.session_proxy_service_name
       domain_name                     = var.domain_name
+      load_balancer_conf              = file("${path.module}/proxies/client.conf")
+      nginx_conf                      = file("${path.module}/proxies/nginx.conf")
     }
   }
 }
@@ -125,6 +127,8 @@ resource "nomad_job" "session_proxy" {
       session_proxy_port_number  = var.session_proxy_port.port
       session_proxy_port_name    = var.session_proxy_port.name
       session_proxy_service_name = var.session_proxy_service_name
+      load_balancer_conf         = file("${path.module}/proxies/session.conf")
+      nginx_conf                 = file("${path.module}/proxies/nginx.conf")
     }
   }
 }
diff --git a/packages/nomad/proxies/client.conf b/packages/nomad/proxies/client.conf
new file mode 100644
index 000000000..9259e1ae9
--- /dev/null
+++ b/packages/nomad/proxies/client.conf
@@ -0,0 +1,91 @@
+map $http_upgrade $conn_upgrade {
+  default     "";
+  "websocket" "Upgrade";
+}
+
+log_format logger-json escape=json
+'{'
+'"source": "client-proxy",'
+'"time": "$time_iso8601",'
+'"resp_body_size": $body_bytes_sent,'
+'"host": "$http_host",'
+'"address": "$remote_addr",'
+'"request_length": $request_length,'
+'"method": "$request_method",'
+'"uri": "$request_uri",'
+'"status": $status,'
+'"user_agent": "$http_user_agent",'
+'"resp_time": $request_time,'
+'"upstream_addr": "$upstream_addr"'
+'}';
+access_log /var/log/nginx/access.log logger-json;
+
+server {
+  listen 3002 default_server;
+
+  server_name _;
+  return 502 "Cannot resolve domain";
+}
+[[ range service "session-proxy" ]]
+server {
+  listen 3002;
+  access_log /var/log/nginx/access.log logger-json;
+
+  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${local.domain_name_escaped}$;
+
+  proxy_set_header Host $host;
+  proxy_set_header X-Real-IP $remote_addr;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $conn_upgrade;
+
+  proxy_hide_header x-frame-options;
+
+  proxy_http_version 1.1;
+
+  client_body_timeout 86400s;
+  client_header_timeout 5s;
+
+  proxy_read_timeout 600s;
+  proxy_send_timeout 86400s;
+
+  proxy_cache_bypass 1;
+  proxy_no_cache 1;
+
+  client_max_body_size 1024m;
+
+  proxy_buffering off;
+  proxy_request_buffering off;
+
+  tcp_nodelay on;
+  tcp_nopush on;
+  sendfile on;
+
+  send_timeout                600s;
+
+  proxy_connect_timeout       30s;
+
+  keepalive_requests 65536;
+  keepalive_timeout 600s;
+
+  gzip off;
+
+  location / {
+    proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;
+  }
+}
+[[ end ]]
+server {
+  listen 3001;
+  location /health {
+    access_log off;
+    add_header 'Content-Type' 'application/json';
+    return 200 '{"status":"UP"}';
+  }
+
+  location /status {
+    access_log off;
+    stub_status;
+    allow all;
+  }
+}
\ No newline at end of file
diff --git a/packages/nomad/proxies/nginx.conf b/packages/nomad/proxies/nginx.conf
new file mode 100644
index 000000000..682ac4573
--- /dev/null
+++ b/packages/nomad/proxies/nginx.conf
@@ -0,0 +1,33 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+    keepalive_time     86400s;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+
+
diff --git a/packages/nomad/proxies/session.conf b/packages/nomad/proxies/session.conf
new file mode 100644
index 000000000..247d11fec
--- /dev/null
+++ b/packages/nomad/proxies/session.conf
@@ -0,0 +1,100 @@
+map $host $dbk_port {
+  default         "";
+  "~^(?<p>\d+)-"  ":$p";
+}
+
+map $host $dbk_session_id {
+  default         "";
+  "~-(?<s>\w+)-"  $s;
+}
+
+map $http_upgrade $conn_upgrade {
+  default     "";
+  "websocket" "Upgrade";
+}
+
+log_format logger-json escape=json
+'{'
+'"source": "session-proxy",'
+'"time": "$time_iso8601",'
+'"resp_body_size": $body_bytes_sent,'
+'"host": "$http_host",'
+'"address": "$remote_addr",'
+'"request_length": $request_length,'
+'"method": "$request_method",'
+'"uri": "$request_uri",'
+'"status": $status,'
+'"user_agent": "$http_user_agent",'
+'"resp_time": $request_time,'
+'"upstream_addr": "$upstream_addr"'
+'}';
+access_log /var/log/nginx/access.log logger-json;
+
+server {
+  listen 3003;
+
+  # DNS server resolved addreses as to <sandbox-id> <ip-address>
+  resolver 127.0.0.1;
+  resolver_timeout 5s;
+
+  proxy_set_header Host $host;
+  proxy_set_header X-Real-IP $remote_addr;
+
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $conn_upgrade;
+
+  proxy_hide_header x-frame-options;
+
+  proxy_http_version 1.1;
+
+  client_body_timeout 86400s;
+  client_header_timeout 5s;
+
+  proxy_read_timeout 600s;
+  proxy_send_timeout 86400s;
+
+  proxy_cache_bypass 1;
+  proxy_no_cache 1;
+
+  client_max_body_size 1024m;
+
+  proxy_buffering off;
+  proxy_request_buffering off;
+
+  tcp_nodelay on;
+  tcp_nopush on;
+  sendfile on;
+
+  send_timeout                600s;
+
+  proxy_connect_timeout       3s;
+
+  keepalive_requests 65536;
+  keepalive_timeout 600s;
+
+  gzip off;
+
+  location / {
+    if ($dbk_session_id = "") {
+      return 502 "Missing sandbox domain";
+    }
+
+    proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri;
+  }
+}
+
+server {
+  listen 3004;
+
+  location /health {
+    access_log off;
+    add_header 'Content-Type' 'application/json';
+    return 200 '{"status":"UP"}';
+  }
+
+  location /status {
+    access_log off;
+    stub_status;
+    allow all;
+  }
+}
\ No newline at end of file
diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index ac11c312a..e95df1eed 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -18,6 +18,14 @@ variable "session_proxy_service_name" {
   type = string
 }
 
+variable "load_balancer_conf" {
+  type = string
+}
+
+variable "nginx_conf" {
+  type = string
+}
+
 job "session-proxy" {
   type = "system"
   datacenters = [var.gcp_zone]
@@ -75,6 +83,7 @@ job "session-proxy" {
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
+        data            = var.load_balancer_conf
         destination     = "local/conf.d/load-balancer.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
@@ -185,43 +194,10 @@ EOF
       template {
         left_delimiter  = "[["
         right_delimiter = "]]"
+        data            = var.nginx_conf
         destination     = "local/nginx.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
-        data            = <<EOF
-
-user  nginx;
-worker_processes  auto;
-
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
-
-
-events {
-    worker_connections  1024;
-}
-
-
-http {
-    default_type  application/octet-stream;
-
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log  /var/log/nginx/access.log  main;
-
-    sendfile        on;
-    #tcp_nopush     on;
-
-    keepalive_timeout  65;
-    keepalive_time     86400s;
-
-    #gzip  on;
-
-    include /etc/nginx/conf.d/*.conf;
-}
-EOF
       }
     }
   }

From 6cc16bae533cdd075c19d02f4b6e5293a0073c1e Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 17:02:53 +0200
Subject: [PATCH 07/12] Refactor nginx configs to files

---
 packages/nomad/proxies/client.conf  | 17 ++++++++---------
 packages/nomad/proxies/session.conf | 13 ++++++-------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/packages/nomad/proxies/client.conf b/packages/nomad/proxies/client.conf
index 9259e1ae9..2bdd0b085 100644
--- a/packages/nomad/proxies/client.conf
+++ b/packages/nomad/proxies/client.conf
@@ -24,7 +24,7 @@ server {
   listen 3002 default_server;
 
   server_name _;
-  return 502 "Cannot resolve domain";
+  return 502 "Cannot connect to sandbox";
 }
 [[ range service "session-proxy" ]]
 server {
@@ -46,7 +46,8 @@ server {
   client_body_timeout 86400s;
   client_header_timeout 5s;
 
-  proxy_read_timeout 600s;
+  # proxy_read_timeout 600s;
+  proxy_read_timeout 1d;
   proxy_send_timeout 86400s;
 
   proxy_cache_bypass 1;
@@ -60,16 +61,14 @@ server {
   tcp_nodelay on;
   tcp_nopush on;
   sendfile on;
+  # send_timeout                600s;
+  # proxy_connect_timeout       30s;
 
-  send_timeout                600s;
-
-  proxy_connect_timeout       30s;
-
-  keepalive_requests 65536;
   keepalive_timeout 600s;
+  keepalive_requests 2048;
+  # keepalive_time 86400s;
 
-  gzip off;
-
+  # gzip off;
   location / {
     proxy_pass $scheme://[[ .Address ]]:[[ .Port ]]$request_uri;
   }
diff --git a/packages/nomad/proxies/session.conf b/packages/nomad/proxies/session.conf
index 247d11fec..cc90d2545 100644
--- a/packages/nomad/proxies/session.conf
+++ b/packages/nomad/proxies/session.conf
@@ -65,18 +65,17 @@ server {
   tcp_nopush on;
   sendfile on;
 
-  send_timeout                600s;
+  # send_timeout                600s;
 
-  proxy_connect_timeout       3s;
-
-  keepalive_requests 65536;
+  proxy_connect_timeout 3s;
+  keepalive_requests 2048;
   keepalive_timeout 600s;
-
-  gzip off;
+  # keepalive_time 86400s;
+  # gzip off;
 
   location / {
     if ($dbk_session_id = "") {
-      return 502 "Missing sandbox domain";
+      return 502 "Cannot connect to sandbox";
     }
 
     proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri;

From 51e4879682eaa69e22626194a066830795a0c4ed Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 17:04:23 +0200
Subject: [PATCH 08/12] Remove already added confs

---
 packages/nomad/proxies/client.conf  | 1 -
 packages/nomad/proxies/session.conf | 1 -
 2 files changed, 2 deletions(-)

diff --git a/packages/nomad/proxies/client.conf b/packages/nomad/proxies/client.conf
index 2bdd0b085..eca798177 100644
--- a/packages/nomad/proxies/client.conf
+++ b/packages/nomad/proxies/client.conf
@@ -66,7 +66,6 @@ server {
 
   keepalive_timeout 600s;
   keepalive_requests 2048;
-  # keepalive_time 86400s;
 
   # gzip off;
   location / {
diff --git a/packages/nomad/proxies/session.conf b/packages/nomad/proxies/session.conf
index cc90d2545..2e15b378a 100644
--- a/packages/nomad/proxies/session.conf
+++ b/packages/nomad/proxies/session.conf
@@ -70,7 +70,6 @@ server {
   proxy_connect_timeout 3s;
   keepalive_requests 2048;
   keepalive_timeout 600s;
-  # keepalive_time 86400s;
   # gzip off;
 
   location / {

From 94ba7019dfa66ed4cf82c2a8e0e972c15962a467 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 17:05:06 +0200
Subject: [PATCH 09/12] Remove `keepalive_timeout` from nginx.conf

---
 packages/nomad/proxies/nginx.conf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/nomad/proxies/nginx.conf b/packages/nomad/proxies/nginx.conf
index 682ac4573..30017188d 100644
--- a/packages/nomad/proxies/nginx.conf
+++ b/packages/nomad/proxies/nginx.conf
@@ -22,7 +22,6 @@ http {
     sendfile        on;
     #tcp_nopush     on;
 
-    keepalive_timeout  65;
     keepalive_time     86400s;
 
     #gzip  on;

From 50388dde9b4700a72f88eea839e65f51ebd14ed9 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Thu, 18 Jul 2024 17:05:37 +0200
Subject: [PATCH 10/12] Remove extra data

---
 packages/nomad/session-proxy.hcl | 102 -------------------------------
 1 file changed, 102 deletions(-)

diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index e95df1eed..3695441e0 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -87,108 +87,6 @@ job "session-proxy" {
         destination     = "local/conf.d/load-balancer.conf"
         change_mode     = "signal"
         change_signal   = "SIGHUP"
-        data            = <<EOF
-map $host $dbk_port {
-  default         "";
-  "~^(?<p>\d+)-"  ":$p";
-}
-
-map $host $dbk_session_id {
-  default         "";
-  "~-(?<s>\w+)-"  $s;
-}
-
-map $http_upgrade $conn_upgrade {
-  default     "";
-  "websocket" "Upgrade";
-}
-
-log_format logger-json escape=json
-'{'
-'"source": "session-proxy",'
-'"time": "$time_iso8601",'
-'"resp_body_size": $body_bytes_sent,'
-'"host": "$http_host",'
-'"address": "$remote_addr",'
-'"request_length": $request_length,'
-'"method": "$request_method",'
-'"uri": "$request_uri",'
-'"status": $status,'
-'"user_agent": "$http_user_agent",'
-'"resp_time": $request_time,'
-'"upstream_addr": "$upstream_addr"'
-'}';
-access_log /var/log/nginx/access.log logger-json;
-
-server {
-  listen 3003;
-
-  # DNS server resolved addreses as to <sandbox-id> <ip-address>
-  resolver 127.0.0.1;
-  resolver_timeout 5s;
-
-  proxy_set_header Host $host;
-  proxy_set_header X-Real-IP $remote_addr;
-
-  proxy_set_header Upgrade $http_upgrade;
-  proxy_set_header Connection $conn_upgrade;
-
-  proxy_hide_header x-frame-options;
-
-  proxy_http_version 1.1;
-
-  client_body_timeout 86400s;
-  client_header_timeout 5s;
-
-  proxy_read_timeout 600s;
-  proxy_send_timeout 86400s;
-
-  proxy_cache_bypass 1;
-  proxy_no_cache 1;
-
-  client_max_body_size 1024m;
-
-  proxy_buffering off;
-  proxy_request_buffering off;
-
-  tcp_nodelay on;
-  tcp_nopush on;
-  sendfile on;
-
-  send_timeout                600s;
-
-  proxy_connect_timeout       3s;
-
-  keepalive_requests 65536;
-  keepalive_timeout 600s;
-  # keepalive_time 86400s;
-  # gzip off;
-
-  location / {
-    if ($dbk_session_id = "") {
-      return 502 "Cannot connect to sandbox";
-    }
-
-    proxy_pass $scheme://$dbk_session_id$dbk_port$request_uri;
-  }
-}
-
-server {
-  listen 3004;
-
-  location /health {
-    access_log off;
-    add_header 'Content-Type' 'application/json';
-    return 200 '{"status":"UP"}';
-  }
-
-  location /status {
-    access_log off;
-    stub_status;
-    allow all;
-  }
-}
-EOF
       }
 
       template {

From b6d52c9301a6708d7c1d6eaf3ab858cbb9207868 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Fri, 19 Jul 2024 14:21:19 +0200
Subject: [PATCH 11/12] Revert resources

---
 packages/nomad/session-proxy.hcl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/nomad/session-proxy.hcl b/packages/nomad/session-proxy.hcl
index 3695441e0..4b8331ea6 100644
--- a/packages/nomad/session-proxy.hcl
+++ b/packages/nomad/session-proxy.hcl
@@ -74,9 +74,10 @@ job "session-proxy" {
         ]
       }
 
+      // TODO: Saner resources
       resources {
-        memory_max = 2048
-        memory = 1024
+        memory_max = 6000
+        memory = 6000
         cpu    = 1024
       }
 

From 8c85409c0aa9525c8ff7fc9cdf72c9425468cc94 Mon Sep 17 00:00:00 2001
From: Jakub Novak <jakub@e2b.dev>
Date: Fri, 19 Jul 2024 16:03:52 +0200
Subject: [PATCH 12/12] Fix missing template variable

---
 packages/nomad/client-proxy.hcl    | 10 +---------
 packages/nomad/main.tf             |  5 +++--
 packages/nomad/proxies/client.conf |  2 +-
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/packages/nomad/client-proxy.hcl b/packages/nomad/client-proxy.hcl
index bd502775f..eeedfc7e4 100644
--- a/packages/nomad/client-proxy.hcl
+++ b/packages/nomad/client-proxy.hcl
@@ -26,10 +26,6 @@ variable "session_proxy_service_name" {
   type = string
 }
 
-variable "domain_name" {
-  type = string
-}
-
 variable "load_balancer_conf" {
   type = string
 }
@@ -38,10 +34,6 @@ variable "nginx_conf" {
   type = string
 }
 
-locals {
-  domain_name_escaped = replace(var.domain_name, ".", "\\.")
-}
-
 job "client-proxy" {
   datacenters = [var.gcp_zone]
 
@@ -85,7 +77,7 @@ job "client-proxy" {
         network_mode = "host"
         ports        = [var.client_proxy_health_port_name, var.client_proxy_port_name]
         volumes = [
-          "local:/etc/nginx",
+          "local:/etc/nginx/",
           "/var/log/client-proxy:/var/log/nginx"
         ]
       }
diff --git a/packages/nomad/main.tf b/packages/nomad/main.tf
index ff6da06b1..86bbc6c25 100644
--- a/packages/nomad/main.tf
+++ b/packages/nomad/main.tf
@@ -110,8 +110,9 @@ resource "nomad_job" "client_proxy" {
       client_proxy_health_port_name   = var.client_proxy_health_port.name
       client_proxy_health_port_path   = var.client_proxy_health_port.path
       session_proxy_service_name      = var.session_proxy_service_name
-      domain_name                     = var.domain_name
-      load_balancer_conf              = file("${path.module}/proxies/client.conf")
+      load_balancer_conf              = templatefile("${path.module}/proxies/client.conf", {
+        domain_name_escaped = replace(var.domain_name, ".", "\\.")
+      })
       nginx_conf                      = file("${path.module}/proxies/nginx.conf")
     }
   }
diff --git a/packages/nomad/proxies/client.conf b/packages/nomad/proxies/client.conf
index eca798177..aa58514e0 100644
--- a/packages/nomad/proxies/client.conf
+++ b/packages/nomad/proxies/client.conf
@@ -31,7 +31,7 @@ server {
   listen 3002;
   access_log /var/log/nginx/access.log logger-json;
 
-  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${local.domain_name_escaped}$;
+  server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.${domain_name_escaped}$;
 
   proxy_set_header Host $host;
   proxy_set_header X-Real-IP $remote_addr;