From ebbd53d50c3e0143007ad4d91d27ab23a7b58d9f Mon Sep 17 00:00:00 2001 From: Josh Sandlin Date: Fri, 31 Jul 2026 14:40:52 -0400 Subject: [PATCH 1/4] redis data to host to defend against losing volume --- docker-compose.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5d75918..5914eea 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,9 +23,6 @@ services: - "127.0.0.1:5000:5000" volumes: - ./zot-config.json:/etc/zot/config.json:ro - # Registry blobs live on the host, not in the container layer. Docker - # creates this directory on first start; zot runs as root so no chown - # dance is needed. Path matches storage.rootDirectory in zot-config.json. - ./zot-data:/var/lib/zot networks: - ttlsh-network @@ -55,7 +52,7 @@ services: restart: always command: ["redis-server", "--appendonly", "yes"] volumes: - - redis-data:/data + - ./redis-data:/data networks: - ttlsh-network healthcheck: @@ -68,6 +65,3 @@ services: networks: ttlsh-network: driver: bridge - -volumes: - redis-data: From ea63b832149f5f79dc2f72525e6c6d78040e8355 Mon Sep 17 00:00:00 2001 From: Josh Sandlin Date: Fri, 31 Jul 2026 14:44:38 -0400 Subject: [PATCH 2/4] move zot and web to upstreams with keepalives and remove websocket adjacent headers --- ansible/templates/nginx_site.conf.j2 | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ansible/templates/nginx_site.conf.j2 b/ansible/templates/nginx_site.conf.j2 index ecc2c55..4dbde6d 100644 --- a/ansible/templates/nginx_site.conf.j2 +++ b/ansible/templates/nginx_site.conf.j2 @@ -17,15 +17,12 @@ server { # Next.js web app location / { - proxy_pass http://127.0.0.1:3000; + proxy_pass http://ttlsh_web; proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; } # Block listing catalog @@ -41,12 +38,25 @@ server { proxy_send_timeout 3600s; proxy_read_timeout 3600s; send_timeout 3600s; - proxy_pass http://127.0.0.1:5000; + proxy_pass http://ttlsh_zot; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $host; + proxy_set_header Connection ""; proxy_request_buffering off; } } + +upstream ttlsh_zot { + server 127.0.0.1:5000; + keepalive 64; + keepalive_timeout 60s; +} + +upstream ttlsh_web { + server 127.0.0.1:3000; + keepalive 32; + keepalive_timeout 60s; +} \ No newline at end of file From d200d60637bcfd4a193a49012b071098b750408f Mon Sep 17 00:00:00 2001 From: Josh Sandlin Date: Fri, 31 Jul 2026 14:45:48 -0400 Subject: [PATCH 3/4] introduce readTimeout and writeTimeout config overrides to handle higher latencies with object storage --- ansible/templates/zot-config.json.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/templates/zot-config.json.j2 b/ansible/templates/zot-config.json.j2 index 62a8787..3fb362c 100644 --- a/ansible/templates/zot-config.json.j2 +++ b/ansible/templates/zot-config.json.j2 @@ -37,7 +37,9 @@ "http": { "address": "0.0.0.0", "port": "5000", - "compat": ["docker2s2"] + "compat": ["docker2s2"], + "readTimeout": "3600s", + "writeTimeout": "3600s" }, "log": { "level": "warn" From b7616f0123100d8043ee30d5bd66a483f74568ae Mon Sep 17 00:00:00 2001 From: Josh Sandlin Date: Fri, 31 Jul 2026 14:47:14 -0400 Subject: [PATCH 4/4] was seeing a lot of timeout retries here when reduced to 5 seconds --- sidecar/internal/registry/registry.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sidecar/internal/registry/registry.go b/sidecar/internal/registry/registry.go index 8bc4f0f..9f35789 100644 --- a/sidecar/internal/registry/registry.go +++ b/sidecar/internal/registry/registry.go @@ -21,11 +21,7 @@ func New(baseURL string) *Client { return &Client{ baseURL: baseURL, http: &http.Client{ - // A manifest delete is a metadata operation against a zot running - // alongside this process, so it should be fast. Failing quickly - // keeps one wedged tag from stalling the rest of the sweep; the row - // stays in the store and is retried on the next tick. - Timeout: 5 * time.Second, + Timeout: 30 * time.Second, }, } }