feat(nginx): cache by blob sha instead of uri (coreweave#34)

* enable caching by sha for blobs

* add request method; disable head conversion

* add cache key header, use original cache key var in redirects

entrypoint.sh

@@ -153,6 +153,7 @@ echo -n "" >/etc/nginx/nginx.manifest.caching.config.conf
     # First tier caching of manifests; configure via MANIFEST_CACHE_PRIMARY_REGEX and MANIFEST_CACHE_PRIMARY_TIME
     location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_PRIMARY_REGEX} {
         set \$docker_proxy_request_type "manifest-primary";
+        set \$cache_key \$uri;
         proxy_cache_valid ${MANIFEST_CACHE_PRIMARY_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -162,6 +163,7 @@ EOD
     # Secondary tier caching of manifests; configure via MANIFEST_CACHE_SECONDARY_REGEX and MANIFEST_CACHE_SECONDARY_TIME
     location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_SECONDARY_REGEX} {
         set \$docker_proxy_request_type "manifest-secondary";
+        set \$cache_key \$uri;
         proxy_cache_valid ${MANIFEST_CACHE_SECONDARY_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -171,6 +173,7 @@ EOD
     # Default tier caching for manifests. Caches for ${MANIFEST_CACHE_DEFAULT_TIME} (from MANIFEST_CACHE_DEFAULT_TIME)
     location ~ ^/v2/(.*)/manifests/ {
         set \$docker_proxy_request_type "manifest-default";
+        set \$cache_key \$uri;
         proxy_cache_valid ${MANIFEST_CACHE_DEFAULT_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -180,6 +183,7 @@ EOD
     # Manifest caching is disabled. Enable it with ENABLE_MANIFEST_CACHE=true
     location ~ ^/v2/(.*)/manifests/ {
         set \$docker_proxy_request_type "manifest-default-disabled";
+        set \$cache_key \$uri;
         proxy_cache_valid 0s;
         include "/etc/nginx/nginx.manifest.stale.conf";
     }

nginx.conf

@@ -252,6 +252,9 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
         proxy_ignore_client_abort on;
         proxy_cache_revalidate on;
 
+        # Avoid conversion of HEAD method to GET
+        proxy_cache_convert_head off;
+
         # Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
         proxy_hide_header      Set-Cookie;
         proxy_ignore_headers   X-Accel-Expires Expires Cache-Control Set-Cookie;
@@ -278,13 +281,15 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
         # For blob requests by digest, do cache, and treat redirects.
         location ~ ^/v2/(.*)/blobs/sha256:(.*) {
             set $docker_proxy_request_type "blob-by-digest";
+            set $cache_key $request_method$2;
             include "/etc/nginx/nginx.manifest.common.conf";
         }
 
         # For manifest requests by digest, do cache, and treat redirects.
         # These are some of the requests that DockerHub will throttle.
         location ~ ^/v2/(.*)/manifests/sha256:(.*) {
             set $docker_proxy_request_type "manifest-by-digest";
+            set $cache_key $request_method$uri;
             include "/etc/nginx/nginx.manifest.common.conf";
         }
 
@@ -297,6 +302,7 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
         # Since these are mutable, we invalidate them immediately and keep them only in case the backend is down
         location ~ ^/v2/(.*)/blobs/ {
             set $docker_proxy_request_type "blob-mutable";
+            set $cache_key $request_method$uri;
             proxy_cache_valid 0s;
             include "/etc/nginx/nginx.manifest.stale.conf";
         }
@@ -322,7 +328,8 @@ echo "Docker configured with HTTPS_PROXY=$scheme://$http_host/"
             proxy_cache $cache;
             # But we store the result with the cache key of the original request URI
             # so that future clients don't need to follow the redirect too
-            proxy_cache_key $original_uri$slice_range;
+            proxy_cache_key $cache_key$slice_range;
+            add_header X-Docker-Registry-Proxy-Cache-Key-Status "$cache_key$slice_range";
         }
 
         # by default, dont cache anything.

nginx.manifest.common.conf

@@ -4,8 +4,9 @@
     proxy_pass https://$targetHost;
     proxy_cache $cache;
     slice 4m;
-    proxy_cache_key   $uri$slice_range;
+    proxy_cache_key   $cache_key$slice_range;
     proxy_set_header   Range $slice_range;
+    add_header X-Docker-Registry-Proxy-Cache-Key-Status "$cache_key$slice_range";
     proxy_http_version 1.1;
     proxy_intercept_errors on;
     error_page 301 302 307 = @handle_redirects;