Merge pull request OSGeo#10730 from rouault/no_cache

/vsicurl/: honor 'Cache-Control: no-cache' header

autotest/gcore/vsicurl.py

@@ -1288,3 +1288,89 @@ def test_vsicurl_404_repeated_same_resource(server):
 
     with pytest.raises(Exception, match="404"):
         gdal.Open("/vsicurl/http://localhost:%d/does/not/exist.bin" % server.port)
+
+
+###############################################################################
+
+
+@gdaltest.enable_exceptions()
+def test_vsicurl_cache_control_not_set(server):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/", 404)
+    handler.add("HEAD", "/test.txt", 200, {"Content-Length": "3"})
+    handler.add("GET", "/test.txt", 200, {}, "foo")
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL(
+            "/vsicurl/http://localhost:%d/test.txt" % server.port,
+            "rb",
+        )
+        assert f is not None
+        data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        gdal.VSIFCloseL(f)
+        assert data == "foo"
+
+    with webserver.install_http_handler(webserver.SequentialHandler()):
+        f = gdal.VSIFOpenL(
+            "/vsicurl/http://localhost:%d/test.txt" % server.port,
+            "rb",
+        )
+        assert f is not None
+        data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        gdal.VSIFCloseL(f)
+        assert data == "foo"
+
+
+###############################################################################
+
+
+@gdaltest.enable_exceptions()
+def test_vsicurl_cache_control_no_cache(server):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/", 404)
+    handler.add(
+        "HEAD", "/test.txt", 200, {"Content-Length": "3", "Cache-Control": "no-cache"}
+    )
+    handler.add(
+        "GET",
+        "/test.txt",
+        200,
+        {"Content-Length": "3", "Cache-Control": "no-cache"},
+        "foo",
+    )
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL(
+            "/vsicurl/http://localhost:%d/test.txt" % server.port,
+            "rb",
+        )
+        assert f is not None
+        data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        gdal.VSIFCloseL(f)
+        assert data == "foo"
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/", 404)
+    handler.add(
+        "HEAD", "/test.txt", 200, {"Content-Length": "6", "Cache-Control": "no-cache"}
+    )
+    handler.add(
+        "GET",
+        "/test.txt",
+        200,
+        {"Content-Length": "6", "Cache-Control": "no-cache"},
+        "barbaz",
+    )
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL(
+            "/vsicurl/http://localhost:%d/test.txt" % server.port,
+            "rb",
+        )
+        assert f is not None
+        data = gdal.VSIFReadL(1, 6, f).decode("ascii")
+        gdal.VSIFCloseL(f)
+        assert data == "barbaz"

autotest/gcore/vsis3.py

@@ -2199,6 +2199,112 @@ def method(request):
     assert gdal.GetLastErrorMsg() == ""
 
 
+###############################################################################
+# Test that PUT invalidates cached data
+
+
+def test_vsis3_put_invalidate(aws_test_config, webserver_port):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
+
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(3, 1, f) == b"foo"
+        finally:
+            gdal.VSIFCloseL(f)
+
+    handler = webserver.SequentialHandler()
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(3, 1, f) == b"foo"
+        finally:
+            gdal.VSIFCloseL(f)
+
+    handler = webserver.SequentialHandler()
+    handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200)
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "wb")
+        assert f is not None
+        try:
+            assert gdal.VSIFWriteL("barbaw", 1, 6, f) == 6
+        finally:
+            gdal.VSIFCloseL(f)
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
+
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(6, 1, f) == b"barbaw"
+        finally:
+            gdal.VSIFCloseL(f)
+
+
+###############################################################################
+# Test that CopyFile invalidates cached data
+
+
+def test_vsis3_copy_invalidate(aws_test_config, webserver_port, tmp_vsimem):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
+
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(3, 1, f) == b"foo"
+        finally:
+            gdal.VSIFCloseL(f)
+
+    handler = webserver.SequentialHandler()
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(3, 1, f) == b"foo"
+        finally:
+            gdal.VSIFCloseL(f)
+
+    handler = webserver.SequentialHandler()
+    handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200)
+    memfilename = str(tmp_vsimem / "tmp.bin")
+    with webserver.install_http_handler(handler), gdaltest.tempfile(
+        memfilename, b"barbaw"
+    ):
+        gdal.CopyFile(memfilename, "/vsis3/s3_fake_bucket3/test_put_invalidate.bin")
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
+    handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
+
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
+        assert f is not None
+        try:
+            assert gdal.VSIFReadL(6, 1, f) == b"barbaw"
+        finally:
+            gdal.VSIFCloseL(f)
+
+
 ###############################################################################
 # Test simple PUT support with retry logic
 

port/cpl_vsil_curl.cpp

@@ -1363,7 +1363,6 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError,
                 }
             }
 
-            if (bGetHeaders)
             {
                 char **papszHeaders =
                     CSLTokenizeString2(sWriteFuncHeaderData.pBuffer, "\r\n", 0);
@@ -1374,7 +1373,17 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError,
                         CPLParseNameValue(papszHeaders[i], &pszKey);
                     if (pszKey && pszValue)
                     {
-                        m_aosHeaders.SetNameValue(pszKey, pszValue);
+                        if (bGetHeaders)
+                        {
+                            m_aosHeaders.SetNameValue(pszKey, pszValue);
+                        }
+                        if (EQUAL(pszKey, "Cache-Control") &&
+                            EQUAL(pszValue, "no-cache") &&
+                            CPLTestBool(CPLGetConfigOption(
+                                "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES")))
+                        {
+                            m_bCached = false;
+                        }
                     }
                     CPLFree(pszKey);
                 }