feat(storage): make it possible to disable Content-Type sniffing

As described in google/go-cloud#3298 (comment), we want to disable automatic `Content-Type` detection when inserting an object to Google Cloud Storage (GCS). Previously it wasn't possible to disable this auto-detection, even though `googleapi.MediaOptions` provides a `ForceEmptyContentType` option (https://github.com/googleapis/google-api-go-client/blob/v0.165.0/googleapi/googleapi.go#L283). We enable this by adding a `Writer` option to set this value. Closes googleapis#9430
stanhu · Feb 16, 2024 · 066692e · 066692e
1 parent 2090651
commit 066692e
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 20 deletions.
diff --git a/storage/client.go b/storage/client.go
@@ -254,6 +254,9 @@ type openWriterParams struct {
 	// attrs - see `Writer.ObjectAttrs`.
 	// Required.
 	attrs *ObjectAttrs
+	// forceEmptyContentType - Disables auto-detect of Content-Type
+	// Optional.
+	forceEmptyContentType bool
 	// conds - see `Writer.o.conds`.
 	// Optional.
 	conds *Conditions

diff --git a/storage/http_client.go b/storage/http_client.go
@@ -885,7 +885,7 @@ func (c *httpStorageClient) OpenWriter(params *openWriterParams, opts ...storage
 	mediaOpts := []googleapi.MediaOption{
 		googleapi.ChunkSize(params.chunkSize),
 	}
-	if c := attrs.ContentType; c != "" {
+	if c := attrs.ContentType; c != "" || params.forceEmptyContentType {
 		mediaOpts = append(mediaOpts, googleapi.ContentType(c))
 	}
 	if params.chunkRetryDeadline != 0 {

diff --git a/storage/integration_test.go b/storage/integration_test.go
@@ -2415,8 +2415,9 @@ func TestIntegration_WriterContentType(t *testing.T) {
 	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
 		obj := client.Bucket(bucket).Object("content")
 		testCases := []struct {
-			content           string
-			setType, wantType string
+			content                string
+			setType, wantType      string
+			forcedEmptyContentType bool
 		}{
 			{
 				// Sniffed content type.
@@ -2438,9 +2439,23 @@ func TestIntegration_WriterContentType(t *testing.T) {
 				setType:  "image/jpeg",
 				wantType: "image/jpeg",
 			},
+			{
+				content:  "<html><head><title>My first page</title></head></html>",
+				setType:  "",
+				wantType: "text/html; charset=utf-8",
+			},
+			{
+				content:                "<html><head><title>My first page</title></head></html>",
+				setType:                "",
+				wantType:               "",
+				forcedEmptyContentType: true,
+			},
 		}
 		for i, tt := range testCases {
-			if err := writeObject(ctx, obj, tt.setType, []byte(tt.content)); err != nil {
+			writer := newWriter(ctx, obj, tt.setType)
+			writer.ForceEmptyContentType = tt.forcedEmptyContentType
+
+			if err := writeContents(writer, []byte(tt.content)); err != nil {
 				t.Errorf("writing #%d: %v", i, err)
 			}
 			attrs, err := obj.Attrs(ctx)
@@ -5649,10 +5664,7 @@ func deleteObjectIfExists(o *ObjectHandle, retryOpts ...RetryOption) error {
 	return nil
 }
 
-func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error {
-	w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
-	w.ContentType = contentType
-
+func writeContents(w *Writer, contents []byte) error {
 	if contents != nil {
 		if _, err := w.Write(contents); err != nil {
 			_ = w.Close()
@@ -5662,6 +5674,19 @@ func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, con
 	return w.Close()
 }
 
+func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error {
+	w := newWriter(ctx, obj, contentType)
+
+	return writeContents(w, contents)
+}
+
+func newWriter(ctx context.Context, obj *ObjectHandle, contentType string) *Writer {
+	w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
+	w.ContentType = contentType
+
+	return w
+}
+
 func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) {
 	r, err := obj.NewReader(ctx)
 	if err != nil {

diff --git a/storage/writer.go b/storage/writer.go
@@ -88,6 +88,11 @@ type Writer struct {
 	// cancellation.
 	ChunkRetryDeadline time.Duration
 
+	// ForceEmptyContentType is an optional parameter that is used to disable
+	// auto-detection of Content-Type. By default, if a blank Content-Type
+	// is provided, then gax.DetermineContentType is called to sniff the type.
+	ForceEmptyContentType bool
+
 	// ProgressFunc can be used to monitor the progress of a large write
 	// operation. If ProgressFunc is not nil and writing requires multiple
 	// calls to the underlying service (see
@@ -180,18 +185,19 @@ func (w *Writer) openWriter() (err error) {
 	isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true)
 	opts := makeStorageOpts(isIdempotent, w.o.retry, w.o.userProject)
 	params := &openWriterParams{
-		ctx:                w.ctx,
-		chunkSize:          w.ChunkSize,
-		chunkRetryDeadline: w.ChunkRetryDeadline,
-		bucket:             w.o.bucket,
-		attrs:              &w.ObjectAttrs,
-		conds:              w.o.conds,
-		encryptionKey:      w.o.encryptionKey,
-		sendCRC32C:         w.SendCRC32C,
-		donec:              w.donec,
-		setError:           w.error,
-		progress:           w.progress,
-		setObj:             func(o *ObjectAttrs) { w.obj = o },
+		ctx:                   w.ctx,
+		chunkSize:             w.ChunkSize,
+		chunkRetryDeadline:    w.ChunkRetryDeadline,
+		bucket:                w.o.bucket,
+		attrs:                 &w.ObjectAttrs,
+		conds:                 w.o.conds,
+		encryptionKey:         w.o.encryptionKey,
+		sendCRC32C:            w.SendCRC32C,
+		donec:                 w.donec,
+		setError:              w.error,
+		progress:              w.progress,
+		setObj:                func(o *ObjectAttrs) { w.obj = o },
+		forceEmptyContentType: w.ForceEmptyContentType,
 	}
 	if err := w.ctx.Err(); err != nil {
 		return err // short-circuit