feat: Doc encryption with symmetric key

cli/collection_create.go

@@ -17,14 +17,25 @@
 	"github.com/spf13/cobra"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/internal/db"
 )
 
 func MakeCollectionCreateCommand() *cobra.Command {
 	var file string
+	var shouldEncrypt bool
 	var cmd = &cobra.Command{
-		Use:   "create [-i --identity] <document>",
+		Use:   "create [-i --identity] [-e --encrypt] <document>",
 		Short: "Create a new document.",
 		Long: `Create a new document.
+
+Options:
+    -i, --identity 
+        Marks the document as private and set the identity as the owner. The access to the document
+		and permissions are controlled by ACP (Access Control Policy).
+
+	-e, --encrypt
+		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
+		symmetric key for encryption using AES-GCM.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -69,6 +80,9 @@
 				return cmd.Usage()
 			}
 
+			txn, _ := db.TryGetContextTxn(cmd.Context())
+			setContextDocEncryption(cmd, shouldEncrypt, txn)
+
 			if client.IsJSONArray(docData) {
 				docs, err := client.NewDocsFromJSON(docData, col.Definition())
 				if err != nil {
@@ -84,6 +98,8 @@
 			return col.Create(cmd.Context(), doc)
 		},
 	}
+	cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false,
+		"Flag to enable encryption of the document")
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
 	return cmd
 }

cli/utils.go

@@ -25,8 +25,10 @@
 
 	acpIdentity "github.com/sourcenetwork/defradb/acp/identity"
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/http"
 	"github.com/sourcenetwork/defradb/internal/db"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 	"github.com/sourcenetwork/defradb/keyring"
 )
 
@@ -160,6 +162,19 @@
 	return nil
 }
 
+// setContextDocEncryption sets doc encryption for the current command context.
+func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) {
+	if !shouldEncrypt {
+		return
+	}
+	ctx := cmd.Context()
+	if txn != nil {
+		ctx = encryption.ContextWithStore(ctx, txn)
+	}
+	ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true})
+	cmd.SetContext(ctx)
+}
+
 // setContextRootDir sets the rootdir for the current command context.
 func setContextRootDir(cmd *cobra.Command) error {
 	rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir")

client/document.go

@@ -118,7 +118,7 @@ func NewDocFromMap(data map[string]any, collectionDefinition CollectionDefinitio
 	return doc, nil
 }
 
-var jsonArrayPattern = regexp.MustCompile(`^\s*\[.*\]\s*$`)
+var jsonArrayPattern = regexp.MustCompile(`(?s)^\s*\[.*\]\s*$`)
 
 // IsJSONArray returns true if the given byte array is a JSON Array.
 func IsJSONArray(obj []byte) bool {

client/document_test.go

@@ -206,3 +206,66 @@ func TestNewFromJSON_WithInvalidJSONFieldValueSimpleString_Error(t *testing.T) {
 	_, err := NewDocFromJSON(objWithJSONField, def)
 	require.ErrorContains(t, err, "invalid JSON payload. Payload: blah")
 }
+
+func TestIsJSONArray(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected bool
+	}{
+		{
+			name:     "Valid JSON Array",
+			input:    []byte(`[{"name":"John","age":21},{"name":"Islam","age":33}]`),
+			expected: true,
+		},
+		{
+			name:     "Valid Empty JSON Array",
+			input:    []byte(`[]`),
+			expected: true,
+		},
+		{
+			name:     "Valid JSON Object",
+			input:    []byte(`{"name":"John","age":21}`),
+			expected: false,
+		},
+		{
+			name:     "Invalid JSON String",
+			input:    []byte(`{"name":"John","age":21`),
+			expected: false,
+		},
+		{
+			name:     "Non-JSON String",
+			input:    []byte(`Hello, World!`),
+			expected: false,
+		},
+		{
+			name:     "Array of Primitives",
+			input:    []byte(`[1, 2, 3, 4]`),
+			expected: true,
+		},
+		{
+			name:     "Nested JSON Array",
+			input:    []byte(`[[1, 2], [3, 4]]`),
+			expected: true,
+		},
+		{
+			name: "Valid JSON Array with Whitespace",
+			input: []byte(`
+				[
+					{	"name": "John", "age": 21	},
+					{	"name": "Islam", "age": 33	}
+				]
+			`),
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := IsJSONArray(tt.input)
+			if actual != tt.expected {
+				t.Errorf("IsJSONArray(%s) = %v; expected %v", tt.input, actual, tt.expected)
+			}
+		})
+	}
+}

client/request/consts.go

@@ -21,10 +21,13 @@ const (
 
 	Cid         = "cid"
 	Input       = "input"
+	Inputs      = "inputs"
 	FieldName   = "field"
 	FieldIDName = "fieldId"
 	ShowDeleted = "showDeleted"
 
+	EncryptArgName = "encrypt"
+
 	FilterClause  = "filter"
 	GroupByClause = "groupBy"
 	LimitClause   = "limit"

client/request/mutation.go

@@ -41,6 +41,15 @@ type ObjectMutation struct {
 	//
 	// This is ignored for [DeleteObjects] mutations.
 	Input map[string]any
+
+	// Inputs is the array of json representations of the fieldName-value pairs of document
+	// properties to mutate.
+	//
+	// This is ignored for [DeleteObjects] mutations.
+	Inputs []map[string]any
+
+	// Encrypt is a boolean flag that indicates whether the input data should be encrypted.
+	Encrypt bool
 }
 
 // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as

datastore/multi.go

@@ -23,11 +23,13 @@ var (
 	headStoreKey   = rootStoreKey.ChildString("heads")
 	blockStoreKey  = rootStoreKey.ChildString("blocks")
 	peerStoreKey   = rootStoreKey.ChildString("ps")
+	encStoreKey    = rootStoreKey.ChildString("enc")
 )
 
 type multistore struct {
 	root   DSReaderWriter
 	data   DSReaderWriter
+	enc    DSReaderWriter
 	head   DSReaderWriter
 	peer   DSBatching
 	system DSReaderWriter
@@ -43,6 +45,7 @@ func MultiStoreFrom(rootstore ds.Datastore) MultiStore {
 	ms := &multistore{
 		root:   rootRW,
 		data:   prefix(rootRW, dataStoreKey),
+		enc:    prefix(rootRW, encStoreKey),
 		head:   prefix(rootRW, headStoreKey),
 		peer:   namespace.Wrap(rootstore, peerStoreKey),
 		system: prefix(rootRW, systemStoreKey),
@@ -57,6 +60,11 @@ func (ms multistore) Datastore() DSReaderWriter {
 	return ms.data
 }
 
+// Encstore implements MultiStore.
+func (ms multistore) Encstore() DSReaderWriter {
+	return ms.enc
+}
+
 // Headstore implements MultiStore.
 func (ms multistore) Headstore() DSReaderWriter {
 	return ms.head

datastore/store.go

@@ -34,26 +34,26 @@ type Rootstore interface {
 type MultiStore interface {
 	Rootstore() DSReaderWriter
 
-	// Datastore is a wrapped root DSReaderWriter
-	// under the /data namespace
+	// Datastore is a wrapped root DSReaderWriter under the /data namespace
 	Datastore() DSReaderWriter
 
-	// Headstore is a wrapped root DSReaderWriter
-	// under the /head namespace
+	// Encstore is a wrapped root DSReaderWriter under the /enc namespace
+	// This store is used for storing symmetric encryption keys for doc encryption.
+	// The store keys are comprised of docID + field name.
+	Encstore() DSReaderWriter
+
+	// Headstore is a wrapped root DSReaderWriter under the /head namespace
 	Headstore() DSReaderWriter
 
-	// Peerstore is a wrapped root DSReaderWriter
-	// as a ds.Batching, embedded into a DSBatching
+	// Peerstore is a wrapped root DSReaderWriter as a ds.Batching, embedded into a DSBatching
 	// under the /peers namespace
 	Peerstore() DSBatching
 
-	// Blockstore is a wrapped root DSReaderWriter
-	// as a Blockstore, embedded into a Blockstore
+	// Blockstore is a wrapped root DSReaderWriter as a Blockstore, embedded into a Blockstore
 	// under the /blocks namespace
 	Blockstore() Blockstore
 
-	// Headstore is a wrapped root DSReaderWriter
-	// under the /system namespace
+	// Headstore is a wrapped root DSReaderWriter under the /system namespace
 	Systemstore() DSReaderWriter
 }
 

docs/website/references/cli/defradb_client_collection_create.md

@@ -5,6 +5,15 @@ Create a new document.
 ### Synopsis
 
 Create a new document.
+
+Options:
+    -i, --identity 
+        Marks the document as private and set the identity as the owner. The access to the document
+		and permissions are controlled by ACP (Access Control Policy).
+
+	-e, --encrypt
+		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
+		symmetric key for encryption using AES-GCM.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -24,12 +33,13 @@ Example: create from stdin:
 
 
 ```
-defradb client collection create [-i --identity] <document> [flags]
+defradb client collection create [-i --identity] [-e --encrypt] <document> [flags]
 ```
 
 ### Options
 
 ```
+  -e, --encrypt       Flag to enable encryption of the document
   -f, --file string   File containing document(s)
   -h, --help          help for create
 ```

http/client.go

@@ -349,12 +349,16 @@ func (c *Client) ExecRequest(
 		result.GQL.Errors = []error{err}
 		return result
 	}
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, methodURL.String(), bytes.NewBuffer(body))
 	if err != nil {
 		result.GQL.Errors = []error{err}
 		return result
 	}
 	err = c.http.setDefaultHeaders(req)
+
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	if err != nil {
 		result.GQL.Errors = []error{err}
 		return result

http/client_collection.go

@@ -24,6 +24,7 @@
 	sse "github.com/vito/go-sse/sse"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 )
 
 var _ client.Collection = (*Collection)(nil)
@@ -78,6 +79,8 @@
 		return err
 	}
 
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	_, err = c.http.request(req)
 	if err != nil {
 		return err
@@ -114,6 +117,8 @@
 		return err
 	}
 
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	_, err = c.http.request(req)
 	if err != nil {
 		return err
@@ -125,6 +130,15 @@
 	return nil
 }
 
+func setDocEncryptionFlagIfNeeded(ctx context.Context, req *http.Request) {
+	encConf := encryption.GetContextConfig(ctx)
+	if encConf.HasValue() && encConf.Value().IsEncrypted {
+		q := req.URL.Query()
+		q.Set(docEncryptParam, "true")
+		req.URL.RawQuery = q.Encode()
+	}
+}
+
 func (c *Collection) Update(
 	ctx context.Context,
 	doc *client.Document,