diff --git a/go.mod b/go.mod index f5dde0cb55..efb6c94d6b 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( golang.org/x/time v0.5.0 google.golang.org/api v0.184.0 google.golang.org/genproto v0.0.0-20240610135401-a8a62080eff3 - google.golang.org/grpc v1.64.1 + google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 vitess.io/vitess v0.15.3 ) @@ -115,6 +115,7 @@ require ( github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apache/arrow/go/v16 v16.0.0 // indirect github.com/apache/thrift v0.20.0 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40 // indirect @@ -180,11 +181,13 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pinecone-io/go-pinecone v1.1.1 // indirect github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index b099503e65..64b001b498 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,7 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/RobinUS2/golang-moving-average v1.0.0/go.mod h1:MdzhY+KoEvi+OBygTPH0OSaKrOJzvILWN2SPQzaKVsY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alpacahq/alpaca-trade-api-go/v2 v2.8.0 h1:9rsXPiOSewm+zGoupFzCjxiOcYjJcCMunuCrXiEsrSc= @@ -159,6 +160,8 @@ github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlE github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.20.0 h1:631+KvYbsBZxmuJjYwhezVsrfc/TbqtZV4QcxOX1fOI= github.com/apache/thrift v0.20.0/go.mod h1:hOk1BQqcp2OLzGsyVXdfMk7YFlMxK3aoEVhjD06QhB8= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.54.2 h1:Wo6AVWcleNHrYa48YzfYz60hzxGRqsJrK5s/qePe+3I= github.com/aws/aws-sdk-go v1.54.2/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= @@ -257,6 +260,7 @@ github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradleyjkemp/cupaloy v2.3.0+incompatible h1:UafIjBvWQmS9i/xRg+CamMrnLTKNzo+bdmT/oH34c2Y= github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk= @@ -618,6 +622,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -694,6 +699,8 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -720,6 +727,8 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pinecone-io/go-pinecone v1.1.1 h1:pKoIiYcBIbrR7gaq0JXPiVnNEtevFYeq/AYL7T0NbbE= +github.com/pinecone-io/go-pinecone v1.1.1/go.mod h1:KfJhn4yThX293+fbtrZLnxe2PJYo8557Py062W4FYKk= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -794,6 +803,7 @@ github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNo github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -1376,6 +1386,8 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20210430044426-28078834f35b h1:D/GTYPo6I1oEo08Bfpuj3xl5XE+UGHj7//5fVyKxhsQ= google.golang.org/grpc/examples v0.0.0-20210430044426-28078834f35b/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= diff --git a/materialize-pinecone/.snapshots/TestSpecification b/materialize-pinecone/.snapshots/TestSpecification index 35fa5b4d14..87f1ea05f6 100644 --- a/materialize-pinecone/.snapshots/TestSpecification +++ b/materialize-pinecone/.snapshots/TestSpecification @@ -9,12 +9,6 @@ "description": "Pinecone index for this materialization. Must already exist and have appropriate dimensions for the embedding model used.", "order": 0 }, - "environment": { - "type": "string", - "title": "Pinecone Environment", - "description": "Cloud region for your Pinecone project. Example: us-central1-gcp", - "order": 1 - }, "pineconeApiKey": { "type": "string", "title": "Pinecone API Key", @@ -53,7 +47,6 @@ "type": "object", "required": [ "index", - "environment", "pineconeApiKey", "openAiApiKey" ], @@ -66,11 +59,14 @@ "namespace": { "type": "string", "title": "Pinecone Namespace", - "description": "Name of the Pinecone namespace that this collection will materialize vectors into. For Pinecone starter plans, leave blank to use no namespace. Only a single binding can have a blank namespace, and Pinecone starter plans can only materialize a single binding.", + "description": "Name of the Pinecone namespace that this collection will materialize vectors into.", "x-collection-name": true } }, "type": "object", + "required": [ + "namespace" + ], "title": "Pinecone Collection" }, "documentation_url": "https://go.estuary.dev/materialize-pinecone" diff --git a/materialize-pinecone/client/client.go b/materialize-pinecone/client/client.go index 1f94a89186..e08985e2ec 100644 --- a/materialize-pinecone/client/client.go +++ b/materialize-pinecone/client/client.go @@ -27,7 +27,7 @@ type OpenAIEmbeddingsResponse struct { type Embedding struct { Object string `json:"object"` - Embedding []float64 `json:"embedding"` + Embedding []float32 `json:"embedding"` Index int `json:"index"` } @@ -140,207 +140,6 @@ func (c *OpenAiClient) VerifyModelExists(ctx context.Context) error { return nil } -type PineconeUpsertRequest struct { - Vectors []Vector `json:"vectors"` - Namespace string `json:"namespace"` -} - -type Vector struct { - Id string `json:"id"` - Values []float64 `json:"values"` - Metadata map[string]interface{} `json:"metadata"` -} - -type PineconeUpsertResponse struct { - UpsertedCount int `json:"upsertedCount"` -} - -type PineconeIndexStatsResponse struct { - Namespaces map[string]interface{} `json:"namespaces"` - Dimension int `json:"dimension"` - IndexFullness float64 `json:"index_fullness"` -} - -type PineconeIndexDescribeResponse struct { - Database struct { - MetadataConfig struct { - Indexed []string `json:"indexed"` - } `json:"metadata_config"` - PodType string `json:"pod_type"` - } `json:"database"` -} - -type whoamiResponse struct { - ProjectName string `json:"project_name"` -} - -type pineconeUpsertError struct { - Code int `json:"code"` - Message string `json:"message"` - Details []string `json:"details"` -} - -type PineconeClient struct { - http *http.Client - index string - projectName string - environment string - apiKey string -} - -func NewPineconeClient(ctx context.Context, index string, environment string, apiKey string) (*PineconeClient, error) { - c := &PineconeClient{ - http: http.DefaultClient, - index: index, - environment: environment, - apiKey: apiKey, - } - - whoami, err := c.whoami(ctx) - if err != nil { - return nil, err - } - - c.projectName = whoami.ProjectName - - return c, nil -} - -func (c *PineconeClient) baseUrl() string { - return fmt.Sprintf("https://%s-%s.svc.%s.pinecone.io", c.index, c.projectName, c.environment) -} - -func (c *PineconeClient) DescribeIndexStats(ctx context.Context) (PineconeIndexStatsResponse, error) { - req, err := http.NewRequestWithContext(ctx, "POST", c.baseUrl()+"/describe_index_stats", nil) - if err != nil { - return PineconeIndexStatsResponse{}, err - } - req.Header.Set("accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Api-Key", c.apiKey) - req.Header.Set("User-Agent", "source_tag=estuary") - - res, err := c.http.Do(req) - if err != nil { - return PineconeIndexStatsResponse{}, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return PineconeIndexStatsResponse{}, fmt.Errorf("PineconeClient DescribeIndexStats unexpected status: %s", res.Status) - } - - indexResponse := PineconeIndexStatsResponse{} - if err := json.NewDecoder(res.Body).Decode(&indexResponse); err != nil { - return PineconeIndexStatsResponse{}, err - } - - return indexResponse, nil -} - -func (c *PineconeClient) DescribeIndex(ctx context.Context) (PineconeIndexDescribeResponse, error) { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://controller.%s.pinecone.io/databases/%s", c.environment, c.index), nil) - if err != nil { - return PineconeIndexDescribeResponse{}, err - } - req.Header.Set("accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Api-Key", c.apiKey) - req.Header.Set("User-Agent", "source_tag=estuary") - - res, err := c.http.Do(req) - if err != nil { - return PineconeIndexDescribeResponse{}, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return PineconeIndexDescribeResponse{}, fmt.Errorf("PineconeClient DescribeIndex unexpected status: %s", res.Status) - } - - indexResponse := PineconeIndexDescribeResponse{} - if err := json.NewDecoder(res.Body).Decode(&indexResponse); err != nil { - return PineconeIndexDescribeResponse{}, err - } - - return indexResponse, nil -} - -func (c *PineconeClient) Upsert(ctx context.Context, req PineconeUpsertRequest) error { - res, err := withRetry(ctx, func() (*http.Response, error) { - body := new(bytes.Buffer) - if err := json.NewEncoder(body).Encode(&req); err != nil { - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, "POST", c.baseUrl()+"/vectors/upsert", body) - if err != nil { - return nil, err - } - req.Header.Set("accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Api-Key", c.apiKey) - req.Header.Set("User-Agent", "source_tag=estuary") - - return c.http.Do(req) - }) - if err != nil { - return err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - var errorBody pineconeUpsertError - if err := json.NewDecoder(res.Body).Decode(&errorBody); err != nil { - log.WithField("error", err).Warn("could not decode error response body") - } else if errorBody.Message != "" { - return fmt.Errorf("pinecone vector upsert failed (%s): %s", res.Status, errorBody.Message) - } else { - log.WithField("errorBody", errorBody).Warn("errorBody error message was empty") - } - - return fmt.Errorf("PineconeClient Upsert unexpected status: %s", res.Status) - } - - upsertResponse := PineconeUpsertResponse{} - if err := json.NewDecoder(res.Body).Decode(&upsertResponse); err != nil { - return err - } - - if len(req.Vectors) != upsertResponse.UpsertedCount { - return fmt.Errorf("upserted unexpected vector count: %d expected vs %d upserted", len(req.Vectors), upsertResponse.UpsertedCount) - } - - return nil -} - -func (c *PineconeClient) whoami(ctx context.Context) (whoamiResponse, error) { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://controller.%s.pinecone.io/actions/whoami", c.environment), nil) - if err != nil { - return whoamiResponse{}, err - } - req.Header.Set("Api-Key", c.apiKey) - req.Header.Set("accept", "application/json") - req.Header.Set("User-Agent", "source_tag=estuary") - - res, err := c.http.Do(req) - if err != nil { - return whoamiResponse{}, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return whoamiResponse{}, fmt.Errorf("PineconeClient whoami unexpected status: %s", res.Status) - } - - whoami := whoamiResponse{} - if err := json.NewDecoder(res.Body).Decode(&whoami); err != nil { - return whoamiResponse{}, err - } - - return whoami, nil -} - var ( maxRetries = 10 initialBackoff float64 = 200 // Milliseconds diff --git a/materialize-pinecone/driver.go b/materialize-pinecone/driver.go index 7feaaa35eb..10817b459e 100644 --- a/materialize-pinecone/driver.go +++ b/materialize-pinecone/driver.go @@ -12,19 +12,17 @@ import ( "github.com/estuary/connectors/materialize-pinecone/client" pf "github.com/estuary/flow/go/protocols/flow" pm "github.com/estuary/flow/go/protocols/materialize" + "github.com/pinecone-io/go-pinecone/pinecone" log "github.com/sirupsen/logrus" ) const ( - textEmbeddingAda002 = "text-embedding-ada-002" - textEmbeddingAda002VectorLength = 1536 - starterPodType = "starter" - emptyNamespaceResourcePathSentinel = "FLOW_EMPTY_NAMESPACE" + textEmbeddingAda002 = "text-embedding-ada-002" + textEmbeddingAda002VectorLength = 1536 ) type config struct { Index string `json:"index" jsonschema:"title=Pinecone Index" jsonschema_extras:"order=0"` - Environment string `json:"environment" jsonschema:"title=Pinecone Environment" jsonschema_extras:"order=1"` PineconeApiKey string `json:"pineconeApiKey" jsonschema:"title=Pinecone API Key" jsonschema_extras:"secret=true,order=2"` OpenAiApiKey string `json:"openAiApiKey" jsonschema:"title=OpenAI API Key" jsonschema_extras:"secret=true,order=3"` EmbeddingModel string `json:"embeddingModel,omitempty" jsonschema:"title=Embedding Model ID,default=text-embedding-ada-002" jsonschema_extras:"order=4"` @@ -66,7 +64,6 @@ func (advancedConfig) GetFieldDocString(fieldName string) string { func (c *config) Validate() error { var requiredProperties = [][]string{ {"index", c.Index}, - {"environment", c.Environment}, {"pineconeApiKey", c.PineconeApiKey}, {"openAiApiKey", c.OpenAiApiKey}, } @@ -79,8 +76,11 @@ func (c *config) Validate() error { return nil } -func (c *config) pineconeClient(ctx context.Context) (*client.PineconeClient, error) { - return client.NewPineconeClient(ctx, c.Index, c.Environment, c.PineconeApiKey) +func (c *config) pineconeClient() (*pinecone.Client, error) { + return pinecone.NewClient(pinecone.NewClientParams{ + ApiKey: c.PineconeApiKey, + SourceTag: "estuary", + }) } func (c *config) openAiClient() *client.OpenAiClient { @@ -93,19 +93,23 @@ func (c *config) openAiClient() *client.OpenAiClient { } type resource struct { - Namespace string `json:"namespace,omitempty" jsonschema:"title=Pinecone Namespace" jsonschema_extras:"x-collection-name=true"` + Namespace string `json:"namespace" jsonschema:"title=Pinecone Namespace" jsonschema_extras:"x-collection-name=true"` } func (resource) GetFieldDocString(fieldName string) string { switch fieldName { case "Namespace": - return "Name of the Pinecone namespace that this collection will materialize vectors into. For Pinecone starter plans, leave blank to use no namespace. Only a single binding can have a blank namespace, and Pinecone starter plans can only materialize a single binding." + return "Name of the Pinecone namespace that this collection will materialize vectors into." default: return "" } } func (r resource) Validate() error { + if r.Namespace == "" { + return fmt.Errorf("missing namespace") + } + return nil } @@ -142,17 +146,21 @@ func (d driver) Validate(ctx context.Context, req *pm.Request_Validate) (*pm.Res } // Validate connectivity and that the index exists and is appropriately dimensioned. - pc, err := cfg.pineconeClient(ctx) + pc, err := cfg.pineconeClient() if err != nil { return nil, err } - if indexStats, err := pc.DescribeIndexStats(ctx); err != nil { - return nil, fmt.Errorf("connecting to Pinecone: %w", err) - } else if cfg.EmbeddingModel == textEmbeddingAda002 && indexStats.Dimension != textEmbeddingAda002VectorLength { + + idx, err := pc.DescribeIndex(ctx, cfg.Index) + if err != nil { + return nil, fmt.Errorf("describing index: %w", err) + } + + if cfg.EmbeddingModel == textEmbeddingAda002 && idx.Dimension != textEmbeddingAda002VectorLength { return nil, fmt.Errorf( "index '%s' has dimensions of %d but must be %d for embedding model '%s'", cfg.Index, - indexStats.Dimension, + idx.Dimension, textEmbeddingAda002VectorLength, textEmbeddingAda002, ) @@ -160,48 +168,24 @@ func (d driver) Validate(ctx context.Context, req *pm.Request_Validate) (*pm.Res return nil, err } - // Log a warning message if the 'flow_document' metadata field has not been excluded from - // metadata indexing. This is a high cardinality field and is potentially large, and such fields - // are not recommended to be indexed. - indexDescribe, err := pc.DescribeIndex(ctx) - if err != nil { - return nil, fmt.Errorf("describing index: %w", err) - } - entry := log.WithFields(log.Fields{ - "index": cfg.Index, - "environment": cfg.Environment, - }) - if indexDescribe.Database.MetadataConfig.Indexed == nil { - // If no explicit metadata configuration for which fields are indexed has been provided all fields are indexed. - entry.Warn("Metadata field 'flow_document' will be indexed since this index is not configured with selective metadata indexing. Consider using selective metadata indexing to prevent this field from being indexed to optimize memory utilization.") - } else if slices.Contains(indexDescribe.Database.MetadataConfig.Indexed, "flow_document") { - entry.Warn("Metadata field 'flow_document' will be indexed. This may not result in optimal memory utilization for the index.") - } - - if indexDescribe.Database.PodType == starterPodType { - // "Starter" pod types on the free tier cannot use namespaces. Namespaces are required for - // differentiating multiple bindings in the same index, so starter pods cannot have more - // than 1 binding, and the binding can't have a namespace set. - if len(req.Bindings) > 1 { - return nil, fmt.Errorf( - "Your Pinecone index '%s' is of type '%s', which cannot use Pinecone's namespace feature. This materialization is thus unable to have more than one bound collection. Consider removing bindings, or upgrade your Pinecone plan.", - cfg.Index, - starterPodType, - ) + // Log a warning message if the 'flow_document' metadata field has not been + // excluded from metadata indexing for pod-based indices. This is a high + // cardinality field and is potentially large, and such fields are not + // recommended to be indexed. + if idx.Spec.Pod != nil { + var indexed []string + if idx.Spec.Pod.MetadataConfig != nil && idx.Spec.Pod.MetadataConfig.Indexed != nil { + indexed = *idx.Spec.Pod.MetadataConfig.Indexed } - if len(req.Bindings) == 1 { - res, err := resolveResourceConfig(req.Bindings[0].ResourceConfigJson) - if err != nil { - return nil, fmt.Errorf("resolving resource config for single starter plan index binding: %w", err) - } - if res.Namespace != "" { - return nil, fmt.Errorf( - "Your Pinecone index '%s' is of type '%s', which cannot use Pinecone's namespace feature. You can still proceed by removing the Pinecone Namespace from your bound collection's Resource Configuration, or by upgrading your Pinecone plan.", - cfg.Index, - starterPodType, - ) - } + entry := log.WithFields(log.Fields{ + "index": cfg.Index, + }) + if len(indexed) == 0 { + // If no explicit metadata configuration for which fields are indexed has been provided all fields are indexed. + entry.Warn("Metadata field 'flow_document' will be indexed since this index is not configured with selective metadata indexing. Consider using selective metadata indexing to prevent this field from being indexed to optimize memory utilization.") + } else if slices.Contains(indexed, "flow_document") { + entry.Warn("Metadata field 'flow_document' will be indexed. This may not result in optimal memory utilization for the index.") } } @@ -237,15 +221,10 @@ func (d driver) Validate(ctx context.Context, req *pm.Request_Validate) (*pm.Res constraints[projection.Field] = constraint } - nsPath := res.Namespace - if res.Namespace == "" { - nsPath = emptyNamespaceResourcePathSentinel - } - out = append(out, &pm.Response_Validated_Binding{ Constraints: constraints, DeltaUpdates: true, - ResourcePath: []string{nsPath}, + ResourcePath: []string{cfg.Index, res.Namespace}, }) } @@ -265,6 +244,16 @@ func (d driver) NewTransactor(ctx context.Context, open pm.Request_Open) (m.Tran return nil, nil, err } + pc, err := cfg.pineconeClient() + if err != nil { + return nil, nil, err + } + + idx, err := pc.DescribeIndex(ctx, cfg.Index) + if err != nil { + return nil, nil, fmt.Errorf("describing index: %w", err) + } + var bindings []binding for _, b := range open.Materialization.Bindings { res, err := resolveResourceConfig(b.ResourceConfigJson) @@ -272,26 +261,20 @@ func (d driver) NewTransactor(ctx context.Context, open pm.Request_Open) (m.Tran return nil, nil, err } + conn, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: res.Namespace}) + if err != nil { + return nil, nil, fmt.Errorf("creating index connection: %w", err) + } + bindings = append(bindings, binding{ - namespace: res.Namespace, + conn: conn, dataHeaders: b.FieldSelection.AllFields(), }) } - pc, err := cfg.pineconeClient(ctx) - if err != nil { - return nil, nil, err - } - - log.WithFields(log.Fields{ - "index": cfg.Index, - "environment": cfg.Environment, - }).Info("starting materialize-pinecone") - return &transactor{ - pineconeClient: pc, - openAiClient: cfg.openAiClient(), - bindings: bindings, + openAiClient: cfg.openAiClient(), + bindings: bindings, }, &pm.Response_Opened{}, nil } diff --git a/materialize-pinecone/transactor.go b/materialize-pinecone/transactor.go index 515b274cb9..59048e52d6 100644 --- a/materialize-pinecone/transactor.go +++ b/materialize-pinecone/transactor.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/base64" "encoding/json" "fmt" "strings" @@ -10,7 +9,9 @@ import ( m "github.com/estuary/connectors/go/protocols/materialize" "github.com/estuary/connectors/materialize-pinecone/client" pf "github.com/estuary/flow/go/protocols/flow" + "github.com/pinecone-io/go-pinecone/pinecone" "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/types/known/structpb" ) // TODO(whb): These may need tuning based on real-world use. @@ -20,16 +21,15 @@ var ( ) type transactor struct { - pineconeClient *client.PineconeClient - openAiClient *client.OpenAiClient - bindings []binding + openAiClient *client.OpenAiClient + bindings []binding group *errgroup.Group groupCtx context.Context } type binding struct { - namespace string + conn *pinecone.IndexConnection dataHeaders []string } @@ -55,9 +55,22 @@ func (t *transactor) Store(it *m.StoreIterator) (m.StartCommitFunc, error) { t.group, t.groupCtx = errgroup.WithContext(ctx) t.group.SetLimit(concurrentWorkers) - batches := make(map[string][]upsertDoc) + var batch []upsertDoc + lastBinding := -1 for it.Next() { + if lastBinding == -1 { + lastBinding = it.Binding + } + + if it.Binding != lastBinding { + if err := t.sendBatch(t.bindings[lastBinding], batch); err != nil { + return nil, fmt.Errorf("sending batch of documents: %w", err) + } + batch = nil + lastBinding = it.Binding + } + b := t.bindings[it.Binding] allFields := append(it.Key, it.Values...) @@ -74,31 +87,26 @@ func (t *transactor) Store(it *m.StoreIterator) (m.StartCommitFunc, error) { return nil, err } - namespace := t.bindings[it.Binding].namespace - - batches[namespace] = append(batches[namespace], upsertDoc{ + batch = append(batch, upsertDoc{ input: embeddingInput, - key: base64.RawURLEncoding.EncodeToString(it.PackedKey), + key: fmt.Sprintf("%x", it.PackedKey), // Only the document is included as metadata. metadata: map[string]interface{}{ "flow_document": string(it.RawJSON), }, }) - if len(batches[namespace]) >= batchSize { - if err := t.sendBatch(namespace, batches[namespace]); err != nil { + if len(batch) >= batchSize { + if err := t.sendBatch(b, batch); err != nil { return nil, fmt.Errorf("sending batch of documents: %w", err) } - batches[namespace] = nil + batch = nil } } - // Flush remaining partial batches. - for namespace, batch := range batches { - if len(batch) > 0 { - if err := t.sendBatch(namespace, batch); err != nil { - return nil, fmt.Errorf("flushing documents batch: %w", err) - } + if len(batch) != 0 { + if err := t.sendBatch(t.bindings[lastBinding], batch); err != nil { + return nil, fmt.Errorf("sending batch of documents: %w", err) } } @@ -134,7 +142,7 @@ func makeInput(fields map[string]interface{}) (string, error) { return out.String(), nil } -func (t *transactor) sendBatch(namespace string, batch []upsertDoc) error { +func (t *transactor) sendBatch(b binding, batch []upsertDoc) error { select { case <-t.groupCtx.Done(): return t.group.Wait() @@ -150,22 +158,27 @@ func (t *transactor) sendBatch(namespace string, batch []upsertDoc) error { return fmt.Errorf("openAI creating embeddings: %w", err) } - upsert := client.PineconeUpsertRequest{ - Vectors: []client.Vector{}, - Namespace: namespace, - } + var vecs []*pinecone.Vector for _, e := range embeddings { - thisMeta := batch[e.Index] - upsert.Vectors = append(upsert.Vectors, client.Vector{ - Id: thisMeta.key, + thisUpsert := batch[e.Index] + + metadata, err := structpb.NewStruct(thisUpsert.metadata) + if err != nil { + return fmt.Errorf("creating metadata: %w", err) + } + + vecs = append(vecs, &pinecone.Vector{ + Id: thisUpsert.key, Values: e.Embedding, - Metadata: thisMeta.metadata, + Metadata: metadata, }) } - if err := t.pineconeClient.Upsert(t.groupCtx, upsert); err != nil { + if count, err := b.conn.UpsertVectors(t.groupCtx, vecs); err != nil { return fmt.Errorf("pinecone upserting batch: %w", err) + } else if int(count) != len(batch) { + return fmt.Errorf("pinecone upserted %d vectors vs. expected %d", count, len(batch)) } return nil }) diff --git a/tests/materialize/materialize-pinecone/cleanup.sh b/tests/materialize/materialize-pinecone/cleanup.sh index 537c7dcc83..974a99299d 100755 --- a/tests/materialize/materialize-pinecone/cleanup.sh +++ b/tests/materialize/materialize-pinecone/cleanup.sh @@ -2,20 +2,10 @@ set -e -function deleteNamespace() { - curl --request POST \ - --url https://${PINECONE_INDEX}-${PINECONE_PROJECT_ID}.svc.${PINECONE_ENVIRONMENT}.pinecone.io/vectors/delete \ - --header "Api-Key: ${PINECONE_API_KEY}" \ - --header "accept: application/json" \ - --header "content-type: application/json" \ - --data " - { - \"deleteAll\": true, - \"namespace\": \"${1}\" - } - " +function clearNamespace() { + go run ${TEST_DIR}/materialize-pinecone/fetch-data.go --clear-namespace "$1" } -deleteNamespace "simple" -deleteNamespace "duplicated-keys" -deleteNamespace "multiple-types" +clearNamespace "simple" +clearNamespace "duplicated-keys" +clearNamespace "multiple-types" diff --git a/tests/materialize/materialize-pinecone/config-extra.json b/tests/materialize/materialize-pinecone/config-extra.json deleted file mode 100644 index 8bfa387343..0000000000 --- a/tests/materialize/materialize-pinecone/config-extra.json +++ /dev/null @@ -1 +0,0 @@ -{"projectId": "b5d672c"} diff --git a/tests/materialize/materialize-pinecone/config.yaml b/tests/materialize/materialize-pinecone/config.yaml index c8f6135dfc..9e4d224f76 100644 --- a/tests/materialize/materialize-pinecone/config.yaml +++ b/tests/materialize/materialize-pinecone/config.yaml @@ -1,5 +1,4 @@ -index: flow-index -environment: us-central1-gcp +index: flow-test pineconeApiKey_sops: ENC[AES256_GCM,data:qv41PZ2Kj/4jvM2Ock/DpUR8zfDJvpRqxsijq+sPGx1GMJ1T,iv:8L2s+tqoMlyBvOMe6Wg10zfvNHtLPeEWtaq8jdorfgE=,tag:55690uYVPpnKwKdAuSMIEQ==,type:str] openAiApiKey_sops: ENC[AES256_GCM,data:lgevd66PE6OYHueHN5YDCql2WfZXl6I0QNomXPYSvt/vJsStK+kKLVp24wrrmaNiBfqu,iv:ajDHjXd9ETEiTr45G754DosBKgbeMShF7KRsB6/rLXY=,tag:u0ZL9tb7VUC7hR3NtJ/88A==,type:str] sops: @@ -11,8 +10,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-05-13T16:48:53Z" - mac: ENC[AES256_GCM,data:71hY02/irE7cK5jd567uH3hfvUhNykSyNYdoDWtCZX7t+6+jGjh09mwI7O3fikgnxOeh8hsgsGHyA4MMBtEnyoc5w4dMWxV5DC47RgFdImIDtZf+JMDxjUlmvqQp6n59gJSZxkfRD+LjEGvwr5dflV+tALpc03UiZNAfHmg88No=,iv:PhxkCXNzzlQFHU0jVy8QVKHaaE10odKnwUdQVAC7C7Q=,tag:srZfYgoKosqXUoQnAGcd2A==,type:str] + lastmodified: "2024-10-04T21:12:52Z" + mac: ENC[AES256_GCM,data:9ImcQdtnWos3qIwPX+ks1Q+X2nQG5jGO8zfxi4Ww1GgCWOqCp9TzYhI20Th+raJvqlC5XfbPzjPgwANQ7GFTLQVWNrXmKgRUu1XNx6vKkcSoZTe2PjTE4EUB2S2dGobVfdAPjYbBSXQt37m4Sw+h1PVBMn+t9zMXlsBulKoEpRE=,iv:1YHrlavY/VFIP+vEelSl3xKh3jGhNXtGPYagPT1NeTA=,tag:Y0OX3RyH8SlJae8rjW6QUg==,type:str] pgp: [] encrypted_suffix: _sops version: 3.8.1 diff --git a/tests/materialize/materialize-pinecone/fetch-data.go b/tests/materialize/materialize-pinecone/fetch-data.go index 53b289d1fd..c0178d6056 100644 --- a/tests/materialize/materialize-pinecone/fetch-data.go +++ b/tests/materialize/materialize-pinecone/fetch-data.go @@ -1,109 +1,91 @@ package main import ( - "bytes" "context" "encoding/json" "flag" "fmt" "log" - "net/http" "os" "sort" -) - -type fetchReq struct { - TopK int `json:"topK"` - IncludeMetadata bool `json:"includeMetadata"` - IncludeValues bool `json:"includeValues"` - Namespace string `json:"namespace"` - Vector []float64 `json:"vector"` -} - -type fetchRes struct { - Matches []match `json:"matches"` - Namespace string `json:"namespace"` - Results []interface{} `json:"results"` -} + "time" -type match struct { - Id string `json:"id"` - Metadata map[string]interface{} `json:"metadata"` - Score int `json:"score"` - Values interface{} `json:"values"` // Slice of float64 -} + "github.com/pinecone-io/go-pinecone/pinecone" +) var printVectors = flag.Bool("print-vectors", false, "Print the full vectors rather than a placeholder. May be useful for debugging.") +var clearNamespace = flag.Bool("clear-namespace", false, "remove all vectors from the namespace") func main() { - index, ok := os.LookupEnv("PINECONE_INDEX") - if !ok { - log.Fatal("must set PINECONE_INDEX") - } - - env, ok := os.LookupEnv("PINECONE_ENVIRONMENT") - if !ok { - log.Fatal("must set PINECONE_ENVIRONMENT") - } - - apiKey, ok := os.LookupEnv("PINECONE_API_KEY") - if !ok { - log.Fatal("must set PINECONE_API_KEY") - } - - projectId, ok := os.LookupEnv("PINECONE_PROJECT_ID") - if !ok { - log.Fatal("must set PINECONE_PROJECT_ID") - } + ctx := context.Background() flag.Parse() args := flag.Args() if len(args) != 1 { - log.Fatal("must provide namespace to query as a single argument") - } - - ctx := context.Background() - - fetch := fetchReq{ - TopK: 10, - IncludeMetadata: true, - IncludeValues: true, - Namespace: args[0], - Vector: []float64{}, + log.Fatal("must provide namespace to query or clear as a single argument") } - // Dummy query vector containing 1536 (the required length for text-embedding-ada-002) 0's. - for i := 0; i < 1536; i++ { - fetch.Vector = append(fetch.Vector, 0) + pc, err := pinecone.NewClient(pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + SourceTag: "estuary", + }) + if err != nil { + log.Fatal(err) } - body := new(bytes.Buffer) - if err := json.NewEncoder(body).Encode(&fetch); err != nil { + idx, err := pc.DescribeIndex(ctx, os.Getenv("PINECONE_INDEX")) + if err != nil { log.Fatal(err) } - req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://%s-%s.svc.%s.pinecone.io/query", index, projectId, env), body) + conn, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: args[0]}) if err != nil { log.Fatal(err) } - req.Header.Set("accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Api-Key", apiKey) - res, err := http.DefaultClient.Do(req) + if *clearNamespace { + if err := conn.DeleteAllVectorsInNamespace(ctx); err != nil { + log.Fatal(err) + } + fmt.Printf("cleared namespace %s\n", args[0]) + os.Exit(0) + } + + var ids []string + + // Vectors take a long time to be queryable in pinecone serverless + // instances. + for attempt := 0; attempt < 6; attempt++ { + idRes, err := conn.ListVectors(ctx, &pinecone.ListVectorsRequest{}) + if err != nil { + log.Fatal(err) + } + + for _, id := range idRes.VectorIds { + ids = append(ids, *id) + } + + if len(ids) > 0 { + break + } + time.Sleep(10 * time.Second) + } + + vecRes, err := conn.FetchVectors(ctx, ids) if err != nil { log.Fatal(err) } - out := fetchRes{} - if err := json.NewDecoder(res.Body).Decode(&out); err != nil { - log.Fatal(err) + var vecs []pinecone.Vector + + for _, v := range vecRes.Vectors { + vecs = append(vecs, *v) } // Deterministic output ordering, sorted by ID. - sort.Slice(out.Matches, func(i, j int) bool { - return out.Matches[i].Id < out.Matches[j].Id + sort.Slice(vecs, func(i, j int) bool { + return vecs[i].Id < vecs[j].Id }) // The vectors themselves are very long and the floating point values of any point in the vector @@ -112,11 +94,16 @@ func main() { // testing. You can get a decent qualitative idea of if a vector is wrong by how many of its // points are different and how different they are. if !*printVectors { - for idx := range out.Matches { - out.Matches[idx].Values = "" + for idx := range vecs { + vecs[idx].Values = nil } } + out := map[string]any{ + "namespace": args[0], + "results": vecs, + } + formatted, err := json.MarshalIndent(out, "", "\t") if err != nil { log.Fatal(err) diff --git a/tests/materialize/materialize-pinecone/setup.sh b/tests/materialize/materialize-pinecone/setup.sh index 880d807ee7..9de9df2747 100755 --- a/tests/materialize/materialize-pinecone/setup.sh +++ b/tests/materialize/materialize-pinecone/setup.sh @@ -35,9 +35,6 @@ resources_json_template='[ export CONNECTOR_CONFIG="$(decrypt_config ${TEST_DIR}/${CONNECTOR}/config.yaml)" export PINECONE_INDEX="$(echo $CONNECTOR_CONFIG | jq -r .index)" -export PINECONE_ENVIRONMENT="$(echo $CONNECTOR_CONFIG | jq -r .environment)" export PINECONE_API_KEY="$(echo $CONNECTOR_CONFIG | jq -r .pineconeApiKey)" -CONFIG_EXTRA="$(cat ${TEST_DIR}/${CONNECTOR}/config-extra.json)" -export PINECONE_PROJECT_ID="$(echo $CONFIG_EXTRA | jq -r .projectId)" export RESOURCES_CONFIG="$(echo "$resources_json_template" | envsubst | jq -c)" diff --git a/tests/materialize/materialize-pinecone/snapshot.json b/tests/materialize/materialize-pinecone/snapshot.json index 3757950f85..adf265d74f 100644 --- a/tests/materialize/materialize-pinecone/snapshot.json +++ b/tests/materialize/materialize-pinecone/snapshot.json @@ -17,222 +17,169 @@ } ] { - "matches": [ + "namespace": "simple", + "results": [ { - "id": "FQE", + "id": "1501", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"13814000-1dd2-11b2-8000-071353030311\"},\"canary\":\"amputation's\",\"id\":1}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQI", + "id": "1502", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1419d680-1dd2-11b2-8000-071353030311\"},\"canary\":\"armament's\",\"id\":2}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQM", + "id": "1503", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"14b26d00-1dd2-11b2-8000-071353030311\"},\"canary\":\"splatters\",\"id\":3}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQQ", + "id": "1504", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"154b0380-1dd2-11b2-8000-071353030311\"},\"canary\":\"strengthen\",\"id\":4}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQU", + "id": "1505", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"15e39a00-1dd2-11b2-8000-071353030311\"},\"canary\":\"Kringle's\",\"id\":5}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQY", + "id": "1506", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"167c3080-1dd2-11b2-8000-071353030311\"},\"canary\":\"grosbeak's\",\"id\":6}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQc", + "id": "1507", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"7545a800-1dda-11b2-8000-071353030311\"},\"canary\":\"pieced\",\"id\":7}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQg", + "id": "1508", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"75de3e80-1dda-11b2-8000-071353030311\"},\"canary\":\"roaches\",\"id\":8}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQk", + "id": "1509", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"7676d500-1dda-11b2-8000-071353030311\"},\"canary\":\"devilish\",\"id\":9}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQo", + "id": "150a", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"770f6b80-1dda-11b2-8000-071353030311\"},\"canary\":\"glucose's\",\"id\":10}" - }, - "score": 0, - "values": "" + } } - ], - "namespace": "simple", - "results": [] + ] } { - "matches": [ + "namespace": "duplicated-keys", + "results": [ { - "id": "FQE", + "id": "1501", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"77a80200-1dda-11b2-8000-071353030311\"},\"id\":1,\"int\":6,\"str\":\"str 6\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQI", + "id": "1502", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"78409880-1dda-11b2-8000-071353030311\"},\"id\":2,\"int\":7,\"str\":\"str 7\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQM", + "id": "1503", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"78d92f00-1dda-11b2-8000-071353030311\"},\"id\":3,\"int\":8,\"str\":\"str 8\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQQ", + "id": "1504", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"7971c580-1dda-11b2-8000-071353030311\"},\"id\":4,\"int\":9,\"str\":\"str 9\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQU", + "id": "1505", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"7a0a5c00-1dda-11b2-8000-071353030311\"},\"id\":5,\"int\":10,\"str\":\"str 10\"}" - }, - "score": 0, - "values": "" + } } - ], - "namespace": "duplicated-keys", - "results": [] + ] } { - "matches": [ + "namespace": "multiple-types", + "results": [ { - "id": "FQE", + "id": "1501", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1b40e480-1dd2-11b2-8000-071353030311\"},\"array_int\":[11,12],\"bool_field\":false,\"float_field\":1.1,\"id\":1,\"multiple\":1,\"nested\":{\"id\":\"i1\"},\"nullable_int\":null,\"str_field\":\"str1\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQI", + "id": "1502", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1bd97b00-1dd2-11b2-8000-071353030311\"},\"array_int\":[21,22],\"bool_field\":true,\"float_field\":2.2,\"id\":2,\"multiple\":2.2,\"nested\":{\"id\":\"i2\"},\"nullable_int\":2,\"str_field\":\"str2\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQM", + "id": "1503", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1c721180-1dd2-11b2-8000-071353030311\"},\"array_int\":[31,32],\"bool_field\":false,\"float_field\":3.3,\"id\":3,\"multiple\":true,\"nested\":{\"id\":\"i3\"},\"nullable_int\":null,\"str_field\":\"str3\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQQ", + "id": "1504", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1d0aa800-1dd2-11b2-8000-071353030311\"},\"array_int\":[41,42],\"bool_field\":true,\"float_field\":4.4,\"id\":4,\"multiple\":false,\"nested\":{\"id\":\"i4\"},\"nullable_int\":4,\"str_field\":\"str4\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQU", + "id": "1505", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"1da33e80-1dd2-11b2-8000-071353030311\"},\"array_int\":[51,52],\"bool_field\":false,\"float_field\":5.5,\"id\":5,\"multiple\":\"string five\",\"nested\":{\"id\":\"i5\"},\"nullable_int\":null,\"str_field\":\"str5\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQY", + "id": "1506", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"8098d380-1dda-11b2-8000-071353030311\"},\"array_int\":[61,62],\"bool_field\":true,\"float_field\":66.66,\"id\":6,\"multiple\":[\"one\",2,true],\"nested\":{\"id\":\"i6\"},\"nullable_int\":6,\"str_field\":\"str6 v2\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQc", + "id": "1507", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"81316a00-1dda-11b2-8000-071353030311\"},\"array_int\":[71,72],\"bool_field\":false,\"float_field\":77.77,\"id\":7,\"multiple\":{\"object\":\"seven\"},\"nested\":{\"id\":\"i7\"},\"nullable_int\":null,\"str_field\":\"str7 v2\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQg", + "id": "1508", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"81ca0080-1dda-11b2-8000-071353030311\"},\"array_int\":[81,82],\"bool_field\":true,\"float_field\":88.88,\"id\":8,\"multiple\":null,\"nested\":{\"id\":\"i8\"},\"nullable_int\":8,\"str_field\":\"str8 v2\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQk", + "id": "1509", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"82629700-1dda-11b2-8000-071353030311\"},\"array_int\":[91,92],\"binary_field\":\"YWxvaGEK\",\"bool_field\":false,\"float_field\":99.99,\"id\":9,\"nested\":{\"id\":\"i9\"},\"nullable_int\":null,\"str_field\":\"str9 v2\"}" - }, - "score": 0, - "values": "" + } }, { - "id": "FQo", + "id": "150a", "metadata": { "flow_document": "{\"_meta\":{\"uuid\":\"82fb2d80-1dda-11b2-8000-071353030311\"},\"array_int\":[1,2],\"binary_field\":\"c2F5xY1uYXJhCg==\",\"bool_field\":true,\"float_field\":1010.101,\"id\":10,\"nested\":{\"id\":\"i10\"},\"nullable_int\":10,\"str_field\":\"str10 v2\"}" - }, - "score": 0, - "values": "" + } } - ], - "namespace": "multiple-types", - "results": [] + ] } [ "applied.actionDescription",