Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GODRIVER-2859 Add search index management helpers #1311

Merged
merged 15 commits into from
Sep 19, 2023
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,17 @@ functions:
params:
file: lb-expansion.yml

run-search-index-tests:
- command: shell.exec
type: test
params:
shell: "bash"
working_dir: src/go.mongodb.org/mongo-driver
script: |
${PREPARE_SHELL}
TEST_INDEX_URI="${TEST_INDEX_URI}" \
make evg-test-search-index

stop-load-balancer:
- command: shell.exec
params:
Expand Down Expand Up @@ -2277,6 +2288,14 @@ tasks:
${PREPARE_SHELL}
./.evergreen/run-deployed-lambda-aws-tests.sh

- name: "test-search-index"
commands:
- func: "bootstrap-mongo-orchestration"
vars:
VERSION: "latest"
TOPOLOGY: "replica_set"
- func: "run-search-index-tests"

axes:
- id: version
display_name: MongoDB Version
Expand Down Expand Up @@ -2635,6 +2654,44 @@ task_groups:
tasks:
- test-aws-lambda-deployed

- name: test-search-index-task-group
setup_group:
- func: fetch-source
- func: prepare-resources
- command: subprocess.exec
params:
working_dir: src/go.mongodb.org/mongo-driver
binary: bash
add_expansions_to_env: true
env:
MONGODB_VERSION: "7.0"
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
- command: expansions.update
params:
file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml
- command: shell.exec
params:
working_dir: src/go.mongodb.org/mongo-driver
shell: bash
script: |-
echo "TEST_INDEX_URI: ${MONGODB_URI}" > atlas-expansion.yml
- command: expansions.update
params:
file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml
teardown_group:
- command: subprocess.exec
params:
working_dir: src/go.mongodb.org/mongo-driver
binary: bash
add_expansions_to_env: true
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
setup_group_can_fail_task: true
setup_group_timeout_secs: 1800
tasks:
- test-search-index

buildvariants:
- name: static-analysis
display_name: "Static Analysis"
Expand Down Expand Up @@ -2784,6 +2841,12 @@ buildvariants:
tasks:
- test-aws-lambda-task-group

- matrix_name: "searchindex-test"
matrix_spec: { version: ["7.0"], os-faas-80: ["rhel80-large-go-1-20"] }
display_name: "Search Index ${version} ${os-faas-80}"
tasks:
- test-search-index-task-group
qingyang-hu marked this conversation as resolved.
Show resolved Hide resolved

- name: testgcpkms-variant
display_name: "GCP KMS"
run_on:
Expand Down
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ evg-test-load-balancers:
go test $(BUILD_TAGS) ./mongo/integration -run TestLoadBalancerSupport -v -timeout $(TEST_TIMEOUT)s >> test.suite
go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite

.PHONY: evg-test-search-index
evg-test-search-index:
go test ./mongo/integration -run TestSearchIndexProse -v -timeout $(TEST_TIMEOUT)s
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should append to the test.suite file so that the test output can be parsed by Evergreen.

Suggested change
go test ./mongo/integration -run TestSearchIndexProse -v -timeout $(TEST_TIMEOUT)s
go test ./mongo/integration -run TestSearchIndexProse -v -timeout $(TEST_TIMEOUT)s >> test.suite


.PHONY: evg-test-ocsp
evg-test-ocsp:
go test -v ./mongo -run TestOCSP $(OCSP_TLS_SHOULD_SUCCEED) >> test.suite
Expand Down
16 changes: 11 additions & 5 deletions bson/raw.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,18 @@ func (r Raw) LookupErr(key ...string) (RawValue, error) {
// elements. If the document is not valid, the elements up to the invalid point will be returned
// along with an error.
func (r Raw) Elements() ([]RawElement, error) {
elems, err := bsoncore.Document(r).Elements()
relems := make([]RawElement, 0, len(elems))
for _, elem := range elems {
relems = append(relems, RawElement(elem))
var relems []RawElement
if doc := bsoncore.Document(r); len(doc) > 0 {
elems, err := doc.Elements()
if err != nil {
return relems, err
}
relems = make([]RawElement, 0, len(elems))
for _, elem := range elems {
relems = append(relems, RawElement(elem))
}
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional: Consider inverting the if condition and returning early so you can un-indent the remaining code.

E.g.

func (r Raw) Elements() ([]RawElement, error) {
	doc := bsoncore.Document(r)
	if len(doc) == 0 {
		return nil, nil
	}

	elems, err := doc.Elements()
	if err != nil {
		return nil, err
	}

	relems := make([]RawElement, 0, len(elems))
	for _, elem := range elems {
		relems = append(relems, RawElement(elem))
	}

	return relems, nil
}

return relems, err
return relems, nil
}

// Values returns this document as a slice of values. The returned slice will contain valid values.
Expand Down
7 changes: 7 additions & 0 deletions mongo/collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -1773,6 +1773,13 @@ func (coll *Collection) Indexes() IndexView {
return IndexView{coll: coll}
}

// SearchIndexes returns a SearchIndexView instance that can be used to perform operations on the search indexes for the collection.
func (coll *Collection) SearchIndexes() SearchIndexView {
return SearchIndexView{
coll: coll,
}
}

// Drop drops the collection on the server. This method ignores "namespace not found" errors so it is safe to drop
// a collection that does not exist on the server.
func (coll *Collection) Drop(ctx context.Context) error {
Expand Down
255 changes: 255 additions & 0 deletions mongo/integration/search_index_prose_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
// Copyright (C) MongoDB, Inc. 2023-present.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0

package integration

import (
"context"
"os"
"sync"
"testing"
"time"

"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/internal/assert"
"go.mongodb.org/mongo-driver/internal/require"
"go.mongodb.org/mongo-driver/internal/uuid"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/integration/mtest"
"go.mongodb.org/mongo-driver/mongo/options"
)

func TestSearchIndexProse(t *testing.T) {
t.Parallel()

const timeout = 5 * time.Minute

uri := os.Getenv("TEST_INDEX_URI")
if uri == "" {
t.Skip("skipping")
}

opts := options.Client().ApplyURI(uri).SetTimeout(timeout)
mt := mtest.New(t, mtest.NewOptions().ClientOptions(opts).MinServerVersion("7.0").Topologies(mtest.ReplicaSet))

mt.Run("case 1: Driver can successfully create and list search indexes", func(mt *mtest.T) {
ctx := context.Background()

_, err := mt.Coll.InsertOne(ctx, bson.D{})
require.NoError(mt, err, "failed to insert")

view := mt.Coll.SearchIndexes()

definition := bson.D{{"mappings", bson.D{{"dynamic", false}}}}
searchName := "test-search-index"
model := mongo.SearchIndexModel{
Definition: definition,
Name: &searchName,
}
index, err := view.CreateOne(ctx, model)
require.NoError(mt, err, "failed to create index")
require.Equal(mt, searchName, index, "unmatched name")

var doc bson.Raw
for doc == nil {
cursor, err := view.List(ctx, &index)
require.NoError(mt, err, "failed to list")

if !cursor.Next(ctx) {
break
}
if cursor.Current.Lookup("queryable").Boolean() {
doc = cursor.Current
} else {
t.Logf("cursor: %s, sleep 5 seconds...", cursor.Current.String())
time.Sleep(5 * time.Second)
}
}
require.NotNil(mt, doc, "got empty document")
assert.Equal(mt, searchName, doc.Lookup("name").StringValue(), "unmatched name")
expected, err := bson.Marshal(definition)
require.NoError(mt, err, "failed to marshal definition")
actual := doc.Lookup("latestDefinition").Value
assert.Equal(mt, expected, actual, "unmatched definition")
})

mt.Run("case 2: Driver can successfully create multiple indexes in batch", func(mt *mtest.T) {
ctx := context.Background()

_, err := mt.Coll.InsertOne(ctx, bson.D{})
require.NoError(mt, err, "failed to insert")

view := mt.Coll.SearchIndexes()

definition := bson.D{{"mappings", bson.D{{"dynamic", false}}}}
searchNames := []string{"test-search-index-1", "test-search-index-2"}
models := make([]mongo.SearchIndexModel, len(searchNames))
for i := range searchNames {
models[i] = mongo.SearchIndexModel{
Definition: definition,
Name: &searchNames[i],
}
}
indexes, err := view.CreateMany(ctx, models)
require.NoError(mt, err, "failed to create index")
require.Equal(mt, len(indexes), 2, "expected 2 indexes")
for _, searchName := range searchNames {
require.Contains(mt, indexes, searchName)
}

getDocument := func(index string) bson.Raw {
t.Helper()

for {
cursor, err := view.List(ctx, &index)
require.NoError(mt, err, "failed to list")

if !cursor.Next(ctx) {
return nil
}
if cursor.Current.Lookup("queryable").Boolean() {
return cursor.Current
}
t.Logf("cursor: %s, sleep 5 seconds...", cursor.Current.String())
time.Sleep(5 * time.Second)
}
}

var wg sync.WaitGroup
wg.Add(len(searchNames))
for i := range searchNames {
go func(name string) {
defer wg.Done()

doc := getDocument(name)
require.NotNil(mt, doc, "got empty document")
assert.Equal(mt, name, doc.Lookup("name").StringValue(), "unmatched name")
expected, err := bson.Marshal(definition)
require.NoError(mt, err, "failed to marshal definition")
actual := doc.Lookup("latestDefinition").Value
assert.Equal(mt, expected, actual, "unmatched definition")
}(searchNames[i])
}
wg.Wait()
})

mt.Run("case 3: Driver can successfully drop search indexes", func(mt *mtest.T) {
ctx := context.Background()

_, err := mt.Coll.InsertOne(ctx, bson.D{})
require.NoError(mt, err, "failed to insert")

view := mt.Coll.SearchIndexes()

definition := bson.D{{"mappings", bson.D{{"dynamic", false}}}}
searchName := "test-search-index"
model := mongo.SearchIndexModel{
Definition: definition,
Name: &searchName,
}
index, err := view.CreateOne(ctx, model)
require.NoError(mt, err, "failed to create index")
require.Equal(mt, searchName, index, "unmatched name")

var doc bson.Raw
for doc == nil {
cursor, err := view.List(ctx, &index)
require.NoError(mt, err, "failed to list")

if !cursor.Next(ctx) {
break
}
if cursor.Current.Lookup("queryable").Boolean() {
doc = cursor.Current
} else {
t.Logf("cursor: %s, sleep 5 seconds...", cursor.Current.String())
time.Sleep(5 * time.Second)
}
}
require.NotNil(mt, doc, "got empty document")
require.Equal(mt, searchName, doc.Lookup("name").StringValue(), "unmatched name")

err = view.DropOne(ctx, searchName)
require.NoError(mt, err, "failed to drop index")
for {
cursor, err := view.List(ctx, &index)
require.NoError(mt, err, "failed to list")

if !cursor.Next(ctx) {
break
}
t.Logf("cursor: %s, sleep 5 seconds...", cursor.Current.String())
time.Sleep(5 * time.Second)
}
})

mt.Run("case 4: Driver can update a search index", func(mt *mtest.T) {
ctx := context.Background()

_, err := mt.Coll.InsertOne(ctx, bson.D{})
require.NoError(mt, err, "failed to insert")

view := mt.Coll.SearchIndexes()

definition := bson.D{{"mappings", bson.D{{"dynamic", false}}}}
searchName := "test-search-index"
model := mongo.SearchIndexModel{
Definition: definition,
Name: &searchName,
}
index, err := view.CreateOne(ctx, model)
require.NoError(mt, err, "failed to create index")
require.Equal(mt, searchName, index, "unmatched name")

getDocument := func(index string) bson.Raw {
t.Helper()

for {
cursor, err := view.List(ctx, &index)
require.NoError(mt, err, "failed to list")

if !cursor.Next(ctx) {
return nil
}
if cursor.Current.Lookup("queryable").Boolean() {
return cursor.Current
}
t.Logf("cursor: %s, sleep 5 seconds...", cursor.Current.String())
time.Sleep(5 * time.Second)
}
}

doc := getDocument(searchName)
require.NotNil(mt, doc, "got empty document")
require.Equal(mt, searchName, doc.Lookup("name").StringValue(), "unmatched name")

definition = bson.D{{"mappings", bson.D{{"dynamic", true}}}}
err = view.UpdateOne(ctx, searchName, definition)
require.NoError(mt, err, "failed to drop index")
doc = getDocument(searchName)
require.NotNil(mt, doc, "got empty document")
assert.Equal(mt, searchName, doc.Lookup("name").StringValue(), "unmatched name")
assert.Equal(mt, "READY", doc.Lookup("status").StringValue(), "unexpected status")
expected, err := bson.Marshal(definition)
require.NoError(mt, err, "failed to marshal definition")
actual := doc.Lookup("latestDefinition").Value
assert.Equal(mt, expected, actual, "unmatched definition")
})

mt.Run("case 5: dropSearchIndex suppresses namespace not found errors", func(mt *mtest.T) {
ctx := context.Background()

id, err := uuid.New()
require.NoError(mt, err)

collection := mt.CreateCollection(mtest.Collection{
Name: id.String(),
}, false)

err = collection.SearchIndexes().DropOne(ctx, "foo")
require.NoError(mt, err)
})
}
Loading
Loading