Skip to content

Commit

Permalink
Properly implement dynamic field support
Browse files Browse the repository at this point in the history
  • Loading branch information
@roji @weianweigan
roji authored and weianweigan committed Sep 12, 2023
1 parent 43aaa15 commit ea6ec4f
Show file tree
Hide file tree
Showing 7 changed files with 425 additions and 36 deletions.
44 changes: 32 additions & 12 deletions Milvus.Client.Tests/DataTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,6 @@ public async Task Timestamp_conversion()
{
DateTime before = DateTime.UtcNow;

await Client.CreateCollectionAsync(
CollectionName,
new[]
{
FieldSchema.Create<long>("id", isPrimaryKey: true),
FieldSchema.CreateFloatVector("float_vector", 2)
});

await Task.Delay(100);

MutationResult mutationResult = await InsertDataAsync(3, 4);
Expand Down Expand Up @@ -140,6 +132,37 @@ public async Task FlushAllAsync_and_wait()
Assert.True(segmentInfos.All(p => p.State == SegmentState.Flushed));
}

[Fact]
public async Task Insert_dynamic_field()
{
await Collection.DropAsync();

await TestEnvironment.Client.CreateCollectionAsync(
Collection.Name,
new CollectionSchema
{
Fields =
{
FieldSchema.Create<long>("id", isPrimaryKey: true),
FieldSchema.CreateFloatVector("float_vector", 2)
},
EnableDynamicFields = true
});

await Collection.InsertAsync(
new FieldData[]
{
FieldData.Create("id", new[] { 1L, 2L }),
FieldData.CreateFloatVector("float_vector", new ReadOnlyMemory<float>[]
{
new[] { 1f, 2f },
new[] { 3f, 4f }
}),
FieldData.CreateVarChar("unknown_varchar", new[] { "dynamic str1", "dynamic str2" }, isDynamic: true),
FieldData.Create("unknown_int", new[] { 8L, 9L }, isDynamic: true)
});
}

private async Task<MutationResult> InsertDataAsync(long id1, long id2)
=> await Collection.InsertAsync(
new FieldData[]
Expand All @@ -154,10 +177,7 @@ private async Task<MutationResult> InsertDataAsync(long id1, long id2)

public class DataCollectionFixture : IAsyncLifetime
{
public MilvusCollection Collection { get; }

public DataCollectionFixture()
=> Collection = TestEnvironment.Client.GetCollection(CollectionName);
public MilvusCollection Collection { get; } = TestEnvironment.Client.GetCollection(CollectionName);

public async Task InitializeAsync()
{
Expand Down
172 changes: 164 additions & 8 deletions Milvus.Client.Tests/SearchQueryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,82 @@ public async Task Query_with_limit()
Assert.Collection(idData.Data, id => Assert.Equal(2, id));
}

[Fact]
public async Task Query_dynamic_field()
{
await Collection.DropAsync();

await Client.CreateCollectionAsync(
Collection.Name,
new CollectionSchema
{
Fields =
{
FieldSchema.Create<long>("id", isPrimaryKey: true),
FieldSchema.CreateFloatVector("float_vector", 2)
},
EnableDynamicFields = true
});

await Collection.CreateIndexAsync(
"float_vector", IndexType.Flat, SimilarityMetricType.L2, "float_vector_idx", new Dictionary<string, string>());

await Collection.InsertAsync(
new FieldData[]
{
FieldData.Create("id", new[] { 1L, 2L, 3L }),
FieldData.CreateFloatVector("float_vector", new ReadOnlyMemory<float>[]
{
new[] { 1f, 2f },
new[] { 3f, 4f },
new[] { 5f, 6f }
}),
FieldData.CreateVarChar("dynamic_varchar", new[] { "str1", "str2", "str3" }, isDynamic: true),
FieldData.Create("dynamic_long", new[] { 8L, 9L, 10L }, isDynamic: true),
FieldData.Create("dynamic_bool", new[] { true, false, true }, isDynamic: true)
});

await Collection.LoadAsync();
await Collection.WaitForCollectionLoadAsync(
waitingInterval: TimeSpan.FromMilliseconds(100), timeout: TimeSpan.FromMinutes(1));

IReadOnlyList<FieldData> fields = await Collection.QueryAsync(
"dynamic_long > 8",
new QueryParameters { OutputFields = { "*" } });

var idData = (FieldData<long>)Assert.Single(fields, d => d.FieldName == "id");
Assert.Equal(MilvusDataType.Int64, idData.DataType);
Assert.Equal(2, idData.RowCount);
Assert.False(idData.IsDynamic);
Assert.Collection(idData.Data,
id => Assert.Equal(2, id),
id => Assert.Equal(3, id));

var dynamicVarcharData = (FieldData<string>)Assert.Single(fields, d => d.FieldName == "dynamic_varchar");
Assert.Equal(MilvusDataType.VarChar, dynamicVarcharData.DataType);
Assert.Equal(2, dynamicVarcharData.RowCount);
Assert.True(dynamicVarcharData.IsDynamic);
Assert.Collection(dynamicVarcharData.Data,
s => Assert.Equal("str2", s),
s => Assert.Equal("str3", s));

var dynamicLongData = (FieldData<long>)Assert.Single(fields, d => d.FieldName == "dynamic_long");
Assert.Equal(MilvusDataType.Int64, dynamicLongData.DataType);
Assert.Equal(2, dynamicLongData.RowCount);
Assert.True(dynamicVarcharData.IsDynamic);
Assert.Collection(dynamicLongData.Data,
i => Assert.Equal(9L, i),
i => Assert.Equal(10L, i));

var dynamicBoolData = (FieldData<bool>)Assert.Single(fields, d => d.FieldName == "dynamic_bool");
Assert.Equal(MilvusDataType.Bool, dynamicBoolData.DataType);
Assert.Equal(2, dynamicBoolData.RowCount);
Assert.True(dynamicBoolData.IsDynamic);
Assert.Collection(dynamicBoolData.Data,
Assert.False,
Assert.True);
}

[Fact]
public async Task Search_with_minimal_inputs()
{
Expand Down Expand Up @@ -224,13 +300,96 @@ public async Task Search_with_json_filter()
Assert.Collection(results.Limits, l => Assert.Equal(2, l));
}

[Fact]
public async Task Search_with_dynamic_field_filter()
{
await Collection.DropAsync();

await Client.CreateCollectionAsync(
Collection.Name,
new CollectionSchema
{
Fields =
{
FieldSchema.Create<long>("id", isPrimaryKey: true),
FieldSchema.CreateFloatVector("float_vector", 2)
},
EnableDynamicFields = true
});

await Collection.CreateIndexAsync(
"float_vector", IndexType.Flat, SimilarityMetricType.L2, "float_vector_idx", new Dictionary<string, string>());

await Collection.InsertAsync(
new FieldData[]
{
FieldData.Create("id", new[] { 1L, 2L, 3L }),
FieldData.CreateFloatVector("float_vector", new ReadOnlyMemory<float>[]
{
new[] { 1f, 2f },
new[] { 3f, 4f },
new[] { 5f, 6f }
}),
FieldData.CreateVarChar("dynamic_varchar", new[] { "str1", "str2", "str3" }, isDynamic: true),
FieldData.Create("dynamic_long", new[] { 8L, 9L, 10L }, isDynamic: true),
FieldData.Create("dynamic_bool", new[] { true, false, true }, isDynamic: true)
});

await Collection.LoadAsync();
await Collection.WaitForCollectionLoadAsync(
waitingInterval: TimeSpan.FromMilliseconds(100), timeout: TimeSpan.FromMinutes(1));

var results = await Collection.SearchAsync(
"float_vector",
new ReadOnlyMemory<float>[] { new[] { 3f, 4f } },
SimilarityMetricType.L2,
limit: 2,
new()
{
Expression = """dynamic_long > 8""",
OutputFields = { "*" }
});

var fields = results.FieldsData;

var idData = (FieldData<long>)Assert.Single(fields, d => d.FieldName == "id");
Assert.Equal(MilvusDataType.Int64, idData.DataType);
Assert.Equal(2, idData.RowCount);
Assert.False(idData.IsDynamic);
Assert.Collection(idData.Data,
id => Assert.Equal(2, id),
id => Assert.Equal(3, id));

var dynamicVarcharData = (FieldData<string>)Assert.Single(fields, d => d.FieldName == "dynamic_varchar");
Assert.Equal(MilvusDataType.VarChar, dynamicVarcharData.DataType);
Assert.Equal(2, dynamicVarcharData.RowCount);
Assert.True(dynamicVarcharData.IsDynamic);
Assert.Collection(dynamicVarcharData.Data,
s => Assert.Equal("str2", s),
s => Assert.Equal("str3", s));

var dynamicLongData = (FieldData<long>)Assert.Single(fields, d => d.FieldName == "dynamic_long");
Assert.Equal(MilvusDataType.Int64, dynamicLongData.DataType);
Assert.Equal(2, dynamicLongData.RowCount);
Assert.True(dynamicVarcharData.IsDynamic);
Assert.Collection(dynamicLongData.Data,
i => Assert.Equal(9L, i),
i => Assert.Equal(10L, i));

var dynamicBoolData = (FieldData<bool>)Assert.Single(fields, d => d.FieldName == "dynamic_bool");
Assert.Equal(MilvusDataType.Bool, dynamicBoolData.DataType);
Assert.Equal(2, dynamicBoolData.RowCount);
Assert.True(dynamicBoolData.IsDynamic);
Assert.Collection(dynamicBoolData.Data,
Assert.False,
Assert.True);
}

[Theory]
[InlineData(IndexType.BinFlat, SimilarityMetricType.Jaccard)]
[InlineData(IndexType.BinFlat, SimilarityMetricType.Hamming)]
[InlineData(IndexType.BinIvfFlat, SimilarityMetricType.Hamming)]
public async Task Search_binary_vector(
IndexType indexType,
SimilarityMetricType similarityMetricType)
public async Task Search_binary_vector(IndexType indexType, SimilarityMetricType similarityMetricType)
{
MilvusCollection binaryVectorCollection = Client.GetCollection(nameof(Search_binary_vector));
string collectionName = binaryVectorCollection.Name;
Expand Down Expand Up @@ -367,10 +526,7 @@ public async Task Search_with_unsupported_vector_type_throws()

public class QueryCollectionFixture : IAsyncLifetime
{
public MilvusCollection Collection { get; }

public QueryCollectionFixture()
=> Collection = TestEnvironment.Client.GetCollection("CollectionName");
public MilvusCollection Collection { get; } = TestEnvironment.Client.GetCollection(nameof(SearchQueryTests));

public async Task InitializeAsync()
{
Expand Down Expand Up @@ -425,7 +581,7 @@ public Task DisposeAsync()

private readonly QueryCollectionFixture _queryCollectionFixture;
private MilvusCollection Collection => _queryCollectionFixture.Collection;
private string CollectionName => Collection.Name; // TODO: Vemo
private string CollectionName => Collection.Name;

public SearchQueryTests(QueryCollectionFixture queryCollectionFixture)
=> _queryCollectionFixture = queryCollectionFixture;
Expand Down
3 changes: 3 additions & 0 deletions Milvus.Client/BinaryVectorFieldData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,7 @@ internal override Grpc.FieldData ToGrpcFieldData()

return result;
}

internal override object GetValueAsObject(int index)
=> throw new NotSupportedException("Dynamic vector fields are not supported");
}
3 changes: 3 additions & 0 deletions Milvus.Client/ByteStringFieldData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,7 @@ internal override Grpc.FieldData ToGrpcFieldData()
}
};
}

internal override object GetValueAsObject(int index)
=> throw new NotSupportedException("Dynamic ByteString fields are not supported");
}
Loading

0 comments on commit ea6ec4f

Please sign in to comment.