From a88ca3be28db329e87e4dd71bacd43a4d546ebce Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 22 Oct 2024 08:55:44 +0100 Subject: [PATCH 1/2] Add metadata search to mongodb --- src/IntegrationTests/DatabaseTests.cs | 2 +- src/Mongo/src/MongoVectorCollection.cs | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/IntegrationTests/DatabaseTests.cs b/src/IntegrationTests/DatabaseTests.cs index 6b9e4e0..480a2a0 100644 --- a/src/IntegrationTests/DatabaseTests.cs +++ b/src/IntegrationTests/DatabaseTests.cs @@ -298,10 +298,10 @@ public async Task SimilaritySearchWithScores_Ok(SupportedDatabase database) } [TestCase(SupportedDatabase.InMemory)] - //[TestCase(SupportedDatabase.Chroma)] //[TestCase(SupportedDatabase.OpenSearch)] //[TestCase(SupportedDatabase.Postgres)] [TestCase(SupportedDatabase.SqLite)] + [TestCase(SupportedDatabase.Mongo)] //[TestCase(SupportedDatabase.DuckDb)] //[TestCase(SupportedDatabase.Weaviate)] //[TestCase(SupportedDatabase.Elasticsearch)] diff --git a/src/Mongo/src/MongoVectorCollection.cs b/src/Mongo/src/MongoVectorCollection.cs index 1537e12..e31e444 100644 --- a/src/Mongo/src/MongoVectorCollection.cs +++ b/src/Mongo/src/MongoVectorCollection.cs @@ -71,8 +71,27 @@ public async Task SearchAsync(VectorSearchRequest request, }; } - Task> IVectorCollection.SearchByMetadata(Dictionary filters, CancellationToken cancellationToken) + public async Task> SearchByMetadata(Dictionary filters, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + filters = filters ?? throw new ArgumentNullException(nameof(filters)); + + var builder = Builders.Filter; + var filterDefinitions = new List>(); + + foreach (var kvp in filters) + { + // Assuming your Vector class has a Metadata field of type Dictionary + var filter = builder.Eq($"Metadata.{kvp.Key}", kvp.Value); + filterDefinitions.Add(filter); + } + + var combinedFilter = builder.And(filterDefinitions); + + var results = await _mongoCollection + .Find(combinedFilter) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return results; } -} +} \ No newline at end of file From f73459c746aa9a7f99975a400c6c86c9a638adad Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 23 Oct 2024 08:34:59 +0100 Subject: [PATCH 2/2] Added coderabbit suggestions --- src/IntegrationTests/DatabaseTests.cs | 27 +++++++++++++++++++++++++- src/Mongo/src/MongoVectorCollection.cs | 7 ++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/IntegrationTests/DatabaseTests.cs b/src/IntegrationTests/DatabaseTests.cs index 480a2a0..37a19c5 100644 --- a/src/IntegrationTests/DatabaseTests.cs +++ b/src/IntegrationTests/DatabaseTests.cs @@ -1,5 +1,7 @@ using LangChain.DocumentLoaders; using LangChain.Extensions; +using StackExchange.Redis; +using System; namespace LangChain.Databases.IntegrationTests; @@ -336,7 +338,30 @@ public async Task MetadataSearch_Ok(SupportedDatabase database) var items = await vectorCollection.SearchByMetadata(filters); totalItems.Should().HaveCount(2); + items.Should().HaveCount(1); - var vector = items.SingleOrDefault()?.Metadata?["color"].Should().Be("orange"); + var result = items.Single(); + result.Text.Should().Be("orange"); + result.Metadata.Should().ContainKey("color"); + result.Metadata?["color"].Should().Be("orange"); + } + + [TestCase(SupportedDatabase.InMemory)] + //[TestCase(SupportedDatabase.OpenSearch)] + //[TestCase(SupportedDatabase.Postgres)] + [TestCase(SupportedDatabase.SqLite)] + [TestCase(SupportedDatabase.Mongo)] + //[TestCase(SupportedDatabase.DuckDb)] + //[TestCase(SupportedDatabase.Weaviate)] + //[TestCase(SupportedDatabase.Elasticsearch)] + //[TestCase(SupportedDatabase.Milvus)] + public async Task SearchByMetadata_WithNullFilters_ThrowsArgumentException(SupportedDatabase database) + { + await using var environment = await StartEnvironmentForAsync(database); + var vectorCollection = await environment.VectorDatabase.GetOrCreateCollectionAsync(environment.CollectionName, dimensions: environment.Dimensions); + + // Act & Assert + await vectorCollection.Invoking(v => v.SearchByMetadata(null!)) + .Should().ThrowAsync(); } } \ No newline at end of file diff --git a/src/Mongo/src/MongoVectorCollection.cs b/src/Mongo/src/MongoVectorCollection.cs index e31e444..0da1a83 100644 --- a/src/Mongo/src/MongoVectorCollection.cs +++ b/src/Mongo/src/MongoVectorCollection.cs @@ -11,7 +11,7 @@ public class MongoVectorCollection( string? id = null) : VectorCollection(name, id), IVectorCollection { - private IMongoCollection _mongoCollection = mongoContext.GetCollection(name); + private readonly IMongoCollection _mongoCollection = mongoContext.GetCollection(name); public async Task> AddAsync(IReadOnlyCollection items, CancellationToken cancellationToken = default) { @@ -80,6 +80,11 @@ public async Task> SearchByMetadata(Dictionary filt foreach (var kvp in filters) { + if (kvp.Value == null) + { + throw new ArgumentException($"Metadata value for key '{kvp.Key}' cannot be null", nameof(filters)); + } + // Assuming your Vector class has a Metadata field of type Dictionary var filter = builder.Eq($"Metadata.{kvp.Key}", kvp.Value); filterDefinitions.Add(filter);