Merge pull request #14 from OpenSesame/feature/api_delete

Feature/api delete

.gitignore

@@ -6,6 +6,7 @@ Manifest.toml
 /docs/Manifest.toml
 debug_out
 envs*
+sysimage/
 
 # VS Code
 .vscode/*

ext/OpenSearchExt/OpenSearchExt.jl

@@ -180,6 +180,7 @@ function single_query(
  )
 
  full_query = Dict(
+ :size => query.top_k,
  :query => Dict(
  :bool => Dict(
  :must => [knn_query]

resources/dev-openai/.well-known/openapi.yaml

@@ -9,7 +9,7 @@ paths:
  post:
  summary: Query
  description: Accepts search query objects array each with query and optional filter. Break down complex questions into sub-questions. Refine results by criteria, e.g. time / source, don't do this often. Split queries if ResponseTooLargeError occurs.
- operationId: query_query_post
+ operationId: query_post
  requestBody:
  content:
  application/json:
@@ -41,6 +41,7 @@ paths:
  application/json:
  schema:
  $ref: "#/components/schemas/UpsertRequest"
+ required: true
  responses:
  "200":
  description: Successful Response
@@ -57,7 +58,26 @@ paths:
  security:
  - HTTPBearer: []
  # /upsert-file:
- # /delete:
+ /delete:
+ delete:
+ summary: Delete
+ description: Delete one or more documents
+ operationId: delete_docs
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/DeleteRequest"
+ required: true
+ responses:
+ "200":
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/DeleteResponse"
+ security:
+ - HTTPBearer: []
 components:
  schemas:
  DocumentChunkMetadata:
@@ -299,6 +319,29 @@ components:
  type: array
  items:
  type: string
+
+ DeleteRequest:
+ type: object
+ properties:
+ ids:
+ type: array
+ items:
+ type: string
+ filter:
+ type: array
+ items:
+ $ref: "#/components/schemas/DocumentMetadataFilter"
+ delete_all:
+ type: boolean
+ default: false
+ DeleteResponse:
+ type: object
+ required:
+ - success
+ properties:
+ success:
+ type: boolean
+
  securitySchemes:
  HTTPBearer:
  type: http

src/datastore/datastore.jl

@@ -4,10 +4,10 @@ using ..AppServer
 using Mocking
 using DebugDataWriter
 
-include("datastor_common.jl")
+include("datastore_common.jl")
 include("factory.jl")
 
-STORAGE = nothing
+const STORAGE = Ref{AbstractStorage}()
 
 """
 Takes in a list of documents and inserts them into the database.
@@ -17,10 +17,10 @@ then inserts the new ones.
 Return a list of document ids.
 """
 function upsert(documents::AbstractVector{Document}; chunk_token_size=0)::UpsertResponse
- @assert !isnothing(STORAGE) "Storage is not initialized"
+ @assert isassigned(STORAGE) "Storage is not initialized"
 
  delete(
- STORAGE,
+ STORAGE[],
  filter=map(
  document -> DocumentMetadataFilter(document_id=document.id),
  documents
@@ -29,14 +29,14 @@ function upsert(documents::AbstractVector{Document}; chunk_token_size=0)::Upsert
 
  chunks = AppServer.get_document_chunks(documents, chunk_token_size)
 
- return upsert(STORAGE, chunks)
+ return upsert(STORAGE[], chunks)
 end
 
 """
 Takes in a list of queries and filters and returns a list of query results with matching document chunks and scores.
 """
 function query(queries::AbstractVector{Query})::Vector{QueryResult}
- @assert !isnothing(STORAGE) "Storage is not initialized"
+ @assert isassigned(STORAGE) "Storage is not initialized"
 
  # get a list of of just the queries from the Query list
  query_texts = [query.query for query in queries]
@@ -55,10 +55,30 @@ function query(queries::AbstractVector{Query})::Vector{QueryResult}
 
  @debug_output get_debug_id("datastore") "storage query" queries_with_embeddings
 
- return query(STORAGE, queries_with_embeddings)
+ return query(STORAGE[], queries_with_embeddings)
+end
+
+"""
+Removes documents by ids, filter or all together
+Multiple parameters can be used at once.
+
+Returns whether the operation was successful.
+"""
+function delete(;
+ ids::Union{Vector{<:AbstractString}, Nothing},
+ filter::Union{Vector{DocumentMetadataFilter}, Nothing},
+ delete_all::Bool
+)::Bool
+ @assert isassigned(STORAGE) "Storage is not initialized"
+
+ delete_all && return DataStore.delete_all(STORAGE[])
+
+ doc_filter = isnothing(filter) ? map(id -> DocumentMetadataFilter(document_id=id), ids) : filter
+
+ return delete(STORAGE[]; filter=doc_filter)
 end
 
 function __init__()
- global STORAGE = get_datastore()
+ global STORAGE[] = get_datastore()
 end
 end

src/datastore/datastor_common.jl → src/datastore/datastore_common.jl

@@ -2,8 +2,6 @@ using ..GptPluginServer
 
 abstract type AbstractStorage end
 
-include("teststorage.jl")
-
 """
 Takes in a list of list of document chunks and inserts them into the database.
 
@@ -45,6 +43,6 @@ Removes everything in the datastore
 
 Returns whether the operation was successful.
 """
-function delete_all(storage::AbstractStorage)
+function delete_all(storage::AbstractStorage)::Bool
  error("The method 'delete_all' is not implemeted for $(typeof(storage))")
 end

src/datastore/factory.jl

@@ -2,15 +2,19 @@
 using Pkg
 datastore_env() = get(ENV, "DATASTORE", "")
 
-global DATASTORE_MODULE = let
+const DATASTORE_MODULE = Ref{Module}()
+
+begin
  datastore = datastore_env() |> uppercase
 
+ @info string("Requested datastore: ", datastore_env())
+
  # workaround to detect running in test mode
  if isnothing(Pkg.project().name)
  datastore = "TEST"
  end
 
- if isequal(datastore, "ELASTICSEARCH")
+ global DATASTORE_MODULE[] = if isequal(datastore, "ELASTICSEARCH")
  @info "Pluging ElasticsearchClientExt"
  include("../../ext/ElasticsearchClientExt/ElasticsearchClientExt.jl")
 
@@ -21,15 +25,18 @@ global DATASTORE_MODULE = let
 
  OpenSearchExt
  elseif isequal(datastore, "TEST")
+ @info "Dummy storage for the logic check only"
+ include("teststorage.jl")
+
  TestStorageExt
  else
- nothing
+ error("DATASTORE environment variable must be non empty and valid")
  end
 end
-using Pkg
+
 function get_datastore()::Union{AbstractStorage,Nothing}
- @info string("Requested datastore: ", datastore_env())
- isnothing(DATASTORE_MODULE) && error("DATASTORE environment variable must be non empty and valid")
+ global DATASTORE_MODULE
+ isassigned(DATASTORE_MODULE) || error("DATASTORE environment variable must be non empty and valid")
 
- DATASTORE_MODULE.create_storage()
+ DATASTORE_MODULE[].create_storage()
 end

src/datastore/teststorage.jl

@@ -1,7 +1,63 @@
+# Test storage module for logic check only
 module TestStorageExt
+using ..DataStore
 using ..DataStore: AbstractStorage
+
+using ...AppServer:
+ DocumentChunk, 
+ QueryWithEmbedding, QueryResult, 
+ DocumentMetadataFilter, DocumentChunkWithScore,
+ UpsertResponse
+
 mutable struct TestStorage <: AbstractStorage
- data::Any
+ data::Vector{String}
+
+ TestStorage() = new([])
 end
-create_storage() = TestStorage(nothing)
+create_storage() = TestStorage()
+
+function DataStore.upsert(
+ storage::TestStorage,
+ chunks::Dict{String, <:AbstractVector{DocumentChunk}},
+)::UpsertResponse
+ # add documents ids only into the storage-array
+ ids = keys(chunks) |> collect
+ append!(storage.data, ids)
+ return UpsertResponse(ids = ids)
+end
+
+function DataStore.query(
+ storage::TestStorage,
+ queries::AbstractVector{QueryWithEmbedding},
+)::Vector{QueryResult}
+ isempty(storage.data) && return []
+
+ # assume the query contains exact document id as we are storing
+ # Otherwise, we need to implement a real mechanism for calculating embeddings. 
+ return map(queries) do query_with_embedding
+ text_query = query_with_embedding.query
+ res = findfirst(s -> isequal(s, text_query), storage.data)
+
+ QueryResult(
+ query = text_query,
+ results = isnothing(res) ? [] : [DocumentChunkWithScore(id = text_query, score = 1)],
+ )
+ end
+end
+
+function DataStore.delete(
+ storage::TestStorage;
+ filter::Vector{DocumentMetadataFilter},
+)::Bool
+ ids = getfield.(filter, :document_id)
+ initial_length = length(storage.data)
+ filter!(i -> i ∉ ids, storage.data)
+ return initial_length != length(storage.data)
+end
+
+function DataStore.delete_all(storage::TestStorage)::Bool
+ empty!(storage.data)
+ return true
+end
+
 end

src/generated/.openapi-generator/FILES

@@ -1,5 +1,7 @@
 README.md
 docs/DefaultApi.md
+docs/DeleteRequest.md
+docs/DeleteResponse.md
 docs/Document.md
 docs/DocumentChunk.md
 docs/DocumentChunkMetadata.md
@@ -20,6 +22,8 @@ docs/ValidationError.md
 src/GptPluginServer.jl
 src/apis/api_DefaultApi.jl
 src/modelincludes.jl
+src/models/model_DeleteRequest.jl
+src/models/model_DeleteResponse.jl
 src/models/model_Document.jl
 src/models/model_DocumentChunk.jl
 src/models/model_DocumentChunkMetadata.jl

src/generated/README.md

@@ -41,13 +41,16 @@ The following server methods must be implemented:
 
 Class | Method | HTTP request | Description
 ------------ | ------------- | ------------- | -------------
-*DefaultApi* | [**query_query_post**](docs/DefaultApi.md#query_query_post) | **POST** /query | Query
+*DefaultApi* | [**delete_docs**](docs/DefaultApi.md#delete_docs) | **DELETE** /delete | Delete
+*DefaultApi* | [**query_post**](docs/DefaultApi.md#query_post) | **POST** /query | Query
 *DefaultApi* | [**upsert_post**](docs/DefaultApi.md#upsert_post) | **POST** /upsert | Data upload
 
 
 
 ## Models
 
+ - [DeleteRequest](docs/DeleteRequest.md)
+ - [DeleteResponse](docs/DeleteResponse.md)
  - [Document](docs/Document.md)
  - [DocumentChunk](docs/DocumentChunk.md)
  - [DocumentChunkMetadata](docs/DocumentChunkMetadata.md)

src/generated/docs/DefaultApi.md

@@ -4,12 +4,42 @@ All URIs are relative to *http://localhost*
 
 Method | HTTP request | Description
 ------------- | ------------- | -------------
-[**query_query_post**](DefaultApi.md#query_query_post) | **POST** /query | Query
+[**delete_docs**](DefaultApi.md#delete_docs) | **DELETE** /delete | Delete
+[**query_post**](DefaultApi.md#query_post) | **POST** /query | Query
 [**upsert_post**](DefaultApi.md#upsert_post) | **POST** /upsert | Data upload
 
 
-# **query_query_post**
-> query_query_post(req::HTTP.Request, query_request::QueryRequest;) -> QueryResponse
+# **delete_docs**
+> delete_docs(req::HTTP.Request, delete_request::DeleteRequest;) -> DeleteResponse
+
+Delete
+
+Delete one or more documents
+
+### Required Parameters
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+ **req** | **HTTP.Request** | The HTTP Request object | 
+**delete_request** | [**DeleteRequest**](DeleteRequest.md)| | 
+
+### Return type
+
+[**DeleteResponse**](DeleteResponse.md)
+
+### Authorization
+
+[HTTPBearer](../README.md#HTTPBearer)
+
+### HTTP request headers
+
+ - **Content-Type**: application/json
+ - **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
+# **query_post**
+> query_post(req::HTTP.Request, query_request::QueryRequest;) -> QueryResponse
 
 Query
 
@@ -38,7 +68,7 @@ Name | Type | Description | Notes
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
 # **upsert_post**
-> upsert_post(req::HTTP.Request; upsert_request=nothing,) -> UpsertResponse
+> upsert_post(req::HTTP.Request, upsert_request::UpsertRequest;) -> UpsertResponse
 
 Data upload
 
@@ -49,12 +79,7 @@ Upload JSON document description
 Name | Type | Description | Notes
 ------------- | ------------- | ------------- | -------------
  **req** | **HTTP.Request** | The HTTP Request object | 
-
-### Optional Parameters
-
-Name | Type | Description | Notes
-------------- | ------------- | ------------- | -------------
- **upsert_request** | [**UpsertRequest**](UpsertRequest.md)| | 
+**upsert_request** | [**UpsertRequest**](UpsertRequest.md)| | 
 
 ### Return type