Add SQL APIs (#6165) (#6166)

* Add SQL APIs

Does not include SQL Translate API at this stage.

Co-authored-by: Steve Gordon <sgordon@hotmail.co.uk>

build/scripts/Targets.fs

@@ -49,7 +49,7 @@ module Main =
         Tests.SetTestEnvironmentVariables parsed
 
         let testChain = ["clean"; "version"; "restore"; "full-build"; ]
-        let buildChain = ["test"; "inherit-doc"; "documentation"; ]
+        let buildChain = ["test"; "inherit-doc"; ]
         let releaseChain =
             [ 
                 "build";

src/Elastic.Clients.Elasticsearch/Api/SqlGetAsyncResponse.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Elastic.Clients.Elasticsearch.Sql;
+
+public partial class SqlGetAsyncResponse
+{
+	[JsonInclude]
+	[JsonPropertyName("rows")]
+	public IReadOnlyCollection<SqlRow> Rows { get; init; }
+}

src/Elastic.Clients.Elasticsearch/Common/LazyDocument.cs

@@ -5,81 +5,37 @@
 using System;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Threading;
-using System.Threading.Tasks;
-using Elastic.Transport;
 
 namespace Elastic.Clients.Elasticsearch
 {
 	/// <summary>
-	/// A lazily deserialized document.
+	/// <para>A lazily deserialized document.</para>
+	/// <para>Holds raw JSON bytes which can be lazily converted to a specific <see cref="Type"/> at a later time.</para>
 	/// </summary>
 	[JsonConverter(typeof(LazyDocumentConverter))]
-	public class LazyDocument
+	public readonly struct LazyDocument
 	{
-		private readonly Serializer _sourceSerializer;
-		private readonly Serializer _requestResponseSerializer;
-		private readonly IMemoryStreamFactory _memoryStreamFactory;
-
 		internal LazyDocument(byte[] bytes, IElasticsearchClientSettings settings)
 		{
 			Bytes = bytes;
-
-			_sourceSerializer = settings.SourceSerializer;
-			_requestResponseSerializer = settings.RequestResponseSerializer;
-			_memoryStreamFactory = settings.MemoryStreamFactory;
+			Settings = settings;
 		}
 
-		internal byte[] Bytes { get; }
-
-		internal T AsUsingRequestResponseSerializer<T>()
-		{
-			using var ms = _memoryStreamFactory.Create(Bytes);
-			return _requestResponseSerializer.Deserialize<T>(ms);
-		}
+		internal byte[]? Bytes { get; }
+		internal IElasticsearchClientSettings? Settings { get; }
 
 		/// <summary>
 		/// Creates an instance of <typeparamref name="T" /> from this
 		/// <see cref="LazyDocument" /> instance.
 		/// </summary>
 		/// <typeparam name="T">The type</typeparam>
-		public T As<T>()
-		{
-			using var ms = _memoryStreamFactory.Create(Bytes);
-			return _sourceSerializer.Deserialize<T>(ms);
-		}
-
-		/// <summary>
-		/// Creates an instance of <paramref name="objectType" /> from this
-		/// <see cref="LazyDocument" /> instance.
-		/// </summary>
-		/// <param name="objectType">The type</param>
-		public object As(Type objectType)
+		public T? As<T>()
 		{
-			using var ms = _memoryStreamFactory.Create(Bytes);
-			return _sourceSerializer.Deserialize(objectType, ms);
-		}
+			if (Bytes is null || Settings is null || Bytes.Length == 0)
+				return default;
 
-		/// <summary>
-		/// Creates an instance of <typeparamref name="T" /> from this
-		/// <see cref="LazyDocument" /> instance.
-		/// </summary>
-		/// <typeparam name="T">The type</typeparam>
-		public ValueTask<T> AsAsync<T>(CancellationToken ct = default)
-		{
-			using var ms = _memoryStreamFactory.Create(Bytes);
-			return _sourceSerializer.DeserializeAsync<T>(ms, ct);
-		}
-
-		/// <summary>
-		/// Creates an instance of <paramref name="objectType" /> from this
-		/// <see cref="LazyDocument" /> instance.
-		/// </summary>
-		/// <param name="objectType">The type</param>
-		public ValueTask<object> AsAsync(Type objectType, CancellationToken ct = default)
-		{
-			using var ms = _memoryStreamFactory.Create(Bytes);
-			return _sourceSerializer.DeserializeAsync(objectType, ms, ct);
+			using var ms = Settings.MemoryStreamFactory.Create(Bytes);
+			return Settings.SourceSerializer.Deserialize<T>(ms);
 		}
 	}
 
@@ -101,6 +57,6 @@ public override LazyDocument Read(ref Utf8JsonReader reader, Type typeToConvert,
 			return new LazyDocument(stream.ToArray(), _settings);
 		}
 
-		public override void Write(Utf8JsonWriter writer, LazyDocument value, JsonSerializerOptions options) => throw new NotImplementedException();
+		public override void Write(Utf8JsonWriter writer, LazyDocument value, JsonSerializerOptions options) => throw new NotImplementedException("We only ever expect to deserialize a LazyDocument on responses.");
 	}
 }

src/Elastic.Clients.Elasticsearch/FutureGenerated/FutureGenerated.cs

@@ -17,6 +17,84 @@
 using System.Linq.Expressions;
 using System.IO;
 
+namespace Elastic.Clients.Elasticsearch.Sql
+{
+	public partial class SqlTranslateResponse
+	{
+		[JsonInclude]
+		[JsonPropertyName("query")]
+		public QueryContainer Query { get; set; }
+	}
+
+	public static class SqlTranslateResponseExtensions
+	{
+		public static SearchRequest AsSearchRequest(this SqlTranslateResponse response)
+			=> new()
+			{
+				Query = response.Query,
+				//Size = response.Size,
+				//Fields = response.Fields
+			};
+	}
+}
+
+namespace Elastic.Clients.Elasticsearch
+{
+	[JsonConverter(typeof(SourceConfigConverter))]
+	public partial class SourceConfig
+	{
+		public bool HasBoolValue => Item1.HasValue;
+
+		public bool HasSourceFilterValue => Item2 is not null;
+
+		public bool TryGetBool(out bool? value)
+		{
+			if (Item1.HasValue)
+			{
+				value = Item1.Value;
+				return true;
+			}
+
+			value = null;
+			return false;
+		}
+
+		public bool TryGetSourceFilter(out SourceFilter? value)
+		{
+			if (Item2 is not null)
+			{
+				value = Item2;
+				return true;
+			}
+
+			value = null;
+			return false;
+		}
+	}
+
+	internal class SourceConfigConverter : JsonConverter<SourceConfig>
+	{
+		public override SourceConfig? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+		{
+			switch (reader.TokenType)
+			{
+				case JsonTokenType.True:
+				case JsonTokenType.False:
+					var value = reader.GetBoolean();
+					return new SourceConfig(value);
+
+				case JsonTokenType.StartObject:
+					var sourceFilter = JsonSerializer.Deserialize<SourceFilter>(ref reader, options);
+					return new SourceConfig(sourceFilter);
+			}
+
+			return null;
+		}
+
+		public override void Write(Utf8JsonWriter writer, SourceConfig value, JsonSerializerOptions options) => throw new NotImplementedException();
+	}
+}
+
 namespace Elastic.Clients.Elasticsearch.Aggregations
 {
 	//public partial class TopMetricsValue
@@ -1129,7 +1207,8 @@ public void Deserialize(ref Utf8JsonReader reader, JsonSerializerOptions options
 	public enum FieldType
 	{
 		Date,
-		Text
+		Text,
+		Long
 	}
 
 	public partial class CountRequest<TDocument> : CountRequest
@@ -1472,6 +1551,10 @@ public override FieldType Read(ref Utf8JsonReader reader, Type typeToConvert, Js
 			{
 				case "date":
 					return FieldType.Date;
+				case "long":
+					return FieldType.Long;
+				case "text":
+					return FieldType.Text;
 			}
 
 			ThrowHelper.ThrowJsonException("Unexpected field type value.");
@@ -1485,6 +1568,12 @@ public override void Write(Utf8JsonWriter writer, FieldType value, JsonSerialize
 				case FieldType.Date:
 					writer.WriteStringValue("date");
 					return;
+				case FieldType.Long:
+					writer.WriteStringValue("long");
+					return;
+				case FieldType.Text:
+					writer.WriteStringValue("text");
+					return;
 			}
 
 			writer.WriteNullValue();
@@ -1965,6 +2054,12 @@ public partial class MatchAllQuery
 		public static implicit operator QueryContainer(MatchAllQuery matchAllQuery) => new(matchAllQuery);
 	}
 
+	public partial class QueryContainer
+	{
+		// TODO - Generate more of these!
+		public TermQuery Term => Variant as TermQuery;
+	}
+
 	//public sealed partial class BoolQueryDescriptor
 	//{
 	//	internal BoolQuery ToQuery()

src/Elastic.Clients.Elasticsearch/Serialization/DictionaryConverter.cs

@@ -89,6 +89,10 @@ public override Dictionary<TKey, TValue> Read(
 					{
 						key = IndexName.Parse(propertyName) as TKey;
 					}
+					else if (typeof(TKey) == typeof(Field))
+					{
+						key = new Field(propertyName) as TKey;
+					}
 					else
 					{
 						key = (TKey)Activator.CreateInstance(typeof(TKey),

src/Elastic.Clients.Elasticsearch/Serialization/UnionConverter.cs

@@ -13,7 +13,13 @@ namespace Elastic.Clients.Elasticsearch;
 
 internal sealed class UnionConverter : JsonConverterFactory
 {
-	public override bool CanConvert(Type typeToConvert) => typeToConvert.Name == typeof(Union<,>).Name || (typeToConvert.BaseType is not null && typeToConvert.BaseType.Name == typeof(Union<,>).Name);
+	private static readonly HashSet<Type> TypesToSkip = new HashSet<Type>
+	{
+		typeof(SourceConfig)
+	};
+
+	public override bool CanConvert(Type typeToConvert) => !TypesToSkip.Contains(typeToConvert) &&
+		(typeToConvert.Name == typeof(Union<,>).Name || (typeToConvert.BaseType is not null && typeToConvert.BaseType.Name == typeof(Union<,>).Name));
 
 	public override JsonConverter CreateConverter(
 		Type type,

src/Elastic.Clients.Elasticsearch/Types/Sql/SqlRow.cs

@@ -0,0 +1,46 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Elastic.Clients.Elasticsearch.Sql;
+
+[JsonConverter(typeof(SqlRowConverter))]
+public sealed class SqlRow : ReadOnlyCollection<SqlValue>
+{
+	public SqlRow(IList<SqlValue> list) : base(list) { }
+}
+
+internal sealed class SqlRowConverter : JsonConverter<SqlRow>
+{
+	public override SqlRow? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+	{
+		if (reader.TokenType == JsonTokenType.Null)
+		{
+			reader.Read();
+			return null;
+		}
+
+		if (reader.TokenType == JsonTokenType.StartArray)
+		{
+			var values = new List<SqlValue>();
+
+			while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
+			{
+				var value = JsonSerializer.Deserialize<SqlValue>(ref reader, options);
+				values.Add(value);
+			}
+
+			return new SqlRow(values);
+		}
+
+		throw new JsonException($"Unexpected JSON token when deserializing {nameof(SqlRow)}.");
+	}
+
+	public override void Write(Utf8JsonWriter writer, SqlRow value, JsonSerializerOptions options) => throw new NotImplementedException();
+}

src/Elastic.Clients.Elasticsearch/Types/Sql/SqlValue.cs

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Elastic.Clients.Elasticsearch.Sql;
+
+[JsonConverter(typeof(SqlValueConverter))]
+public readonly struct SqlValue
+{
+	private readonly LazyDocument _lazyDocument;
+
+	internal SqlValue(LazyDocument lazyDocument) => _lazyDocument = lazyDocument;
+
+	public T? As<T>() => _lazyDocument.As<T>();
+}
+
+internal sealed class SqlValueConverter : JsonConverter<SqlValue>
+{
+	public override SqlValue Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+	{
+		if (reader.TokenType == JsonTokenType.Null)
+		{
+			reader.Read();
+			return default;
+		}
+
+		var lazyDoc = JsonSerializer.Deserialize<LazyDocument>(ref reader, options);
+		return new SqlValue(lazyDoc);
+	}
+
+	public override void Write(Utf8JsonWriter writer, SqlValue value, JsonSerializerOptions options) => throw new NotImplementedException();
+}

src/Elastic.Clients.Elasticsearch/_Generated/Api/ApiUrlsLookup.g.cs

@@ -46,6 +46,11 @@ internal static class ApiUrlsLookups
 		internal static ApiUrls NoNamespacePing = new ApiUrls(new[] { "/" });
 		internal static ApiUrls NoNamespaceSearch = new ApiUrls(new[] { "/_search", "/{index}/_search" });
 		internal static ApiUrls NoNamespaceGetSource = new ApiUrls(new[] { "/{index}/_source/{id}" });
+		internal static ApiUrls SqlClearCursor = new ApiUrls(new[] { "/_sql/close" });
+		internal static ApiUrls SqlDeleteAsync = new ApiUrls(new[] { "/_sql/async/delete/{id}" });
+		internal static ApiUrls SqlGetAsync = new ApiUrls(new[] { "/_sql/async/{id}" });
+		internal static ApiUrls SqlGetAsyncStatus = new ApiUrls(new[] { "/_sql/async/status/{id}" });
+		internal static ApiUrls SqlQuery = new ApiUrls(new[] { "/_sql" });
 		internal static ApiUrls NoNamespaceUpdate = new ApiUrls(new[] { "/{index}/_update/{id}" });
 	}
 }

src/Elastic.Clients.Elasticsearch/_Generated/Api/SourceRequest.g.cs

@@ -199,4 +199,4 @@ protected override void Serialize(Utf8JsonWriter writer, JsonSerializerOptions o
 		{
 		}
 	}
-}
+}