Skip to content

Commit

Permalink
WIP - Upgrade to the new Elasticsearch client
Browse files Browse the repository at this point in the history
  • Loading branch information
niemyjski committed Sep 4, 2024
1 parent a4b976f commit f347b85
Show file tree
Hide file tree
Showing 23 changed files with 283 additions and 282 deletions.
158 changes: 93 additions & 65 deletions src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs

Large diffs are not rendered by default.

18 changes: 10 additions & 8 deletions src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Aggregations;
using Elastic.Clients.Elasticsearch.QueryDsl;
using Foundatio.Parsers.ElasticQueries.Extensions;
using Foundatio.Parsers.ElasticQueries.Visitors;
using Foundatio.Parsers.LuceneQueries;
using Foundatio.Parsers.LuceneQueries.Extensions;
using Foundatio.Parsers.LuceneQueries.Nodes;
using Foundatio.Parsers.LuceneQueries.Visitors;
using Nest;
using Pegasus.Common;

namespace Foundatio.Parsers.ElasticQueries;
Expand Down Expand Up @@ -158,7 +160,7 @@ public async Task<QueryValidationResult> ValidateQueryAsync(string query, QueryV
return context.GetValidationResult();
}

public async Task<QueryContainer> BuildQueryAsync(string query, IElasticQueryVisitorContext context = null)
public async Task<Query> BuildQueryAsync(string query, IElasticQueryVisitorContext context = null)
{
context ??= new ElasticQueryVisitorContext();
context.QueryType = QueryTypes.Query;
Expand All @@ -169,7 +171,7 @@ public async Task<QueryContainer> BuildQueryAsync(string query, IElasticQueryVis
return await BuildQueryAsync(result, context).ConfigureAwait(false);
}

public async Task<QueryContainer> BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null)
public async Task<Query> BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null)
{
context ??= new ElasticQueryVisitorContext();
var q = await query.GetQueryAsync() ?? new MatchAllQuery();
Expand All @@ -195,7 +197,7 @@ public async Task<QueryValidationResult> ValidateAggregationsAsync(string query,
return context.GetValidationResult();
}

public async Task<AggregationContainer> BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null)
public async Task<Aggregation> BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null)
{
context ??= new ElasticQueryVisitorContext();
context.QueryType = QueryTypes.Aggregation;
Expand All @@ -207,7 +209,7 @@ public async Task<AggregationContainer> BuildAggregationsAsync(string aggregatio
}

#pragma warning disable IDE0060 // Remove unused parameter
public async Task<AggregationContainer> BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null)
public async Task<Aggregation> BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null)
{
if (aggregations == null)
return null;
Expand All @@ -227,7 +229,7 @@ public async Task<QueryValidationResult> ValidateSortAsync(string query, QueryVa
return context.GetValidationResult();
}

public async Task<IEnumerable<IFieldSort>> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null)
public async Task<IEnumerable<SortOptions>> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null)
{
context ??= new ElasticQueryVisitorContext();
context.QueryType = QueryTypes.Sort;
Expand All @@ -238,7 +240,7 @@ public async Task<IEnumerable<IFieldSort>> BuildSortAsync(string sort, IElasticQ
return await BuildSortAsync(result, context).ConfigureAwait(false);
}

public Task<IEnumerable<IFieldSort>> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null)
public Task<IEnumerable<SortOptions>> BuildSortAsync(IQueryNode sort, IElasticQueryVisitorContext context = null)
{
context ??= new ElasticQueryVisitorContext();
return GetSortFieldsVisitor.RunAsync(sort, context);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Mapping;
using Foundatio.Parsers.ElasticQueries.Visitors;
using Foundatio.Parsers.LuceneQueries;
using Foundatio.Parsers.LuceneQueries.Visitors;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Nest;

namespace Foundatio.Parsers.ElasticQueries;

Expand Down Expand Up @@ -285,35 +286,35 @@ public ElasticQueryParserConfiguration AddAggregationVisitorAfter<T>(IChainableQ

#endregion

public ElasticQueryParserConfiguration UseMappings<T>(Func<TypeMappingDescriptor<T>, TypeMappingDescriptor<T>> mappingBuilder, IElasticClient client, string index) where T : class
public ElasticQueryParserConfiguration UseMappings<T>(Func<TypeMappingDescriptor<T>, TypeMappingDescriptor<T>> mappingBuilder, ElasticsearchClient client, string index) where T : class
{
MappingResolver = ElasticMappingResolver.Create<T>(mappingBuilder, client, index, logger: _logger);

Check failure on line 291 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 1: cannot convert from 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>>' to 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMapping>'

Check failure on line 291 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 1: cannot convert from 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>>' to 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMapping>'

return this;
}

public ElasticQueryParserConfiguration UseMappings<T>(Func<TypeMappingDescriptor<T>, TypeMappingDescriptor<T>> mappingBuilder, Inferrer inferrer, Func<ITypeMapping> getMapping) where T : class
public ElasticQueryParserConfiguration UseMappings<T>(Func<TypeMappingDescriptor<T>, TypeMappingDescriptor<T>> mappingBuilder, Inferrer inferrer, Func<TypeMapping> getMapping) where T : class
{
MappingResolver = ElasticMappingResolver.Create<T>(mappingBuilder, inferrer, getMapping, logger: _logger);

Check failure on line 298 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 1: cannot convert from 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>>' to 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMapping>'

Check failure on line 298 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 2: cannot convert from 'Elastic.Clients.Elasticsearch.Inferrer' to 'Elastic.Clients.Elasticsearch.ElasticsearchClient'

Check failure on line 298 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 3: cannot convert from 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMapping>' to 'string'

Check failure on line 298 in src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs

View workflow job for this annotation

GitHub Actions / build / build

Argument 1: cannot convert from 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>>' to 'System.Func<Elastic.Clients.Elasticsearch.Mapping.TypeMappingDescriptor<T>, Elastic.Clients.Elasticsearch.Mapping.TypeMapping>'

return this;
}

public ElasticQueryParserConfiguration UseMappings<T>(IElasticClient client)
public ElasticQueryParserConfiguration UseMappings<T>(ElasticsearchClient client)
{
MappingResolver = ElasticMappingResolver.Create<T>(client, logger: _logger);

return this;
}

public ElasticQueryParserConfiguration UseMappings(IElasticClient client, string index)
public ElasticQueryParserConfiguration UseMappings(ElasticsearchClient client, string index)
{
MappingResolver = ElasticMappingResolver.Create(client, index, logger: _logger);

return this;
}

public ElasticQueryParserConfiguration UseMappings(Func<ITypeMapping> getMapping, Inferrer inferrer = null)
public ElasticQueryParserConfiguration UseMappings(Func<TypeMapping> getMapping, Inferrer inferrer = null)
{
MappingResolver = ElasticMappingResolver.Create(getMapping, inferrer, logger: _logger);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
using System;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Aggregations;
using Exceptionless.DateTimeExtensions;
using Foundatio.Parsers.ElasticQueries.Visitors;
using Foundatio.Parsers.LuceneQueries.Extensions;
using Foundatio.Parsers.LuceneQueries.Nodes;
using Foundatio.Parsers.LuceneQueries.Visitors;
using Nest;

namespace Foundatio.Parsers.ElasticQueries.Extensions;

Expand All @@ -15,7 +16,7 @@ public static class DefaultAggregationNodeExtensions
// NOTE: We may want to read this dynamically from server settings.
public const int MAX_BUCKET_SIZE = 10000;

public static Task<AggregationBase> GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context)
public static Task<Aggregation> GetDefaultAggregationAsync(this IQueryNode node, IQueryVisitorContext context)
{
if (node is GroupNode groupNode)
return groupNode.GetDefaultAggregationAsync(context);
Expand All @@ -26,7 +27,7 @@ public static Task<AggregationBase> GetDefaultAggregationAsync(this IQueryNode n
return null;
}

public static async Task<AggregationBase> GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context)
public static async Task<Aggregation> GetDefaultAggregationAsync(this GroupNode node, IQueryVisitorContext context)
{
if (context is not IElasticQueryVisitorContext elasticContext)
throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context));
Expand All @@ -47,11 +48,11 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this GroupN
return GetHistogramAggregation("histogram_" + originalField, field, node.UnescapedProximity, node.UnescapedBoost, context);

case AggregationType.GeoHashGrid:
var precision = GeoHashPrecision.Precision1;
var precision = new GeohashPrecision(1);
if (!String.IsNullOrEmpty(node.UnescapedProximity))
Enum.TryParse(node.UnescapedProximity, out precision);

return new GeoHashGridAggregation("geogrid_" + originalField)
return new Aggregation("geogrid_" + originalField, new GeohashGridAggregation
{
Field = field,
Precision = precision,
Expand All @@ -62,14 +63,14 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this GroupN
{
Script = new InlineScript($"doc['{node.Field}'].lon")
}
};
});

case AggregationType.Terms:
var agg = new TermsAggregation("terms_" + originalField)
{
Field = field,
Size = node.GetProximityAsInt32(),
MinimumDocumentCount = node.GetBoostAsInt32(),
MinDocCount = node.GetBoostAsInt32(),
Meta = new Dictionary<string, object> { { "@field_type", property?.Type } }
};

Expand All @@ -85,7 +86,7 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this GroupN
return null;
}

public static async Task<AggregationBase> GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context)
public static async Task<Aggregation> GetDefaultAggregationAsync(this TermNode node, IQueryVisitorContext context)
{
if (context is not IElasticQueryVisitorContext elasticContext)
throw new ArgumentException("Context must be of type IElasticQueryVisitorContext", nameof(context));
Expand Down Expand Up @@ -134,11 +135,11 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this TermNo
return GetPercentilesAggregation("percentiles_" + originalField, aggField, node.UnescapedProximity, node.UnescapedBoost, context);

case AggregationType.GeoHashGrid:
var precision = GeoHashPrecision.Precision1;
var precision = new GeohashPrecision(1);
if (!String.IsNullOrEmpty(node.UnescapedProximity))
Enum.TryParse(node.UnescapedProximity, out precision);

return new GeoHashGridAggregation("geogrid_" + originalField)
return new GeohashGridAggregation("geogrid_" + originalField)
{
Field = aggField,
Precision = precision,
Expand All @@ -156,7 +157,7 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this TermNo
{
Field = aggField,
Size = node.GetProximityAsInt32(),
MinimumDocumentCount = node.GetBoostAsInt32(),
MinDocCount = node.GetBoostAsInt32(),
Meta = new Dictionary<string, object> { { "@field_type", property?.Type } }
};

Expand All @@ -169,7 +170,7 @@ public static async Task<AggregationBase> GetDefaultAggregationAsync(this TermNo
return null;
}

private static AggregationBase GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
private static Aggregation GetPercentilesAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
{
List<double> percents = null;
if (!String.IsNullOrWhiteSpace(proximity))
Expand All @@ -189,7 +190,7 @@ private static AggregationBase GetPercentilesAggregation(string originalField, s
};
}

private static AggregationBase GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
private static Aggregation GetHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
{
double interval = 50;
if (Double.TryParse(proximity, out double prox))
Expand All @@ -198,25 +199,25 @@ private static AggregationBase GetHistogramAggregation(string originalField, str
return new HistogramAggregation(originalField)
{
Field = field,
MinimumDocumentCount = 0,
MinDocCount = 0,
Interval = interval
};
}

private static AggregationBase GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
private static Aggregation GetDateHistogramAggregation(string originalField, string field, string proximity, string boost, IQueryVisitorContext context)
{
// NOTE: StartDate and EndDate are set in the Repositories QueryBuilderContext.
var start = context.GetDate("StartDate");
var end = context.GetDate("EndDate");
bool isValidRange = start.HasValue && start.Value > DateTime.MinValue && end.HasValue && end.Value < DateTime.MaxValue && start.Value <= end.Value;
var bounds = isValidRange ? new ExtendedBounds<DateMath> { Minimum = start.Value, Maximum = end.Value } : null;
var bounds = isValidRange ? new ExtendedBoundsDate { Min = start.Value, Max = end.Value } : null;

var interval = GetInterval(proximity, start, end);
string timezone = TryConvertTimeUnitToUtcOffset(boost);
var agg = new DateHistogramAggregation(originalField)
{
Field = field,
MinimumDocumentCount = 0,
MinDocCount = 0,
Format = "date_optional_time",
TimeZone = timezone,
Meta = !String.IsNullOrEmpty(boost) ? new Dictionary<string, object> { { "@timezone", boost } } : null,
Expand Down Expand Up @@ -247,55 +248,55 @@ private static string TryConvertTimeUnitToUtcOffset(string boost)
return "+" + timezoneOffset.Value.ToString("hh\\:mm");
}

private static Union<DateInterval, Time> GetInterval(string proximity, DateTime? start, DateTime? end)
private static Union<CalendarInterval, Duration> GetInterval(string proximity, DateTime? start, DateTime? end)
{
if (String.IsNullOrEmpty(proximity))
return GetInterval(start, end);

return proximity.Trim() switch
{
"s" or "1s" or "second" => DateInterval.Second,
"m" or "1m" or "minute" => DateInterval.Minute,
"h" or "1h" or "hour" => DateInterval.Hour,
"d" or "1d" or "day" => DateInterval.Day,
"w" or "1w" or "week" => DateInterval.Week,
"M" or "1M" or "month" => DateInterval.Month,
"q" or "1q" or "quarter" => DateInterval.Quarter,
"y" or "1y" or "year" => DateInterval.Year,
_ => new Union<DateInterval, Time>(proximity),
"s" or "1s" or "second" => CalendarInterval.Second,
"m" or "1m" or "minute" => CalendarInterval.Minute,
"h" or "1h" or "hour" => CalendarInterval.Hour,
"d" or "1d" or "day" => CalendarInterval.Day,
"w" or "1w" or "week" => CalendarInterval.Week,
"M" or "1M" or "month" => CalendarInterval.Month,
"q" or "1q" or "quarter" => CalendarInterval.Quarter,
"y" or "1y" or "year" => CalendarInterval.Year,
_ => new Union<CalendarInterval, Duration>(proximity),
};
}

private static Union<DateInterval, Time> GetInterval(DateTime? utcStart, DateTime? utcEnd, int desiredDataPoints = 100)
private static Union<CalendarInterval, Duration> GetInterval(DateTime? utcStart, DateTime? utcEnd, int desiredDataPoints = 100)
{
if (!utcStart.HasValue || !utcEnd.HasValue || utcStart.Value == DateTime.MinValue)
return DateInterval.Day;
return CalendarInterval.Day;

var totalTime = utcEnd.Value - utcStart.Value;
var timePerBlock = TimeSpan.FromMinutes(totalTime.TotalMinutes / desiredDataPoints);
if (timePerBlock.TotalDays > 1)
{
timePerBlock = timePerBlock.Round(TimeSpan.FromDays(1));
return (Time)timePerBlock;
return (Duration)timePerBlock;
}

if (timePerBlock.TotalHours > 1)
{
timePerBlock = timePerBlock.Round(TimeSpan.FromHours(1));
return (Time)timePerBlock;
return (Duration)timePerBlock;
}

if (timePerBlock.TotalMinutes > 1)
{
timePerBlock = timePerBlock.Round(TimeSpan.FromMinutes(1));
return (Time)timePerBlock;
return (Duration)timePerBlock;
}

timePerBlock = timePerBlock.Round(TimeSpan.FromSeconds(15));
if (timePerBlock.TotalSeconds < 1)
timePerBlock = TimeSpan.FromSeconds(15);

return (Time)timePerBlock;
return (Duration)timePerBlock;
}

public static int? GetProximityAsInt32(this IFieldQueryWithProximityAndBoostNode node)
Expand Down
Loading

0 comments on commit f347b85

Please sign in to comment.