From 89387342afb0a25a010e46f0411bc378bde7362e Mon Sep 17 00:00:00 2001 From: holoncom Date: Thu, 14 Oct 2021 16:25:50 +0200 Subject: [PATCH 1/3] add Try Catch for better debugging; --- .../Lucene/Mapping/JsonObjectMapper.cs | 270 +++++++++--------- 1 file changed, 138 insertions(+), 132 deletions(-) diff --git a/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs b/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs index 1f394ba5..4011430d 100644 --- a/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs +++ b/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs @@ -69,185 +69,191 @@ public void AddJsonToDocument(JToken json, Document doc, FieldConfig config) /// private static void Add(Document doc, string prefix, JToken token, FieldConfig fieldconfig) { - if (token is JObject) + try { - AddProperties(doc, prefix, token as JObject, fieldconfig); - } - else if (token is JArray) - { - var itemsConfig = fieldconfig?.Items; - if (fieldconfig != null && fieldconfig.Index && itemsConfig == null) + if (token is JObject) { - throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json."); + AddProperties(doc, prefix, token as JObject, fieldconfig); } - AddArray(doc, prefix, token as JArray, itemsConfig); - } - else if (token is JValue) - { - JValue value = token as JValue; - bool index = false; - bool sort = false; - if (fieldconfig != null) + else if (token is JArray) { - index = fieldconfig.Index; - sort = fieldconfig.Sort; - if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String) + var itemsConfig = fieldconfig?.Items; + if (fieldconfig != null && fieldconfig.Index && itemsConfig == null) { - DateTime d; - if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d)) - { - value = new JValue(d); - } + throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json."); } + AddArray(doc, prefix, token as JArray, itemsConfig); } - - switch (value.Type) //todo: simple date gets detected as string + else if (token is JValue) { - case JTokenType.Boolean: - if (index || sort) + JValue value = token as JValue; + bool index = false; + bool sort = false; + if (fieldconfig != null) + { + index = fieldconfig.Index; + sort = fieldconfig.Sort; + if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String) { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0)); + DateTime d; + if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d)) + { + value = new JValue(d); + } } - break; + } - case JTokenType.Date: - if (index || sort) - { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks)); + switch (value.Type) //todo: simple date gets detected as string + { + case JTokenType.Boolean: + if (index || sort) + { + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0)); + } + break; + + case JTokenType.Date: + if (index || sort) + { + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks)); - //doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); + //doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); - /* - if (field != null ){ - if (field.IndexType == "datetime") + /* + if (field != null ){ + if (field.IndexType == "datetime") + { + doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + else if (field.IndexType == "date") + { + doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + else if (field.IndexType == "time") + { + doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + } + else { doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); } - else if (field.IndexType == "date") + */ + } + break; + + case JTokenType.Float: + if (index || sort) + { + if (value.Value is float) { - doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value)); } - else if (field.IndexType == "time") + else { - doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value))); } } - else - { - doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); - } - */ - } - break; + break; - case JTokenType.Float: - if (index || sort) - { - if (value.Value is float) + case JTokenType.Guid: + if (index || sort) { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value)); + doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED)); } - else + break; + + case JTokenType.Integer: + if (index || sort) { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value))); + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value))); } - } - break; - - case JTokenType.Guid: - if (index || sort) - { - doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED)); - } - break; - - case JTokenType.Integer: - if (index || sort) - { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value))); - } - break; + break; - case JTokenType.Null: - break; + case JTokenType.Null: + break; - case JTokenType.String: - - if (fieldconfig != null && fieldconfig.IndexType == "key") - { - doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED)); - } - else if (fieldconfig != null && fieldconfig.IndexType == "html") - { - if (index) + case JTokenType.String: + if (fieldconfig != null && fieldconfig.IndexType == "key") { - doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED)); + doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED)); } - if (sort) + else if (fieldconfig != null && fieldconfig.IndexType == "html") { - doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED)); + if (index) + { + doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED)); + } + if (sort) + { + doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } } - } - else if (fieldconfig != null && fieldconfig.IndexType == "file") - { - var val = value.Value.ToString(); - if (!string.IsNullOrEmpty(val)) + else if (fieldconfig != null && fieldconfig.IndexType == "file") { - var fileIndexer = FileIndexerManager.GetFileIndexer(val); - if (fileIndexer != null) + var val = value.Value.ToString(); + if (!string.IsNullOrEmpty(val)) { - var content = fileIndexer.GetContent(val); - if (index) - { - doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED)); - } - if (sort) + var fileIndexer = FileIndexerManager.GetFileIndexer(val); + if (fileIndexer != null) { - doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); + var content = fileIndexer.GetContent(val); + if (index) + { + doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED)); + } + if (sort) + { + doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } } } } - } - else - { - - var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString()); - val = val.Replace('-', ' '); // concider '-' as a space - val = val.Replace(',', ' '); // concider ',' as a space - //var val = LuceneUtils.CleanupText(value.Value.ToString()); - if (index) + else { - doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED)); + var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString()); + val = val.Replace('-', ' '); // concider '-' as a space + val = val.Replace(',', ' '); // concider ',' as a space + //var val = LuceneUtils.CleanupText(value.Value.ToString()); + if (index) + { + doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED)); + } + if (sort) + { + doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); + } } - if (sort) + break; + + case JTokenType.TimeSpan: + if (index || sort) { - doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks)); } - } - break; - - case JTokenType.TimeSpan: - if (index || sort) - { - doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks)); - } - break; + break; - case JTokenType.Uri: - if (index || sort) - { - doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED)); - } - break; + case JTokenType.Uri: + if (index || sort) + { + doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED)); + } + break; - default: - Debug.Fail("Unsupported JValue type: " + value.Type); - break; + default: + Debug.Fail("Unsupported JValue type: " + value.Type); + break; + } + } + else + { + Debug.Fail("Unsupported JToken: " + token); } } - else + catch (Exception e) { - Debug.Fail("Unsupported JToken: " + token); + if(Debugger.IsAttached) Debugger.Break(); + throw; } } From 7fe9dc15648ccaa541942917eb9ac11c38c91238 Mon Sep 17 00:00:00 2001 From: holoncom Date: Thu, 14 Oct 2021 16:26:35 +0200 Subject: [PATCH 2/3] fix issue where error was thrown when trying to index an empty string --- OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs b/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs index 4011430d..5aadb416 100644 --- a/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs +++ b/OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs @@ -174,6 +174,9 @@ private static void Add(Document doc, string prefix, JToken token, FieldConfig f break; case JTokenType.String: + if (value.Value == null) + break; + if (fieldconfig != null && fieldconfig.IndexType == "key") { doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED)); From c5080472d027fa37531860104d264ef89dcab165 Mon Sep 17 00:00:00 2001 From: holoncom Date: Tue, 26 Oct 2021 14:23:42 +0200 Subject: [PATCH 3/3] throw more helpful error when json Parse fails --- OpenContent/Components/OpenContentInfo.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/OpenContent/Components/OpenContentInfo.cs b/OpenContent/Components/OpenContentInfo.cs index 24f1c8cb..e5b6feba 100644 --- a/OpenContent/Components/OpenContentInfo.cs +++ b/OpenContent/Components/OpenContentInfo.cs @@ -78,7 +78,14 @@ public JToken JsonAsJToken { if (_jsonAsJToken == null && !string.IsNullOrEmpty(this.Json)) { - _jsonAsJToken = JToken.Parse(this.Json); + try + { + _jsonAsJToken = JToken.Parse(this.Json); + } + catch (Exception e) + { + throw new Exception($"Failed to parse json from moduleId:{ModuleId}, contentId:{ContentId}", e); + } } // JsonAsJToken is modified (to remove other cultures) return _jsonAsJToken?.DeepClone();