Skip to content

Commit

Permalink
Merge branch 'develop' of https://github.com/sachatrauwaen/OpenContent …
Browse files Browse the repository at this point in the history 
…into develop
  • Loading branch information
@sachatrauwaen
sachatrauwaen committed Nov 16, 2021
2 parents b796c0a + c508047 commit 97c2233
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 132 deletions.
271 changes: 140 additions & 131 deletions OpenContent/Components/Lucene/Mapping/JsonObjectMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,185 +69,194 @@ public void AddJsonToDocument(JToken json, Document doc, FieldConfig config)
/// <param name="fieldconfig"></param>
private static void Add(Document doc, string prefix, JToken token, FieldConfig fieldconfig)
{
if (token is JObject)
try
{
AddProperties(doc, prefix, token as JObject, fieldconfig);
}
else if (token is JArray)
{
var itemsConfig = fieldconfig?.Items;
if (fieldconfig != null && fieldconfig.Index && itemsConfig == null)
if (token is JObject)
{
throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json.");
AddProperties(doc, prefix, token as JObject, fieldconfig);
}
AddArray(doc, prefix, token as JArray, itemsConfig);
}
else if (token is JValue)
{
JValue value = token as JValue;
bool index = false;
bool sort = false;
if (fieldconfig != null)
else if (token is JArray)
{
index = fieldconfig.Index;
sort = fieldconfig.Sort;
if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String)
var itemsConfig = fieldconfig?.Items;
if (fieldconfig != null && fieldconfig.Index && itemsConfig == null)
{
DateTime d;
if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d))
{
value = new JValue(d);
}
throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json.");
}
AddArray(doc, prefix, token as JArray, itemsConfig);
}

switch (value.Type) //todo: simple date gets detected as string
else if (token is JValue)
{
case JTokenType.Boolean:
if (index || sort)
JValue value = token as JValue;
bool index = false;
bool sort = false;
if (fieldconfig != null)
{
index = fieldconfig.Index;
sort = fieldconfig.Sort;
if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0));
DateTime d;
if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d))
{
value = new JValue(d);
}
}
break;
}

case JTokenType.Date:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks));
switch (value.Type) //todo: simple date gets detected as string
{
case JTokenType.Boolean:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0));
}
break;

case JTokenType.Date:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks));

//doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
//doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));

/*
if (field != null ){
if (field.IndexType == "datetime")
/*
if (field != null ){
if (field.IndexType == "datetime")
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else if (field.IndexType == "date")
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else if (field.IndexType == "time")
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
else
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else if (field.IndexType == "date")
*/
}
break;

case JTokenType.Float:
if (index || sort)
{
if (value.Value is float)
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value));
}
else if (field.IndexType == "time")
else
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value)));
}
}
else
{
doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
*/
}
break;
break;

case JTokenType.Float:
if (index || sort)
{
if (value.Value is float)
case JTokenType.Guid:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value));
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else
break;

case JTokenType.Integer:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value)));
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value)));
}
}
break;
break;

case JTokenType.Guid:
if (index || sort)
{
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
break;

case JTokenType.Integer:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value)));
}
break;

case JTokenType.Null:
break;
case JTokenType.Null:
break;

case JTokenType.String:
case JTokenType.String:
if (value.Value == null)
break;

if (fieldconfig != null && fieldconfig.IndexType == "key")
{
doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else if (fieldconfig != null && fieldconfig.IndexType == "html")
{
if (index)
if (fieldconfig != null && fieldconfig.IndexType == "key")
{
doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED));
doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
if (sort)
else if (fieldconfig != null && fieldconfig.IndexType == "html")
{
doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED));
if (index)
{
doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
else if (fieldconfig != null && fieldconfig.IndexType == "file")
{
var val = value.Value.ToString();
if (!string.IsNullOrEmpty(val))
else if (fieldconfig != null && fieldconfig.IndexType == "file")
{
var fileIndexer = FileIndexerManager.GetFileIndexer(val);
if (fileIndexer != null)
var val = value.Value.ToString();
if (!string.IsNullOrEmpty(val))
{
var content = fileIndexer.GetContent(val);
if (index)
var fileIndexer = FileIndexerManager.GetFileIndexer(val);
if (fileIndexer != null)
{
doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
var content = fileIndexer.GetContent(val);
if (index)
{
doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
}
}
else
{

var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString());
val = val.Replace('-', ' '); // concider '-' as a space
val = val.Replace(',', ' '); // concider ',' as a space
//var val = LuceneUtils.CleanupText(value.Value.ToString());
if (index)
else
{
doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED));
var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString());
val = val.Replace('-', ' '); // concider '-' as a space
val = val.Replace(',', ' '); // concider ',' as a space
//var val = LuceneUtils.CleanupText(value.Value.ToString());
if (index)
{
doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
if (sort)
break;

case JTokenType.TimeSpan:
if (index || sort)
{
doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks));
}
}
break;
break;

case JTokenType.TimeSpan:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks));
}
break;

case JTokenType.Uri:
if (index || sort)
{
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED));
}
break;
case JTokenType.Uri:
if (index || sort)
{
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED));
}
break;

default:
Debug.Fail("Unsupported JValue type: " + value.Type);
break;
default:
Debug.Fail("Unsupported JValue type: " + value.Type);
break;
}
}
else
{
Debug.Fail("Unsupported JToken: " + token);
}
}
else
catch (Exception e)
{
Debug.Fail("Unsupported JToken: " + token);
if(Debugger.IsAttached) Debugger.Break();
throw;
}
}

Expand Down
9 changes: 8 additions & 1 deletion OpenContent/Components/OpenContentInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,14 @@ public JToken JsonAsJToken
{
if (_jsonAsJToken == null && !string.IsNullOrEmpty(this.Json))
{
_jsonAsJToken = JToken.Parse(this.Json);
try
{
_jsonAsJToken = JToken.Parse(this.Json);
}
catch (Exception e)
{
throw new Exception($"Failed to parse json from moduleId:{ModuleId}, contentId:{ContentId}", e);
}
}
// JsonAsJToken is modified (to remove other cultures)
return _jsonAsJToken?.DeepClone();
Expand Down

0 comments on commit 97c2233

Please sign in to comment.