Add URLENCODE, FORMATNUMBER, BUILDROWSETFROMSTRING (#26)

* Add additional AMPscript functions

Adds the following functions:
URLENCODE - Modifies a string to only include characters that are safe to use in URLs.

BUILDROWSETFROMSTRING - Creates a rowset from a character string by splitting the string at the specified delimiter.

FORMATNUMBER - Formats a number as a numeric type, such as a decimal, date, or currency value.

There is a known issue with FORMATNUMBER for rounding digits.

Specifically:
https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings

On .NET Framework and .NET Core up to .NET Core 2.0, the runtime selects the result with the greater least significant digit (that is, using MidpointRounding.AwayFromZero).
On .NET Core 2.1 and later, the runtime selects the result with an even least significant digit (that is, using MidpointRounding.ToEven).

The in-production AMPscript implementation is on .net framework, this is .net 8.

src/Sage.Engine.Tests/Corpus/CorpusLoader.cs

@@ -120,8 +120,8 @@ private static IEnumerable<CorpusData> GetTestsFromFileUsingHandmadeParser(strin
 
  testsFromFile.Add(new CorpusData(testName, code.ToString().Trim(), file, lineStart)
  {
- Output = expectedOutput,
- ParseTree = parseTree.ToString().Trim()
+ Output = expectedOutput.ReplaceLineEndings("\n"),
+ ParseTree = parseTree.ToString().Trim().ReplaceLineEndings("\n")
  });
 
  parseTree = new StringBuilder();
@@ -182,8 +182,8 @@ private static IEnumerable<CorpusData> GetTestsFromFileUsingHandmadeParser(strin
  expectedOutput = programOutput.ToString().Trim();
  testsFromFile.Add(new CorpusData(testName, code.ToString().Trim(), file, lineStart)
  {
- Output = expectedOutput,
- ParseTree = parseTree.ToString().Trim()
+ Output = expectedOutput.ReplaceLineEndings("\n"),
+ ParseTree = parseTree.ToString().Trim().ReplaceLineEndings("\n")
  });
 
  return testsFromFile;

src/Sage.Engine.Tests/Corpus/Function/http.txt

@@ -3,4 +3,43 @@ REDIRECTO
 ==========
 %%=REDIRECTTO("http://example.com")=%%
 ++++++++++
-http://example.com
+http://example.com
+==========
+URLENCODE("Hello World", false, false)
+==========
+%%=URLENCODE("Hello World")=%%
+++++++++++
+Hello World
+==========
+URLENCODE("Hello World", false, false)
+==========
+%%=URLENCODE("Hello World", 0, 1)=%%
+++++++++++
+Hello%20World
+==========
+URLENCODE Only Spaces As HTML
+==========
+%%=URLENCODE("http://example.com?query=Hello World,",false,false)=%%
+++++++++++
+http://example.com?query=Hello%20World,
+==========
+URLENCODE Spaces Plus And All Others
+==========
+%%=URLENCODE("http://example.com?query=Hello World,",true,false)=%%
+++++++++++
+http://example.com?query%3dHello+World%2c
+==========
+URLENCODE Unicode
+==========
+%%=URLEncode('Sample Text: サンプルテキスト', true, true)=%%
+++++++++++
+Sample+Text%3a+%e3%82%b5%e3%83%b3%e3%83%97%e3%83%ab%e3%83%86%e3%82%ad%e3%82%b9%e3%83%88
+==========
+URLENCODE AMPscript In URL
+==========
+%%[
+ SET @url = "http://example.com?add=%%=ADD(1,2)=%%"
+]%%
+%%=URLEncode(@url)=%%
+++++++++++
+http://example.com?add=3

src/Sage.Engine.Tests/Corpus/Function/strings.txt

@@ -244,4 +244,47 @@ set @lowercase = "^[a-z]+$"
 45678
 abc
 
-ABC
+ABC
+==========
+BUILDROWSETFROMSTRING
+==========
+%%[
+SET @STRING = "ONE|TWO|THREE|FOUR"
+SET @ROWS=BUILDROWSETFROMSTRING(@STRING, "|")
+SET @ROWCOUNT=ROWCOUNT(@ROWS)
+]%%
+%%=V(@ROWCOUNT)=%%
+%%=V(FIELD(ROW(@ROWS, 1), 1))=%%
+%%=V(FIELD(ROW(@ROWS, 2), 1))=%%
+%%=V(FIELD(ROW(@ROWS, 3), 1))=%%
+%%=V(FIELD(ROW(@ROWS, 4), 1))=%%
+++++++++++
+4
+ONE
+TWO
+THREE
+FOUR
+==========
+BUILDROWSETFROMSTRING bad row
+==========
+%%[
+SET @STRING = "ONE|TWO|THREE|FOUR"
+SET @ROWS=BUILDROWSETFROMSTRING(@STRING, "|")
+SET @ROWCOUNT=ROWCOUNT(@ROWS)
+]%%
+%%=V(@ROWCOUNT)=%%
+%%=V(FIELD(ROW(@ROWS, 5), 1))=%%
+++++++++++
+!
+==========
+BUILDROWSETFROMSTRING bad field
+==========
+%%[
+SET @STRING = "ONE|TWO|THREE|FOUR"
+SET @ROWS=BUILDROWSETFROMSTRING(@STRING, "|")
+SET @ROWCOUNT=ROWCOUNT(@ROWS)
+]%%
+%%=V(@ROWCOUNT)=%%
+%%=V(FIELD(ROW(@ROWS, 1), 2))=%%
+++++++++++
+!

src/Sage.Engine.Tests/Corpus/Function/utility.txt

@@ -49,4 +49,99 @@ set @num = 12345
 ]%%
 %%=Format(@num,"C", "Numeric")=%%
 ++++++++++
-$12,345.00
+$12,345.00
+==========
+GETSENDTIME matches NOW
+==========
+%%[
+SET @NOW1 = FORMAT(NOW(), "YYYY-MM-dd hh:mm")
+SET @SENDTIME = FORMAT(GETSENDTIME(), "YYYY-MM-dd hh:mm")
+SET @NOW2 = FORMAT(NOW(), "YYYY-MM-dd hh:mm")
+
+/* 
+This tests the time twice (now1 and now2) because if the test runs during a minute boundary, now1 may be :59 and SENDTIME might be :00.
+By doing it before & after, it'll guarantee to catch a minute rollover.
+*/
+
+
+IF (@NOW1 == @SENDTIME) THEN
+]%%
+PASS
+%%[
+ELSEIF (@SENDTIME == @NOW2) THEN
+]%%
+PASS
+%%[
+ELSE
+]%%
+FAIL
+%%[
+ENDIF
+]%%
+++++++++++
+PASS
+==========
+GETSENDTIME(TRUE) matches NOW
+==========
+%%[
+SET @NOW1 = FORMAT(NOW(), "YYYY-MM-dd hh:mm")
+SET @SENDTIME = FORMAT(GETSENDTIME(TRUE), "YYYY-MM-dd hh:mm")
+SET @NOW2 = FORMAT(NOW(), "YYYY-MM-dd hh:mm")
+
+/* 
+This tests the time twice (now1 and now2) because if the test runs during a minute boundary, now1 may be :59 and SENDTIME might be :00.
+By doing it before & after, it'll guarantee to catch a minute rollover.
+*/
+
+IF (@NOW1 == @SENDTIME) THEN
+]%%
+PASS
+%%[
+ELSEIF (@SENDTIME == @NOW2) THEN
+]%%
+PASS
+%%[
+ELSE
+]%%
+FAIL
+%%[
+ENDIF
+]%%
+++++++++++
+PASS
+==========
+FORMATNUMBER Rounding
+==========
+%%=FormatNumber(123.4451,"N2")=%%
+++++
+123.45
+==========
+FORMATNUMBER Converting Strings to Numbers 
+==========
+%%=FormatNumber("1234.56", "N")=%%
+++++
+1,234.56
+==========
+FORMATNUMBER Normalizing Numbers 
+==========
+%%=FormatNumber("1,234.56","G","en_US")=%%
+++++
+1234.56
+==========
+FORMATNUMBER Localizing Numbers 
+==========
+%%=FormatNumber("123.4451","C2","de_DE")=%%
+++++
+123,45 €
+==========
+FORMATNUMBER Converting Numbers to Percentages
+==========
+%%[
+ Set @val1 = 42
+ Set @val2 = 326
+ Set @pct = Divide(@val1, @val2)
+]%%
+
+%%=FormatNumber(@pct, "P3")=%%
+++++
+12.883%

src/Sage.Engine.Tests/Corpus/Incompatible/Compatibility.ampscript

@@ -35,4 +35,25 @@ set @result = DATEPARSE("2023-05-01 2:30PM-1:00", 0)
 ]%%
 %%=v(@result)=%%
 ++++++++++
-5/1/2023 11:30:00 AM
+5/1/2023 11:30:00 AM
+==========
+FORMATNUMBER Rounding
+==========
+%%[
+/*
+https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings
+
+Specifically:
+On .NET Framework and .NET Core up to .NET Core 2.0, the runtime selects the result with the greater least significant digit (that is, using MidpointRounding.AwayFromZero).
+On .NET Core 2.1 and later, the runtime selects the result with an even least significant digit (that is, using MidpointRounding.ToEven).
+
+The existing implementation is on .net framework, this is .net core 2.1.
+
+This means that rounding will be different for situations like below.
+
+The AMPscript implementation is 123.45, AMPscript Core is 123.44 due to this difference.
+*/
+]%%
+%%=FormatNumber(123.445,"N2")=%%
+++++
+123.44

src/Sage.Engine.Tests/EngineTestAttribute.cs

@@ -99,7 +99,7 @@ public ParserTestAttribute(string baseDirectory) : base(baseDirectory)
 
  protected override object GetTestValidationData(CorpusData data)
  {
- return new ParserTestResult(data.ParseTree);
+ return new ParserTestResult(data.ParseTree?.ReplaceLineEndings("\n") ?? string.Empty);
  }
  }
 
@@ -114,7 +114,7 @@ public RuntimeTestAttribute(string baseDirectory) : base(baseDirectory)
 
  protected override object GetTestValidationData(CorpusData data)
  {
- return new EngineTestResult(data.Output);
+ return new EngineTestResult(data.Output?.ReplaceLineEndings("\n") ?? string.Empty);
  }
  }
 }

src/Sage.Engine.Tests/Functions/FunctionTests.cs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
 
+using MarketingCloudIntegration.Render;
+
 namespace Sage.Engine.Tests.Functions
 {
  using NUnit.Framework;
@@ -23,23 +25,18 @@ public EngineTestResult TestFunctionsFromScriptCode(CorpusData test)
  [Category("Compatibility")]
  public EngineTestResult TestFunctionsFromScriptCodeCompatibility(CorpusData test)
  {
- var result = TestUtils.GetOutputFromTest(_serviceProvider, test);
- Assert.That(result.Output, Is.EqualTo(test.Output));
-
  if (test.Output == "!")
  {
  Assert.Throws<AggregateException>(() =>
  TestCompatibility(test.Code, test.SubscriberContext?.GetAttributes()).Wait()
  );
+ return new EngineTestResult("!");
  }
  else
  {
- var compatibilityResult = TestCompatibility(test.Code, test.SubscriberContext?.GetAttributes()).Result;
-
- Assert.That(compatibilityResult.renderedContent?.Trim().ReplaceLineEndings("\n"), Is.EqualTo(test.Output?.ReplaceLineEndings("\n")));
+ RenderResponse compatibilityResult = TestCompatibility(test.Code, test.SubscriberContext?.GetAttributes()).Result;
+ return new EngineTestResult(compatibilityResult.renderedContent?.Trim().ReplaceLineEndings("\n"));
  }
-
- return result;
  }
  }
 }

src/Sage.Engine.Tests/TestUtils.cs

@@ -22,7 +22,7 @@ public static ParserTestResult GetParseResultFromTest(CorpusData test)
  ParserTestResult result;
 
  IParseTree parseTree = AntlrParser.Parse(test.Code, Console.Out, Console.Error);
- string serializedTree = AntlrParser.SerializeTree(parseTree);
+ string serializedTree = AntlrParser.SerializeTree(parseTree).ReplaceLineEndings("\n");
  result = new ParserTestResult(serializedTree);
 
  File.WriteAllText(Path.Combine(TestContext.CurrentContext.WorkDirectory, $"{test.FileFriendlyName}_expected.txt"),
@@ -52,7 +52,7 @@ public static EngineTestResult GetOutputFromTest(IServiceProvider serviceProvide
 
  try
  {
- var engineResult = new EngineTestResult(CSharpCompiler.CompileAndExecute(options, GetTestRuntimeContext(serviceProvider, options, test), out CompileResult compileResult).Trim());
+ var engineResult = new EngineTestResult(CSharpCompiler.CompileAndExecute(options, GetTestRuntimeContext(serviceProvider, options, test), out CompileResult compileResult).ReplaceLineEndings("\n").Trim());
 
  File.WriteAllText(Path.Combine(TestContext.CurrentContext.WorkDirectory, $"{test.FileFriendlyName}_transpiled.cs"), compileResult.TranspiledSource);
 

src/Sage.Engine/Globalization/CompatibleGlobalizationSettings.cs

@@ -9,23 +9,62 @@ namespace Sage.Engine.Runtime
 {
  public static class CompatibleGlobalizationSettings
  {
+ /// <summary>
+ /// AMPscript supports both "en-us" AND "en_us". In order to quickly support this lookup, a dictionary of names->cultures is made and populated
+ /// on demand.
+ /// </summary>
+ private static readonly Lazy<Dictionary<string, CultureInfo>> s_cultureLookup = new(CacheCultureNameLookup, LazyThreadSafetyMode.PublicationOnly);
+
+ /// <summary>
+ /// Caches a lookup of name->culture, along with any modifications that are needed to be compatible with the existing language.
+ ///
+ /// This cache is populated at first access.
+ /// </summary>
+ private static Dictionary<string, CultureInfo> CacheCultureNameLookup()
+ {
+ CultureInfo[] validCultures = CultureInfo.GetCultures(CultureTypes.AllCultures);
+ var returnDictionary = new Dictionary<string, CultureInfo>(validCultures.Length, StringComparer.InvariantCultureIgnoreCase);
+ foreach (CultureInfo culture in validCultures)
+ {
+ CultureInfo usedCulture = GenerateCompatibleCulture(culture);
+ returnDictionary[usedCulture.Name] = usedCulture;
+
+ if (usedCulture.Name.IndexOf("-", StringComparison.Ordinal) > 0)
+ {
+ returnDictionary[usedCulture.Name.Replace("-", "_")] = usedCulture;
+ }
+ }
+
+ return returnDictionary;
+ }
+
  /// <summary>
  /// There will be many differences between linux, mac, windows for globalization settings.
  /// Example, see: https://github.com/dotnet/runtime/issues/18345
  /// In that issue, there is no desire to support common settings between different OSs.
  /// In order to maintain compatibility with existing code, the culture infos are hardcoded.
- /// For now, just support en-US. In the future, a more robust test stratgy and validation needs created.
+ /// For now, just support en-US. In the future, a more robust test strategy and validation needs created.
  /// </summary>
- public static CultureInfo GetCulture(string culture)
+ private static CultureInfo GenerateCompatibleCulture(CultureInfo culture)
  {
- CultureInfo returnCulture = (CultureInfo)(new CultureInfo(culture, false)).Clone();
-
- if (culture.ToLower() == "en-us")
+ if (culture.Name.ToLower() == "en-us")
  {
+ var returnCulture = (CultureInfo)culture.Clone();
  returnCulture.DateTimeFormat.ShortDatePattern = "M/d/yyyy";
+ returnCulture.NumberFormat.NumberDecimalDigits = 2;
+
+ return returnCulture;
  }
 
- return returnCulture;
+ return culture;
+ }
+
+ /// <summary>
+ /// Obtains the compatible CultureInfo from the given culture name
+ /// </summary>
+ public static CultureInfo GetCulture(string culture)
+ {
+ return s_cultureLookup.Value[culture];
  }
  }
 }