From 308d8eb8afe54cdc97a2f01ea5ee37e78f90f6d6 Mon Sep 17 00:00:00 2001 From: cidrugHug8 Date: Sun, 17 Dec 2023 08:53:12 +0900 Subject: [PATCH] Refactored namespace and updated class names. Added explanations to README.md. --- BleuNet/{BleuScore.cs => Metrics.cs} | 37 +------------- BleuNet/Utility.cs | 72 ++++++++++++++++++++++++++++ BleuNetTest/BleuTests.cs | 62 ++++++++++++------------ ConsoleTest/Program.cs | 31 +++++++----- README.md | 20 ++++++-- 5 files changed, 139 insertions(+), 83 deletions(-) rename BleuNet/{BleuScore.cs => Metrics.cs} (94%) create mode 100644 BleuNet/Utility.cs diff --git a/BleuNet/BleuScore.cs b/BleuNet/Metrics.cs similarity index 94% rename from BleuNet/BleuScore.cs rename to BleuNet/Metrics.cs index fe09e01..c22af7c 100644 --- a/BleuNet/BleuScore.cs +++ b/BleuNet/Metrics.cs @@ -1,11 +1,9 @@ using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Text.RegularExpressions; namespace BleuNet { - public static class BleuScore + public static class Metrics { /// /// Fraction class. @@ -348,38 +346,6 @@ private static Dictionary Ngrams(string[] words, int n) return ngrams; } - public static string[] Tokenize(string line, bool lc = true) - { - string norm = line; - - if (lc) - { - norm = norm.ToLower(); - } - - // language-independent part: - norm = norm.Replace("", ""); - norm = norm.Replace("-\n", ""); - norm = norm.Replace("\n", " "); - norm = norm.Replace(""", "\""); - norm = norm.Replace("&", "&"); - norm = norm.Replace("<", "<"); - norm = norm.Replace(">", ">"); - - // language-dependent part (assuming Western languages): - norm = " " + norm + " "; - norm = Regex.Replace(norm, "([\\{-\\~\\[-\\` -\\&\\(-\\+\\:-\\@\\/])", " $1 "); - norm = Regex.Replace(norm, "([^0-9])([\\.,])", "$1 $2 "); - norm = Regex.Replace(norm, "([\\.,])([^0-9])", " $1 $2"); - norm = Regex.Replace(norm, "([0-9])(-)", "$1 $2 "); - norm = Regex.Replace(norm, "\\s+", " "); // one space only between words - norm = norm.Trim(); // no leading or trailing space - - var segmented = norm.Split(); - - return segmented; - } - public static (double nkt, double precision, double bp) CalculateKendallsTau(string[] reference, string[] hypothesis) { static string MapWordsToUnicode(string[] words, Dictionary wordDict) @@ -500,7 +466,6 @@ static string GetNgram(string text, int start, int length) var p = n / (double)hypothesis.Length; - // 結果を返す return (nkt, p, bp); } diff --git a/BleuNet/Utility.cs b/BleuNet/Utility.cs new file mode 100644 index 0000000..b90a0ad --- /dev/null +++ b/BleuNet/Utility.cs @@ -0,0 +1,72 @@ +using System.Text.RegularExpressions; + +namespace BleuNet +{ + /// + /// The Utility class in the BleuNet namespace. + /// This class provides utility methods for tokenizing strings and benchmarking. + /// + public static class Utility + { + /// + /// Tokenizes the input string into an array of words. + /// + /// The input string to tokenize. + /// A boolean value indicating whether to convert the input string to lower case. Default is true. + /// An array of words. + public static string[] Tokenize(string line, bool lc = true) + { + string norm = line; + + // Convert the string to lower case if lc is true. + if (lc) + { + norm = norm.ToLower(); + } + + // language-independent part: + // Replace certain characters and strings with others. + norm = norm.Replace("", ""); + norm = norm.Replace("-\n", ""); + norm = norm.Replace("\n", " "); + norm = norm.Replace(""", "\""); + norm = norm.Replace("&", "&"); + norm = norm.Replace("<", "<"); + norm = norm.Replace(">", ">"); + + // language-dependent part (assuming Western languages): + // Add spaces around certain characters and strings. + norm = " " + norm + " "; + norm = Regex.Replace(norm, "([\\{-\\~\\[-\\` -\\&\\(-\\+\\:-\\@\\/])", " $1 "); + norm = Regex.Replace(norm, "([^0-9])([\\.,])", "$1 $2 "); + norm = Regex.Replace(norm, "([\\.,])([^0-9])", " $1 $2"); + norm = Regex.Replace(norm, "([0-9])(-)", "$1 $2 "); + norm = Regex.Replace(norm, "\\s+", " "); // one space only between words + norm = norm.Trim(); // no leading or trailing space + + // Split the normalized string into words. + var segmented = norm.Split(); + + return segmented; + } + + /// + /// Runs a benchmark on the CorpusBleu method. + /// + /// The file path to the reference text file. Default is "./reference.txt". + /// The file path to the hypothesis text file. Default is "./hypothesis.txt". + public static void Benchmark(string referenceFilepath="./reference.txt", string hypothesisFilepath= "./hypothesis.txt") + { + for (var i = 0; i < 100; i++) + { + var references = File.ReadAllLines(referenceFilepath) + .Select(x => Utility.Tokenize(x)) + .ToArray(); + var hypotheses = File.ReadAllLines(hypothesisFilepath) + .Select(x => Utility.Tokenize(x)) + .ToArray(); + var bleuScore = Metrics.CorpusBleu(references, hypotheses); + } + } + } +} diff --git a/BleuNetTest/BleuTests.cs b/BleuNetTest/BleuTests.cs index 0cc1cee..d0541bb 100644 --- a/BleuNetTest/BleuTests.cs +++ b/BleuNetTest/BleuTests.cs @@ -14,12 +14,12 @@ public void TestModifiedPrecision1() var references = new string[][] { ref1, ref2 }; - var hyp1UnigramPrecision = BleuScore.ModifiedPrecision(references, hyp1, 1); + var hyp1UnigramPrecision = Metrics.ModifiedPrecision(references, hyp1, 1); Assert.Equal(0.2857, Math.Round(hyp1UnigramPrecision, 4)); Assert.Equal(0.28571428, hyp1UnigramPrecision, 0.0001); - Assert.Equal(0.0, BleuScore.ModifiedPrecision(references, hyp1, 2)); + Assert.Equal(0.0, Metrics.ModifiedPrecision(references, hyp1, 2)); } [Fact] @@ -33,9 +33,9 @@ public void TestModifiedPrecision2() var references = new string[][] { ref1, ref2, ref3 }; - Assert.Equal(1.0, BleuScore.ModifiedPrecision(references, hyp1, 1)); + Assert.Equal(1.0, Metrics.ModifiedPrecision(references, hyp1, 1)); - Assert.Equal(1.0, BleuScore.ModifiedPrecision(references, hyp1, 2)); + Assert.Equal(1.0, Metrics.ModifiedPrecision(references, hyp1, 2)); } [Fact] @@ -50,8 +50,8 @@ public void TestModifiedPrecision3() var references = new string[][] { ref1, ref2, ref3 }; - var hyp1UnigramPrecision = BleuScore.ModifiedPrecision(references, hyp1, 1); - var hyp2UnigramPrecision = BleuScore.ModifiedPrecision(references, hyp2, 1); + var hyp1UnigramPrecision = Metrics.ModifiedPrecision(references, hyp1, 1); + var hyp2UnigramPrecision = Metrics.ModifiedPrecision(references, hyp2, 1); Assert.Equal(0.94444444, hyp1UnigramPrecision, 0.0001); Assert.Equal(0.57142857, hyp2UnigramPrecision, 0.0001); @@ -59,8 +59,8 @@ public void TestModifiedPrecision3() Assert.Equal(0.9444, Math.Round(hyp1UnigramPrecision, 4)); Assert.Equal(0.5714, Math.Round(hyp2UnigramPrecision, 4)); - var hyp1BigramPrecision = BleuScore.ModifiedPrecision(references, hyp1, 2); - var hyp2BigramPrecision = BleuScore.ModifiedPrecision(references, hyp2, 2); + var hyp1BigramPrecision = Metrics.ModifiedPrecision(references, hyp1, 2); + var hyp2BigramPrecision = Metrics.ModifiedPrecision(references, hyp2, 2); Assert.Equal(0.58823529, hyp1BigramPrecision, 0.0001); Assert.Equal(0.07692307, hyp2BigramPrecision, 0.0001); @@ -75,13 +75,13 @@ public void TestBrevityPenalty() var references = new string[][] { Enumerable.Repeat("a", 11).ToArray(), Enumerable.Repeat("a", 8).ToArray() }; var hypothesis = Enumerable.Repeat("a", 7).ToArray(); var hypLen = hypothesis.Length; - var closestRefLen = BleuScore.ClosestRefLength(references, hypLen); - Assert.Equal(0.8669, BleuScore.BrevityPenalty(closestRefLen, hypLen), 0.0001); + var closestRefLen = Metrics.ClosestRefLength(references, hypLen); + Assert.Equal(0.8669, Metrics.BrevityPenalty(closestRefLen, hypLen), 0.0001); references = [Enumerable.Repeat("a", 11).ToArray(), Enumerable.Repeat("a", 8).ToArray(), Enumerable.Repeat("a", 6).ToArray(), Enumerable.Repeat("a", 7).ToArray()]; hypLen = hypothesis.Length; - closestRefLen = BleuScore.ClosestRefLength(references, hypLen); - Assert.Equal(1.0, BleuScore.BrevityPenalty(closestRefLen, hypLen)); + closestRefLen = Metrics.ClosestRefLength(references, hypLen); + Assert.Equal(1.0, Metrics.BrevityPenalty(closestRefLen, hypLen)); } [Fact] @@ -95,7 +95,7 @@ public void TestZeroMatches() for (int n = 1; n < hypothesis.Length; n++) { double[] weights = Enumerable.Repeat(1.0 / n, n).ToArray(); // Uniform weights. - Assert.Equal(0.0, BleuScore.SentenceBleu(references, hypothesis, weights)); + Assert.Equal(0.0, Metrics.SentenceBleu(references, hypothesis, weights)); } } @@ -110,7 +110,7 @@ public void TestFullMatches() for (int n = 1; n < hypothesis.Length; n++) { double[] weights = Enumerable.Repeat(1.0 / n, n).ToArray(); // Uniform weights. - Assert.Equal(1.0, BleuScore.SentenceBleu(references, hypothesis, weights)); + Assert.Equal(1.0, Metrics.SentenceBleu(references, hypothesis, weights)); } } @@ -122,7 +122,7 @@ public void TestPartialMatchesHypothesisLongerThanReference() // Since no 4-grams matches were found the result should be zero // exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0 - Assert.Equal(0.0, BleuScore.SentenceBleu(references, hypothesis), 0.0001); + Assert.Equal(0.0, Metrics.SentenceBleu(references, hypothesis), 0.0001); // Checks that the warning has been raised because len(reference) < 4. // In C#, there's no direct equivalent for Python's warnings, so this part is omitted. @@ -144,14 +144,14 @@ public void TestCaseWhereNIsBiggerThanHypothesisLength() weights[i] = 1.0 / n; } - double bleuScore = BleuScore.SentenceBleu(references, hypothesis, weights); + double bleuScore = Metrics.SentenceBleu(references, hypothesis, weights); Assert.Equal(0.0, bleuScore, 4); references = ["John", "loves", "Mary"]; hypothesis = ["John", "loves", "Mary"]; - bleuScore = BleuScore.SentenceBleu(references, hypothesis, weights); + bleuScore = Metrics.SentenceBleu(references, hypothesis, weights); Assert.Equal(0.0, bleuScore, 4); } @@ -162,7 +162,7 @@ public void TestEmptyHypothesis() var references = new string[] { "The", "candidate", "has", "no", "alignment", "to", "any", "of", "the", "references" }; string[] hypothesis = []; - double bleuScore = BleuScore.SentenceBleu(references, hypothesis); + double bleuScore = Metrics.SentenceBleu(references, hypothesis); Assert.Equal(0.0, bleuScore); } @@ -189,7 +189,7 @@ public void TestEmptyReferences() var references = new string[][] { [] }; var hypothesis = new string[] { "John", "loves", "Mary" }; - double bleuScore = BleuScore.SentenceBleu(references, hypothesis); + double bleuScore = Metrics.SentenceBleu(references, hypothesis); Assert.Equal(0.0, bleuScore); } @@ -200,7 +200,7 @@ public void TestEmptyReferencesAndHypothesis() string[][] references = [[]]; string[] hypothesis = []; - double bleuScore = BleuScore.SentenceBleu(references, hypothesis); + double bleuScore = Metrics.SentenceBleu(references, hypothesis); Assert.Equal(0.0, bleuScore); } @@ -211,7 +211,7 @@ public void TestReferenceOrHypothesisShorterThanFourgrams() var references = new string[] { "let", "it", "go" }; var hypothesis = new string[] { "let", "go", "it" }; - double bleuScore = BleuScore.SentenceBleu(references, hypothesis); + double bleuScore = Metrics.SentenceBleu(references, hypothesis); Assert.Equal(0.0, bleuScore, 4); } @@ -228,7 +228,7 @@ public void TestNumpyWeights() weights[i] = 0.25; } - double bleuScore = BleuScore.SentenceBleu(references, hypothesis, weights); + double bleuScore = Metrics.SentenceBleu(references, hypothesis, weights); Assert.Equal(0.0, bleuScore); } @@ -249,7 +249,7 @@ public void TestCorpusBleuWithBadSentence() // Check that the warning is raised since no. of 2-grams < 0. // Verify that the BLEU output is undesired since no. of 2-grams < 0. - Assert.Equal(0.0, BleuScore.CorpusBleu(references, hypotheses, new double[] { 0.25, 0.25, 0.25, 0.25 }), 0.0001); + Assert.Equal(0.0, Metrics.CorpusBleu(references, hypotheses, new double[] { 0.25, 0.25, 0.25, 0.25 }), 0.0001); } } @@ -364,25 +364,25 @@ public void TestCorpusBleuWithMultipleWeights() var weight2 = new double[] { 0.25, 0.25, 0.25, 0.25 }; var weight3 = new double[] { 0.0, 0.0, 0.0, 1.0 }; - double[] bleuScores = BleuScore.CorpusBleu( + double[] bleuScores = Metrics.CorpusBleu( [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], new double[][] { weight1, weight2, weight3 } ); - Assert.Equal(bleuScores[0], BleuScore.CorpusBleu( + Assert.Equal(bleuScores[0], Metrics.CorpusBleu( [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight1 )); - Assert.Equal(bleuScores[1], BleuScore.CorpusBleu( + Assert.Equal(bleuScores[1], Metrics.CorpusBleu( [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight2 )); - Assert.Equal(bleuScores[2], BleuScore.CorpusBleu( + Assert.Equal(bleuScores[2], Metrics.CorpusBleu( [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight3 @@ -399,7 +399,7 @@ public void TestCorpusRibes0() { string[][][] references = [["The candidate has no alignment to any of the references".Split()]]; string[][] hypothesis = ["John loves Mary".Split()]; - Assert.Equal(0.0, BleuScore.CorppusRibes(references, hypothesis)); + Assert.Equal(0.0, Metrics.CorppusRibes(references, hypothesis)); } [Fact] @@ -407,7 +407,7 @@ public void TestCorpusRibes1() { string[][][] ref1 = [["He enjoys taking a walk in the park every day .".Split()]]; string[][] hyp1 = ["He likes to walk in the park daily .".Split()]; - Assert.Equal(0.883743, BleuScore.CorppusRibes(ref1, hyp1), 0.000001); + Assert.Equal(0.883743, Metrics.CorppusRibes(ref1, hyp1), 0.000001); } [Fact] @@ -419,7 +419,7 @@ public void TestCorpusRibes2() string[][] hyp1 = [ "He likes to walk in the park daily, and then enjoys his coffee at a cafe while reading the newspaper, which is his daily routine .".Split() ]; - Assert.Equal(0.678417, BleuScore.CorppusRibes(ref1, hyp1), 0.000001); + Assert.Equal(0.678417, Metrics.CorppusRibes(ref1, hyp1), 0.000001); } [Fact] @@ -435,7 +435,7 @@ public void TestCorpusRibes3() //Assert.Equal(0.634183, BleuScore.CorppusRibes([[ref2]], hyp1), 0.000001); string[][][] references = [[ref1, ref2]]; - Assert.Equal(0.634183, BleuScore.CorppusRibes(references, hyp1), 0.000001); + Assert.Equal(0.634183, Metrics.CorppusRibes(references, hyp1), 0.000001); } } } \ No newline at end of file diff --git a/ConsoleTest/Program.cs b/ConsoleTest/Program.cs index a0abbce..a85c33d 100644 --- a/ConsoleTest/Program.cs +++ b/ConsoleTest/Program.cs @@ -1,5 +1,22 @@ using BleuNet; -using System.IO.MemoryMappedFiles; + +// Define the translated and reference sentences. +string referenceSentence = "The pessimist sees difficulty in every opportunity."; +string translatedSentence = "The pessimist sees difficulty at every opportunity."; + +var referenceSentenceTokens = new string[][] { Utility.Tokenize(referenceSentence) }; +var translatedSentenceTokens = new string[][] { Utility.Tokenize(translatedSentence) }; + +// Calculate the BLEU score. +double score = Metrics.CorpusBleu(referenceSentenceTokens, translatedSentenceTokens); + +// Display the result. +Console.WriteLine("BLEU Score: " + score); + +// Calculate the sentence BLEU score. +double sentenceBleu = Metrics.SentenceBleu(referenceSentenceTokens, Utility.Tokenize(translatedSentence)); +Console.WriteLine("Sentence BLEU Score: " + sentenceBleu); + //// Define the translated and reference sentences. //string referenceSentence = "The pessimist sees difficulty in every opportunity."; @@ -65,14 +82,4 @@ //bleuScore = BleuScore.CorpusBleu(humanTranslation, machineTranslation); //Console.WriteLine(bleuScore); -for (var i = 0; i < 100; i++) -{ - var references = File.ReadAllLines("./reference.txt") - .Select(x => BleuScore.Tokenize(x)) - .ToArray(); - var hypotheses = File.ReadAllLines("./hypothesis.txt") - .Select(x => BleuScore.Tokenize(x)) - .ToArray(); - var bleuScore = BleuScore.CorpusBleu(references, hypotheses); - //Console.WriteLine("BLEU Score: " + bleuScore); -} +Utility.Benchmark(); \ No newline at end of file diff --git a/README.md b/README.md index ac6605a..34f9856 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # BlueNet ## Overview -This library is a C# class library for calculating the BLEU score, a metric for evaluating the quality of machine translations. +This library is a C# class library for calculating the BLEU and RIBES scores, which are metrics for evaluating the quality of machine translations. BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. RIBES (Rank-based Intuitive Bilingual Evaluation Score) is an automatic metric for machine translation evaluation that is based on rank correlation coefficients between word pairs of reference and candidate translations. ## Installation You can add this library to your project using the NuGet package manager. @@ -20,15 +20,27 @@ using BleuNet; string referenceSentence = "The pessimist sees difficulty in every opportunity."; string translatedSentence = "The pessimist sees difficulty at every opportunity."; -var referenceSentenceTokens = new string[][] { BleuScore.Tokenize(referenceSentence) }; -var translatedSentenceTokens = new string[][] { BleuScore.Tokenize(translatedSentence) }; +var referenceSentenceTokens = new string[][] { Utility.Tokenize(referenceSentence) }; +var translatedSentenceTokens = new string[][] { Utility.Tokenize(translatedSentence) }; // Calculate the BLEU score. -double score = BleuScore.CorpusBleu(referenceSentenceTokens, translatedSentenceTokens); +double score = Metrics.CorpusBleu(referenceSentenceTokens, translatedSentenceTokens); // Display the result. Console.WriteLine("BLEU Score: " + score); + +// Calculate the sentence BLEU score. +double sentenceBleu = Metrics.SentenceBleu(referenceSentenceTokens, Utility.Tokenize(translatedSentence)); +Console.WriteLine("Sentence BLEU Score: " + sentenceBleu); ``` +## References + +**BLEU**: +1. Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu, "[BLEU: a Method for Automatic Evaluation of Machine Translation](https://aclanthology.org/P02-1040)" (Papineni et al., ACL 2002) + +**RIBES**: +1. Hideki Isozaki, Tsutomu Hirao, Kevin Duh, Katsuhito Sudoh, Hajime Tsukada, "[Automatic Evaluation of Translation Quality for Distant Language Pairs](https://aclanthology.org/D10-1092)" (Isozaki et al., EMNLP 2010) + ## License This project is licensed under the MIT license.