Skip to content

Commit

Permalink
Unity localisation now removes implicit lines once projects are tagged.
Browse files Browse the repository at this point in the history
Fixes #206
  • Loading branch information
McJones committed Feb 8, 2024
1 parent db6c551 commit b2007b4
Show file tree
Hide file tree
Showing 8 changed files with 295 additions and 56 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- automatically associating assets with localisations
- automatically linking YarnCommand and YarnFunction attributed methods to the dialogue runner
- generating a `ysls` file for all of your Yarn attributed methods
- this is save to `ProjectSettings\Packages\dev.yarnspinner\generated.ysls.json`
- this is saved to `ProjectSettings\Packages\dev.yarnspinner\generated.ysls.json`
- this is an experimental feature to support better editor integration down the line
- as such this defaults to *not* being generated
- enabling/disabling C# linking or ysls generation will force an entire C# reimport
Expand Down Expand Up @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Actions Registration now dumps generated code into the same temporary folder the logs live in
- `ActionsGenerator` will now generate C# warnings for incorrectly named methods that are attributed as `YarnFunction` or `YarnCommand`.
- Fixed a bug where `AudioLineProvider` didn't allow runtime changing of the text locale.
- Fixed a bug where the Unity Localisation strings tables would have duplicate lines after tagging all lines in a project.

### Removed

Expand Down
2 changes: 0 additions & 2 deletions Editor/Importers/YarnProjectImporter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -605,8 +605,6 @@ private void AddStringTableEntries(CompilationResult compilationResult, StringTa
});
}



// We've made changes to the table, so flag it and its shared
// data as dirty.
EditorUtility.SetDirty(table);
Expand Down
84 changes: 49 additions & 35 deletions Editor/Utility/YarnProjectUtility.cs
Original file line number Diff line number Diff line change
Expand Up @@ -385,14 +385,16 @@ private static bool UpdateLocalizationFile(IEnumerable<StringTableEntry> baseLoc
return true;
}

internal static void AddLineTagsToFilesInYarnProject(YarnProjectImporter importer)
private static (List<string>, List<string>) ExtantLineTags(YarnProjectImporter importer)
{
// First, gather all existing line tags across ALL yarn
// projects, so that we don't accidentally overwrite an
// existing one. Do this by finding all yarn scripts in all
// yarn projects, and get the string tags inside them.

var allYarnFiles =
// existing one. Do this by finding all yarn projects,
// and get the string tags inside them.
// By doing it in this way we get the same implicit tags
// from the project as the importer would normally do,
// letting us then do a direct comparision for them.
var allYarnProjects =
// get all yarn projects across the entire project
AssetDatabase.FindAssets($"t:{nameof(YarnProject)}")
// Get the path for each asset's GUID
Expand All @@ -403,60 +405,72 @@ internal static void AddLineTagsToFilesInYarnProject(YarnProjectImporter importe
.OfType<YarnProjectImporter>()
// Ensure that its import data is present
.Where(i => i.ImportData != null)
// Get all of their source scripts, as a single sequence
.SelectMany(i => i.ImportData.yarnFiles)
// Get the path for each asset
.Select(sourceAsset => AssetDatabase.GetAssetPath(sourceAsset))
// get each asset importer for that path
.Select(path => AssetImporter.GetAtPath(path))
// ensure that it's a YarnImporter
.OfType<YarnImporter>()
// get the path for each importer's asset (the compiler
// will use this)
.Select(i => AssetDatabase.GetAssetPath(i))
// remove any nulls, in case any are found
.Where(path => path != null);
// get the project out, and also flag if it is the project for THIS importer
.Select(i => (i.GetProject(), i == importer))
// remove any nulls just in case any are found
.Where(p => p.Item1 != null);

#if YARNSPINNER_DEBUG
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
#endif

var library = Actions.GetLibrary();

var allExistingTags = new List<string>();
var projectImplicitTags = new List<string>();

// Compile all of these, and get whatever existing string tags
// they had. Do each in isolation so that we can continue even
// if a file contains a parse error.
var allExistingTags = allYarnFiles.SelectMany(path =>
// if a project contains a parse error.
foreach (var tuple in allYarnProjects)
{
// Compile this script in strings-only mode to get
// string entries
var compilationJob = Yarn.Compiler.CompilationJob.CreateFromFiles(path);
var project = tuple.Item1;
var compilationJob = Yarn.Compiler.CompilationJob.CreateFromFiles(project.SourceFiles);
compilationJob.CompilationType = Yarn.Compiler.CompilationJob.Type.StringsOnly;
compilationJob.Library = library;

var result = Yarn.Compiler.Compiler.Compile(compilationJob);
bool containsErrors = result.Diagnostics
.Any(d => d.Severity == Compiler.Diagnostic.DiagnosticSeverity.Error);
if (containsErrors) {
Debug.LogWarning($"Can't check for existing line tags in {path} because it contains errors.");
return new string[] { };
bool containsErrors = result.Diagnostics.Any(d => d.Severity == Compiler.Diagnostic.DiagnosticSeverity.Error);
if (containsErrors)
{
Debug.LogWarning($"{project} has errors so cannot be scanned for tagging.");
continue;
}
allExistingTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == false).Select(i => i.Key));

return result.StringTable.Where(i => i.Value.isImplicitTag == false).Select(i => i.Key);
}).ToList(); // immediately execute this query so we can determine timing information
// we add the implicit lines IDs only for this project
if (tuple.Item2)
{
projectImplicitTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == true).Select(i => i.Key));
}
}

#if YARNSPINNER_DEBUG
stopwatch.Stop();
Debug.Log($"Checked {allYarnFiles.Count()} yarn files for line tags in {stopwatch.ElapsedMilliseconds}ms");
Debug.Log($"Checked {allYarnProjects.Count()} yarn files for line tags in {stopwatch.ElapsedMilliseconds}ms");
#endif
return (allExistingTags, projectImplicitTags);
}

internal static void AddLineTagsToFilesInYarnProject(YarnProjectImporter importer)
{
var extantTags = YarnProjectUtility.ExtantLineTags(importer);
var allExistingTags = extantTags.Item1;

#if USE_UNITY_LOCALIZATION
// if we are using Unity localisation we need to first remove the implicit tags for this project from the strings table
if (importer.UseUnityLocalisationSystem && importer.unityLocalisationStringTableCollection != null)
{
foreach (var implicitTag in extantTags.Item2)
{
importer.unityLocalisationStringTableCollection.RemoveEntry(implicitTag);
}
}
#endif

var modifiedFiles = new List<string>();

try
{

AssetDatabase.StartAssetEditing();

foreach (var script in importer.ImportData.yarnFiles)
Expand Down
4 changes: 2 additions & 2 deletions Runtime/LineProviders/UnityLocalisedLineProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
using UnityEngine.ResourceManagement.AsyncOperations;
using UnityEngine.ResourceManagement;
using UnityEngine.Localization.Metadata;

#endif

#if UNITY_EDITOR
Expand Down Expand Up @@ -318,7 +317,8 @@ public override LocalizedLine GetLocalizedLine(Yarn.Line line)
}

#if USE_UNITY_LOCALIZATION
public class LineMetadata : IMetadata {
public class LineMetadata : IMetadata
{
public string nodeName;
public string[] tags;
}
Expand Down
196 changes: 196 additions & 0 deletions Tests/Editor/UnityLocalisationTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.IO;

using UnityEngine;
using UnityEngine.TestTools;
using UnityEditor;

#if USE_UNITY_LOCALIZATION
using UnityEngine.Localization;
using UnityEngine.Localization.Settings;
using UnityEditor.Localization;
#endif

using NUnit.Framework;

using Yarn.Unity;
using Yarn.Unity.Editor;

namespace Yarn.Unity.Tests
{

#if USE_UNITY_LOCALIZATION
[TestFixture]
public class UnityLocalisationTests : IPrebuildSetup, IPostBuildCleanup
{
static string TestFolderName = typeof(UnityLocalisationTests).Name;
static string AssetPath = $"Assets/{TestFolderName}/";

string[] aLines =
{
"This is the first implicit line in YarnA",
"This is the second implicit line in YarnA",
};
string[] bLines =
{
"YarnB: This is the first implict line",
"YarnB: This is the second implicit line"
};

string[] lines => aLines.Concat(bLines).ToArray();

LocalizationSettings oldSettings;

public void Setup()
{
// first we need to make a temporary folder to store all these assets
if (Directory.Exists(AssetPath) == false)
{
AssetDatabase.CreateFolder("Assets", TestFolderName);
}

// first we are assuming that there is already some localisation settings configured
// and if not then we will make some
oldSettings = LocalizationEditorSettings.ActiveLocalizationSettings;
var settings = oldSettings;
if (settings == null)
{
// we have no existing settings
// so we will need to make a localisation settings object first
settings = ScriptableObject.CreateInstance<LocalizationSettings>();
settings.name = "Test Localization Settings";
AssetDatabase.CreateAsset(settings, Path.Combine(AssetPath, "settings.asset"));

// setting this new settings object to be th global settings for the project
AssetDatabase.SaveAssets();
LocalizationEditorSettings.ActiveLocalizationSettings = settings;
}

// we now have a valid settings, but we don't know if it has english locale support
var localeID = new LocaleIdentifier("en");
if (LocalizationSettings.AvailableLocales.GetLocale(localeID) == null)
{
// we don't have an english locale
// we need to make one and add it to the settings and on disk
var locale = Locale.CreateLocale(localeID);
AssetDatabase.CreateAsset(locale, Path.Combine(AssetPath, "en.asset"));
AssetDatabase.SaveAssets();

LocalizationEditorSettings.AddLocale(locale);
}
// at this point it is *highly* likely we have dirty assets, so save them.
AssetDatabase.SaveAssets();

// now we create the string table collection
var tableCollection = LocalizationEditorSettings.CreateStringTableCollection("testcollection", AssetPath);
AssetDatabase.SaveAssets();

// now to configure our projects to use the tables
CreateAndConfigureProject(aLines, "YarnA", AssetPath, "ProjectA", tableCollection);
CreateAndConfigureProject(bLines, "YarnB", AssetPath, "ProjectB", tableCollection);
}

private static void CreateAndConfigureProject(string[] lines, string yarnName, string assetPath, string projectName, StringTableCollection tableCollection)
{
List<string> framing = new List<string>
{
$"title: {yarnName}",
"---",
"==="
};
framing.InsertRange(2, lines);
string[] nodes = { string.Join("\n", framing) };

YarnProject proj;
var newPaths = YarnTestUtility.SetupYarnProject(nodes, new Compiler.Project(), assetPath, projectName, false, out proj);
var importer = AssetImporter.GetAtPath(AssetDatabase.GetAssetPath(proj)) as YarnProjectImporter;

// setting the path to use *only* the files for this project
var a = importer.GetProject();
a.SourceFilePatterns = newPaths.Select(p => $"./{Path.GetFileName(p)}");
a.SaveToFile($"{AssetPath}/{projectName}.yarnproject");

// making it use the tables we made
importer.UseUnityLocalisationSystem = true;
importer.unityLocalisationStringTableCollection = tableCollection;

// flagging it as needing save and reimport
EditorUtility.SetDirty(importer);
importer.SaveAndReimport();
}

public void Cleanup()
{
// put the old settings back
LocalizationEditorSettings.ActiveLocalizationSettings = oldSettings;

// delete the assets we made
AssetDatabase.DeleteAsset(AssetPath);
AssetDatabase.Refresh();
}

public UnityEngine.Localization.Tables.StringTable ValidateSetup()
{
var projectA = AssetImporter.GetAtPath($"{AssetPath}/ProjectA.yarnproject") as YarnProjectImporter;
Assert.NotNull(projectA);

var projectB = AssetImporter.GetAtPath($"{AssetPath}/ProjectB.yarnproject") as YarnProjectImporter;
Assert.NotNull(projectB);

// A and B use the same table so we just grab either of them
var table = projectA.unityLocalisationStringTableCollection.StringTables.First();
// and we need it to not be null
Assert.NotNull(table);

return table;
}

[Test]
public void UnityLocalisation_ImplicitStringsImportedCorrectly()
{
var table = ValidateSetup();

// and it needs to have the same number of lines as our projects have
Assert.AreEqual(table.Count(), lines.Count());

// each value in the table is one of our lines
foreach (var value in table.Values)
{
Assert.That(lines, Contains.Item(value.Value));
}
}

[Test]
public void UnityLocalisation_FormerImplictLinesAreRemovedFromStringTables()
{
var table = ValidateSetup();

var projectA = AssetImporter.GetAtPath($"{AssetPath}/ProjectA.yarnproject") as YarnProjectImporter;
// now we tag the yarn
YarnProjectUtility.AddLineTagsToFilesInYarnProject(projectA);

// and now we make sure it correctly added and removed the lines

// the number of lines shouldn't have changed
Assert.AreEqual(table.Count(), lines.Count());

// each value in the table is one of our lines
foreach (var value in table.Values)
{
Assert.That(lines, Contains.Item(value.Value));
}
}
}
#else
public class UnityLocalisationTests
{
[Test]
public void UnityLocalisation_UnityLocalisationPackageInstalled()
{
Assert.Fail("Unity Localisation package is not installed, tests cannot continue");
}
}
#endif
}
11 changes: 11 additions & 0 deletions Tests/Editor/UnityLocalisationTests.cs.meta

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit b2007b4

Please sign in to comment.