From a710826e0208730df3841ae9e9fcd1006d07c007 Mon Sep 17 00:00:00 2001 From: Pedro413 <44695680+Pedro413@users.noreply.github.com> Date: Thu, 23 May 2024 18:40:27 +1000 Subject: [PATCH] porting async rmt2 generation --- .../Commands/Porting/PortTagCommand.Shader.cs | 81 +++++++- TagTool/Commands/Porting/PortTagCommand.cs | 8 +- .../ShaderGenerator/ShaderGenerator.cs | 101 ++++------ .../ShaderMatching/ShaderMatcherNew.cs | 185 +++++++++++++----- 4 files changed, 256 insertions(+), 119 deletions(-) diff --git a/TagTool/Commands/Porting/PortTagCommand.Shader.cs b/TagTool/Commands/Porting/PortTagCommand.Shader.cs index 7fd1b2b4..1209eb05 100644 --- a/TagTool/Commands/Porting/PortTagCommand.Shader.cs +++ b/TagTool/Commands/Porting/PortTagCommand.Shader.cs @@ -5,11 +5,67 @@ using TagTool.Shaders.ShaderMatching; using System; using TagTool.Shaders.ShaderConverter; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Linq; +using TagTool.Commands.Common; namespace TagTool.Commands.Porting { partial class PortTagCommand { + public List PendingTemplates = new List(); + public Dictionary DeferredRenderMethods = new Dictionary(); // format: base cache tag, (blam tag, converted definition, blam cache definition) + public Dictionary TemplateConversionTasks = new Dictionary(); + + public class TemplateConversionResult + { + public RenderMethodTemplate Definition; + public PixelShader PixelShaderDefinition; + public VertexShader VertexShaderDefinition; + public CachedTag Tag; + } + + public void WaitForPendingTemplateConversion() + { + try + { + Task.WaitAll(TemplateConversionTasks.Values.ToArray()); + } + catch (AggregateException ex) + { + foreach (var inner in ex.InnerExceptions) + new TagToolError(CommandError.CustomError, inner.Message); + throw (ex); + } + } + + public void FinishConvertTemplate(TemplateConversionResult result, + string tagName, + out RenderMethodTemplate rmt2, + out PixelShader pixl, + out VertexShader vtsh) + { + var task = TemplateConversionTasks[tagName]; + TemplateConversionTasks.Remove(tagName); + + if (!task.IsFaulted) + { + rmt2 = result.Definition; + pixl = result.PixelShaderDefinition; + vtsh = result.VertexShaderDefinition; + } + else + { + // rethrow the exception + task.GetAwaiter().GetResult(); + rmt2 = null; + pixl = null; + vtsh = null; + } + } + public ShaderMatcherNew Matcher = new ShaderMatcherNew(); private RasterizerGlobals ConvertRasterizerGlobals(RasterizerGlobals rasg) @@ -21,8 +77,22 @@ private RasterizerGlobals ConvertRasterizerGlobals(RasterizerGlobals rasg) return rasg; } - private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object definition, CachedTag blamTag, object blamDefinition) + private void FinalizeRenderMethods(Stream cacheStream, Stream blamCacheStream) + { + foreach (var deferredRm in DeferredRenderMethods) + { + var definition = ConvertShaderInternal(cacheStream, blamCacheStream, (RenderMethod)deferredRm.Value.Item2, deferredRm.Value.Item1, (RenderMethod)deferredRm.Value.Item3); + + CacheContext.Serialize(cacheStream, deferredRm.Key, definition); + + if (FlagIsSet(PortingFlags.Print)) + Console.WriteLine($"['{deferredRm.Key.Group.Tag}', 0x{deferredRm.Key.Index:X4}] {deferredRm.Key.Name}.{(deferredRm.Key.Group as Cache.Gen3.TagGroupGen3).Name}"); + } + } + + private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object definition, CachedTag blamTag, object blamDefinition, CachedTag edTag, out bool isDeferred) { + isDeferred = false; switch (definition) { case ShaderFoliage rmfl: @@ -37,6 +107,13 @@ private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object case ShaderScreen rmss: case ShaderZonly rmzo: case ShaderCortana rmct: + var rmDef = (RenderMethod)definition; + if (rmDef.ShaderProperties.Count > 0 && PendingTemplates.Contains(rmDef.ShaderProperties[0].Template.Name)) + { + DeferredRenderMethods.Add(edTag, (blamTag, definition, blamDefinition)); // easier to defer and convert later + isDeferred = true; + return definition; + } return ConvertShaderInternal(cacheStream, blamCacheStream, (RenderMethod)definition, blamTag, (RenderMethod)blamDefinition); case ContrailSystem cntl: @@ -151,7 +228,7 @@ private CachedTag FindClosestRmt2(Stream cacheStream, Stream blamCacheStream, Ca { // Verify that the ShaderMatcher is ready to use if (!Matcher.IsInitialized) - Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly)); + Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, this, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly)); return Matcher.FindClosestTemplate(blamRmt2, BlamCache.Deserialize(blamCacheStream, blamRmt2), FlagIsSet(PortingFlags.GenerateShaders)); } diff --git a/TagTool/Commands/Porting/PortTagCommand.cs b/TagTool/Commands/Porting/PortTagCommand.cs index f0cf8d83..3a47522c 100644 --- a/TagTool/Commands/Porting/PortTagCommand.cs +++ b/TagTool/Commands/Porting/PortTagCommand.cs @@ -44,7 +44,7 @@ public partial class PortTagCommand : Command private readonly List ResourceTagGroups = new List { new Tag("snd!"), new Tag("bitm"), new Tag("Lbsp") }; // for null tag detection private DirectoryInfo TempDirectory { get; } = new DirectoryInfo(Path.GetTempPath()); - private BlockingCollection _deferredActions = new BlockingCollection(); + internal BlockingCollection _deferredActions = new BlockingCollection(); internal SemaphoreSlim ConcurrencyLimiter; @@ -131,7 +131,9 @@ public override object Execute(List args) WaitForPendingSoundConversion(); WaitForPendingBitmapConversion(); + WaitForPendingTemplateConversion(); ProcessDeferredActions(); + FinalizeRenderMethods(cacheStream, blamCacheStream); if (BlamCache is GameCacheGen3 gen3Cache) gen3Cache.ResourceCacheGen3.ResourcePageCache.Clear(); @@ -1392,9 +1394,9 @@ public CachedTag ConvertTagInternal(Stream cacheStream, Stream blamCacheStream, { // Verify that the ShaderMatcher is ready to use if (!Matcher.IsInitialized) - Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly)); + Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, this, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly)); - blamDefinition = ConvertShader(cacheStream, blamCacheStream, blamDefinition, blamTag, BlamCache.Deserialize(blamCacheStream, blamTag)); + blamDefinition = ConvertShader(cacheStream, blamCacheStream, blamDefinition, blamTag, BlamCache.Deserialize(blamCacheStream, blamTag), edTag, out isDeferred); if (blamDefinition == null) // convert shader failed return GetDefaultShader(blamTag.Group.Tag); } diff --git a/TagTool/Shaders/ShaderGenerator/ShaderGenerator.cs b/TagTool/Shaders/ShaderGenerator/ShaderGenerator.cs index f4de2966..a832276d 100644 --- a/TagTool/Shaders/ShaderGenerator/ShaderGenerator.cs +++ b/TagTool/Shaders/ShaderGenerator/ShaderGenerator.cs @@ -1047,6 +1047,43 @@ private static void AccumRuntimeParameterType(GameCache cache, Dictionary GatherParametersAsync(Dictionary renderMethodOptions, RenderMethodDefinition rmdf, List options, bool includeGlobal = true) + { + List allRmopParameters = new List(); + + if (includeGlobal) + { + if (rmdf.GlobalOptions != null) + { + var globalRmop = renderMethodOptions[rmdf.GlobalOptions.Name]; + allRmopParameters.AddRange(globalRmop.Parameters); + } + } + + for (int i = 0; i < rmdf.Categories.Count; i++) + { + if (rmdf.Categories[i].ShaderOptions.Count == 0) + continue; + + var option = rmdf.Categories[i].ShaderOptions[i < options.Count ? options[i] : 0]; + + if (option.Option != null) + { + var rmop = renderMethodOptions[option.Option.Name]; + + foreach (var parameter in rmop.Parameters) + { + if (allRmopParameters.Any(x => x.Name == parameter.Name)) // prevent duplicates + continue; + + allRmopParameters.Add(parameter); + } + } + } + + return allRmopParameters; + } + /// /// Non async /// @@ -1550,70 +1587,6 @@ public static GlobalVertexShader GenerateSharedVertexShaders(GameCache cache, Re return glvs; } - public static bool VerifyRmt2Routing(GameCache cache, Stream stream, RenderMethodTemplate rmt2, RenderMethodDefinition rmdf, List options) - { - bool anyMissing = false; - - var allParameters = GatherParameters(cache, stream, rmdf, options); - - var pixl = cache.Deserialize(stream, rmt2.PixelShader); - - foreach (var entry in rmdf.EntryPoints) - { - if (rmt2.EntryPoints[(int)entry.EntryPoint].Count > 0) - { - int iEnd = rmt2.EntryPoints[(int)entry.EntryPoint].Count + rmt2.EntryPoints[(int)entry.EntryPoint].Offset; - for (int i = rmt2.EntryPoints[(int)entry.EntryPoint].Offset; i < iEnd; i++) - { - var pass = rmt2.Passes[i]; - - if (pass.Values[(int)ParameterUsage.PS_Real].Count > 0) - { - foreach (var constant in pixl.Shaders[pixl.EntryPointShaders[(int)entry.EntryPoint].Offset].PCConstantTable.Constants) - { - if (constant.RegisterType != ShaderParameter.RType.Vector) - continue; - - string constantName = cache.StringTable.GetString(constant.ParameterName); - bool found = false; - - int jEnd = pass.Values[(int)ParameterUsage.PS_Real].Offset + pass.Values[(int)ParameterUsage.PS_Real].Count; - for (int j = pass.Values[(int)ParameterUsage.PS_Real].Offset; j < jEnd; j++) - { - if (rmt2.RoutingInfo[j].DestinationIndex == constant.RegisterIndex) - { - found = true; - break; - } - } - - if (!found) - { - jEnd = pass.Values[(int)ParameterUsage.PS_RealExtern].Offset + pass.Values[(int)ParameterUsage.PS_RealExtern].Count; - for (int j = pass.Values[(int)ParameterUsage.PS_RealExtern].Offset; j < jEnd; j++) - { - if (rmt2.RoutingInfo[j].DestinationIndex == constant.RegisterIndex) - { - found = true; - break; - } - } - } - - if (!found) - { - Console.WriteLine($"WARNING: {constantName} not bound in rmt2"); - anyMissing = true; - } - } - } - } - } - } - - return !anyMissing; - } - public static void GenerateExplicitShader(GameCache cache, Stream stream, string explicitShader, out PixelShader pixl, out VertexShader vtsh) { ExplicitGenerator generator = new ExplicitGenerator(); diff --git a/TagTool/Shaders/ShaderMatching/ShaderMatcherNew.cs b/TagTool/Shaders/ShaderMatching/ShaderMatcherNew.cs index f53a2df4..72653f98 100644 --- a/TagTool/Shaders/ShaderMatching/ShaderMatcherNew.cs +++ b/TagTool/Shaders/ShaderMatching/ShaderMatcherNew.cs @@ -7,6 +7,7 @@ using TagTool.Commands.Common; using TagTool.Common; using TagTool.Tags.Definitions; +using System.Threading.Tasks; namespace TagTool.Shaders.ShaderMatching { @@ -16,6 +17,13 @@ public class ShaderMatcherNew private GameCache PortingCache; private Stream BaseCacheStream; private Stream PortingCacheStream; + private Commands.Porting.PortTagCommand PortTagCommand; + // shader type, definition + private Dictionary RenderMethodDefinitions; + private Dictionary PortingRenderMethodDefinitions; + // tag name, definition + private Dictionary RenderMethodOptions; + private Dictionary PortingRenderMethodOptions; public bool IsInitialized { get; private set; } = false; public bool UseMs30 { get; set; } = false; @@ -31,7 +39,13 @@ public ShaderMatcherNew() { } - public void Init(GameCache baseCache, GameCache portingCache, Stream baseCacheStream, Stream portingCacheStream, bool useMS30 = false, bool perfectMatchesOnly = false) + public void Init(GameCache baseCache, + GameCache portingCache, + Stream baseCacheStream, + Stream portingCacheStream, + Commands.Porting.PortTagCommand portTagCommand, + bool useMS30 = false, + bool perfectMatchesOnly = false) { UseMs30 = useMS30; PerfectMatchesOnly = perfectMatchesOnly; @@ -40,6 +54,23 @@ public void Init(GameCache baseCache, GameCache portingCache, Stream baseCacheSt BaseCacheStream = baseCacheStream; PortingCacheStream = portingCacheStream; IsInitialized = true; + PortTagCommand = portTagCommand; + + // we need to store all of these for async. will save cpu time for map ports since we no longer deserialize for every shader tag + RenderMethodDefinitions = new Dictionary(); + PortingRenderMethodDefinitions = new Dictionary(); + RenderMethodOptions = new Dictionary(); + PortingRenderMethodOptions = new Dictionary(); + + foreach (var rmdfTag in baseCache.TagCache.NonNull().Where(x => x.Group.Tag == "rmdf" && !x.Name.StartsWith("ms30\\"))) + RenderMethodDefinitions.Add(rmdfTag.Name.Remove(0, 8), baseCache.Deserialize(baseCacheStream, rmdfTag)); + foreach (var rmdfTag in portingCache.TagCache.NonNull().Where(x => x.Group.Tag == "rmdf")) + PortingRenderMethodDefinitions.Add(rmdfTag.Name.Remove(0, 8), portingCache.Deserialize(portingCacheStream, rmdfTag)); + + foreach (var rmopTag in baseCache.TagCache.NonNull().Where(x => x.Group.Tag == "rmop" && !x.Name.StartsWith("ms30\\"))) + RenderMethodOptions.Add(rmopTag.Name, baseCache.Deserialize(baseCacheStream, rmopTag)); + foreach (var rmopTag in portingCache.TagCache.NonNull().Where(x => x.Group.Tag == "rmop")) + PortingRenderMethodOptions.Add(rmopTag.Name, portingCache.Deserialize(portingCacheStream, rmopTag)); } public void DeInit() @@ -51,6 +82,11 @@ public void DeInit() BaseCacheStream = null; PortingCacheStream = null; IsInitialized = false; + PortTagCommand = null; + RenderMethodDefinitions = null; + PortingRenderMethodDefinitions = null; + RenderMethodOptions = null; + PortingRenderMethodOptions = null; } public Dictionary GetOptionParameters(List options, RenderMethodDefinition rmdf) @@ -61,7 +97,7 @@ public void DeInit() { if (rmdf.Categories[i].ShaderOptions[options[i]].Option != null) { - var rmop = BaseCache.Deserialize(BaseCacheStream, rmdf.Categories[i].ShaderOptions[options[i]].Option); + var rmop = RenderMethodOptions[rmdf.Categories[i].ShaderOptions[options[i]].Option.Name]; foreach (var parameter in rmop.Parameters) if (!optionParameters.ContainsKey(parameter.Name)) optionParameters.Add(parameter.Name, parameter.Type); @@ -79,7 +115,7 @@ public void DeInit() { if (rmdf.Categories[i].ShaderOptions[options[i]].Option != null) { - var rmop = BaseCache.Deserialize(BaseCacheStream, rmdf.Categories[i].ShaderOptions[options[i]].Option); + var rmop = RenderMethodOptions[rmdf.Categories[i].ShaderOptions[options[i]].Option.Name]; foreach (var parameter in rmop.Parameters) if (!optionBlocks.ContainsKey(parameter.Name)) optionBlocks.Add(parameter.Name, parameter); @@ -97,7 +133,7 @@ public Dictionary GetOptionBitmaps(List options, Rend { if (rmdf.Categories[i].ShaderOptions[options[i]].Option != null) { - var rmop = BaseCache.Deserialize(BaseCacheStream, rmdf.Categories[i].ShaderOptions[options[i]].Option); + var rmop = RenderMethodOptions[rmdf.Categories[i].ShaderOptions[options[i]].Option.Name]; foreach (var parameter in rmop.Parameters) if (parameter.Type == RenderMethodOption.ParameterBlock.OptionDataType.Bitmap && parameter.DefaultSamplerBitmap != null && !optionBitmaps.ContainsKey(parameter.Name)) optionBitmaps.Add(parameter.Name, parameter.DefaultSamplerBitmap); @@ -121,17 +157,8 @@ public CachedTag FindClosestTemplate(CachedTag sourceRmt2Tag, RenderMethodTempla return null; } - //if (!UpdatedRmdf.Contains(sourceRmt2Desc.Type)) // will update or generate rmdf as needed - //{ - // if (!ShaderGenerator.RenderMethodDefinitionGenerator.UpdateRenderMethodDefinition(BaseCache, BaseCacheStream, sourceRmt2Desc.Type)) - // Console.WriteLine($"WARNING: rmdf for shader type \"{sourceRmt2Desc.Type}\" could not be updated or generated."); - // else - // Console.WriteLine($"Rmdf for shader type \"{sourceRmt2Desc.Type}\" updated or generated."); - // UpdatedRmdf.Add(sourceRmt2Desc.Type); - //} - // rebuild options to match base cache - sourceRmt2Desc = RebuildRmt2Options(sourceRmt2Desc, BaseCacheStream, PortingCacheStream); + sourceRmt2Desc = RebuildRmt2Options(sourceRmt2Desc); string tagName = $"shaders\\{sourceRmt2Desc.Type}_templates\\_{string.Join("_", sourceRmt2Desc.Options)}"; @@ -150,6 +177,7 @@ public CachedTag FindClosestTemplate(CachedTag sourceRmt2Tag, RenderMethodTempla ScreenSorter screenTemplateSorter = new ScreenSorter(); WaterSorter waterTemplateSorter = new WaterSorter(); + // search foreach (var rmt2Tag in BaseCache.TagCache.NonNull().Where(tag => tag.IsInGroup("rmt2"))) { Rmt2Descriptor destRmt2Desc; @@ -186,9 +214,9 @@ public CachedTag FindClosestTemplate(CachedTag sourceRmt2Tag, RenderMethodTempla // if we found an exact match, return it if (commonOptions == sourceRmt2Desc.Options.Length) { - Console.WriteLine("Found perfect rmt2 match:"); - Console.WriteLine(sourceRmt2Tag.Name); - Console.WriteLine(rmt2Tag.Name); + //Console.WriteLine("Found perfect rmt2 match:"); + //Console.WriteLine(sourceRmt2Tag.Name); + //Console.WriteLine(rmt2Tag.Name); return rmt2Tag; } @@ -242,21 +270,37 @@ public CachedTag FindClosestTemplate(CachedTag sourceRmt2Tag, RenderMethodTempla if (ShaderCache.ExportTemplate(BaseCacheStream, BaseCache, tagName, out CachedTag cachedRmt2Tag)) { - Console.WriteLine($"Found cached rmt2: {tagName}"); + if (PortTagCommand.FlagIsSet(Commands.Porting.PortTagCommand.PortingFlags.Print)) + Console.WriteLine($"['{cachedRmt2Tag.Group.Tag}', 0x{cachedRmt2Tag.Index:X4}] {cachedRmt2Tag.Name}.{(cachedRmt2Tag.Group as Cache.Gen3.TagGroupGen3).Name}"); return cachedRmt2Tag; } - // if we've reached here, we haven't found an extract match. - // now we need to consider other factors such as which options they have, which parameters are missing etc.. - // whatever can be used to narrow it down. - Console.WriteLine($"No rmt2 match found for {sourceRmt2Tag.Name}"); + // potentially async here. depends on: type (cannot be an effect type) and whether the rmt2 exists already. + if (canGenerate && TryGenerateTemplate(tagName, sourceRmt2Desc, out CachedTag generatedRmt2, (Commands.Porting.PortTagCommand.TemplateConversionResult result) => + { + PortTagCommand._deferredActions.Add(() => + { + PortTagCommand.FinishConvertTemplate(result, tagName, out RenderMethodTemplate asyncRmt2, out PixelShader asyncPixl, out VertexShader asyncVtsh); + + if (!BaseCache.TagCache.TryGetTag(tagName + ".pixl", out asyncRmt2.PixelShader)) + asyncRmt2.PixelShader = BaseCache.TagCache.AllocateTag(tagName); + if (!BaseCache.TagCache.TryGetTag(tagName + ".vtsh", out asyncRmt2.VertexShader)) + asyncRmt2.VertexShader = BaseCache.TagCache.AllocateTag(tagName); - if (canGenerate && TryGenerateTemplate(tagName, sourceRmt2Desc, out CachedTag generatedRmt2)) + BaseCache.Serialize(BaseCacheStream, asyncRmt2.PixelShader, asyncPixl); + BaseCache.Serialize(BaseCacheStream, asyncRmt2.VertexShader, asyncVtsh); + BaseCache.Serialize(BaseCacheStream, result.Tag, asyncRmt2); + + if (PortTagCommand.FlagIsSet(Commands.Porting.PortTagCommand.PortingFlags.Print)) + Console.WriteLine($"['{result.Tag.Group.Tag}', 0x{result.Tag.Index:X4}] {result.Tag.Name}.{(result.Tag.Group as Cache.Gen3.TagGroupGen3).Name}"); + }); + })) { - Console.WriteLine($"Generated rmt2: {generatedRmt2.Name}.{generatedRmt2.Group}"); return generatedRmt2; } + Console.WriteLine($"No rmt2 match found for {sourceRmt2Tag.Name}"); + if (PerfectMatchesOnly) return null; @@ -284,34 +328,36 @@ public CachedTag FindClosestTemplate(CachedTag sourceRmt2Tag, RenderMethodTempla } } - private bool TryGenerateTemplate(string tagName, Rmt2Descriptor rmt2Desc, out CachedTag generatedRmt2) + private bool CanGenerateAsync(string shaderType) { - generatedRmt2 = null; + // todo: support rmd but avoid decs + switch (shaderType) + { + case "shader": + case "custom": + case "cortana": + case "halogram": + case "glass": + case "terrain": + case "foliage": + case "water": + case "zonly": + return true; + default: + return false; + } + } - //var generator = rmt2Desc.GetGenerator(true); - //if (generator == null) - // return false; + private bool TryGenerateTemplate(string tagName, Rmt2Descriptor rmt2Desc, out CachedTag generatedRmt2, Action callback) + { + generatedRmt2 = null; - RenderMethodDefinition rmdf; - CachedTag rmdfTag; - if (!BaseCache.TagCache.TryGetTag($"shaders\\{rmt2Desc.Type}.rmdf", out rmdfTag)) + if (!RenderMethodDefinitions.ContainsKey(rmt2Desc.Type)) { new TagToolError(CommandError.CustomMessage, $"No rmdf tag present for {rmt2Desc.Type}"); return false; - - //Console.WriteLine($"Generating rmdf for \"{rmt2Desc.Type}\""); - //rmdf = ShaderGenerator.RenderMethodDefinitionGenerator.GenerateRenderMethodDefinition(BaseCache, BaseCacheStream, generator, rmt2Desc.Type, out glps, out glvs); - //rmdfTag = BaseCache.TagCache.AllocateTag($"shaders\\{rmt2Desc.Type}"); - //BaseCache.Serialize(BaseCacheStream, rmdfTag, rmdf); - //(BaseCache as GameCacheHaloOnlineBase).SaveTagNames(); - // - //rmt2Desc = RebuildRmt2Options(rmt2Desc, BaseCacheStream, PortingCacheStream); - //tagName = $"shaders\\{rmt2Desc.Type}_templates\\_{string.Join("_", rmt2Desc.Options)}"; - } - else - { - rmdf = BaseCache.Deserialize(BaseCacheStream, rmdfTag); } + RenderMethodDefinition rmdf = RenderMethodDefinitions[rmt2Desc.Type]; RenderMethodTemplate rmt2; PixelShader pixl; @@ -319,7 +365,46 @@ private bool TryGenerateTemplate(string tagName, Rmt2Descriptor rmt2Desc, out Ca try { - rmt2 = ShaderGenerator.ShaderGeneratorNew.GenerateTemplateSafe(BaseCache, BaseCacheStream, rmdf, tagName, out pixl, out vtsh); + if (CanGenerateAsync(rmt2Desc.Type)) + { + CachedTag rmt2Tag = BaseCache.TagCache.AllocateTag(tagName); + PortTagCommand.PendingTemplates.Add(tagName); + + var glps = BaseCache.Deserialize(BaseCacheStream, rmdf.GlobalPixelShader); + var glvs = BaseCache.Deserialize(BaseCacheStream, rmdf.GlobalVertexShader); + + // get options in numeric array + List options = new List(); + foreach (var option in tagName.Split('\\')[2].Remove(0, 1).Split('_')) + options.Add(byte.Parse(option)); + + var allRmopParameters = ShaderGenerator.ShaderGeneratorNew.GatherParametersAsync(RenderMethodOptions, rmdf, options); + + PortTagCommand.ConcurrencyLimiter.Wait(); + PortTagCommand.TemplateConversionTasks.Add(tagName, Task.Run(() => + { + try + { + Commands.Porting.PortTagCommand.TemplateConversionResult result = new Commands.Porting.PortTagCommand.TemplateConversionResult(); + + result.Tag = rmt2Tag; + result.Definition = ShaderGenerator.ShaderGeneratorNew.GenerateTemplate(BaseCache, rmdf, glvs, glps, allRmopParameters, tagName, out result.PixelShaderDefinition, out result.VertexShaderDefinition); + + callback(result); + } + finally + { + PortTagCommand.ConcurrencyLimiter.Release(); + } + })); + + generatedRmt2 = rmt2Tag; + return true; + } + else + { + rmt2 = ShaderGenerator.ShaderGeneratorNew.GenerateTemplateSafe(BaseCache, BaseCacheStream, rmdf, tagName, out pixl, out vtsh); + } } catch (Exception ex) { @@ -365,16 +450,16 @@ private CachedTag GetBestTag(SortingInterface sortingInterface, Dictionary /// Rebuilds an rmt2's options in memory so indices match up with the base cache /// - private Rmt2Descriptor RebuildRmt2Options(Rmt2Descriptor srcRmt2Descriptor, Stream baseStream, Stream portingStream) + private Rmt2Descriptor RebuildRmt2Options(Rmt2Descriptor srcRmt2Descriptor) { if (srcRmt2Descriptor.Type != "black" && PortingCache.Version >= CacheVersion.Halo3Beta) { string rmdfName = $"shaders\\{srcRmt2Descriptor.Type}.rmdf"; - if (!BaseCache.TagCache.TryGetTag(rmdfName, out var baseRmdfTag) || !PortingCache.TagCache.TryGetTag(rmdfName, out var portingRmdfTag)) + if (!RenderMethodDefinitions.ContainsKey(srcRmt2Descriptor.Type) || !PortingRenderMethodDefinitions.ContainsKey(srcRmt2Descriptor.Type)) return srcRmt2Descriptor; - var baseRmdfDefinition = BaseCache.Deserialize(BaseCacheStream, baseRmdfTag); - var portingRmdfDefinition = PortingCache.Deserialize(PortingCacheStream, portingRmdfTag); + var baseRmdfDefinition = RenderMethodDefinitions[srcRmt2Descriptor.Type]; + var portingRmdfDefinition = PortingRenderMethodDefinitions[srcRmt2Descriptor.Type]; List newOptions = new List();