From 55d6bd08a87b8ee0ba737c0b59393e64d9241dd6 Mon Sep 17 00:00:00 2001 From: NathanKell Date: Wed, 24 Aug 2022 23:57:40 -0700 Subject: [PATCH 1/8] Rewrite PreFormatConfig to iterate through strings as little as possible. --- GameData/KSPCommunityFixes/Settings.cfg | 10 + KSPCommunityFixes/KSPCommunityFixes.csproj | 1 + .../Performance/ConfigNodePerf.cs | 442 ++++++++++++++++++ 3 files changed, 453 insertions(+) create mode 100644 KSPCommunityFixes/Performance/ConfigNodePerf.cs diff --git a/GameData/KSPCommunityFixes/Settings.cfg b/GameData/KSPCommunityFixes/Settings.cfg index 81bec14..1854cc0 100644 --- a/GameData/KSPCommunityFixes/Settings.cfg +++ b/GameData/KSPCommunityFixes/Settings.cfg @@ -287,6 +287,16 @@ KSP_COMMUNITY_FIXES // and every time you delete a vessel in the Tracking Station FewerSaves = false + ConfigNodePerf = false + CONFIGNODE_PERF_SKIP_PROCESSING_KEYS + { + item = serialized_plugin + } + CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS + { + item = __skipProc__ + } + // ########################## // Modding // ########################## diff --git a/KSPCommunityFixes/KSPCommunityFixes.csproj b/KSPCommunityFixes/KSPCommunityFixes.csproj index 6b10612..8f22b5f 100644 --- a/KSPCommunityFixes/KSPCommunityFixes.csproj +++ b/KSPCommunityFixes/KSPCommunityFixes.csproj @@ -129,6 +129,7 @@ + diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs new file mode 100644 index 0000000..2795be5 --- /dev/null +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -0,0 +1,442 @@ +// - Add support for IConfigNode serialization +// - Add support for Guid serialization + +using HarmonyLib; +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.InteropServices; +using UnityEngine; +using static ConfigNode; +using Random = System.Random; + +namespace KSPCommunityFixes.Modding +{ + class ConfigNodePerf : BasePatch + { + static string[] _skipKeys; + static string[] _skipPrefixes; + static bool _valid = false; + + protected override Version VersionMin => new Version(1, 8, 0); + + protected override void ApplyPatches(List patches) + { + ConfigNode settingsNodeKeys = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_KEYS"); + ConfigNode settingsNodePrefixes = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS"); + + if (settingsNodeKeys != null) + { + _skipKeys = new string[settingsNodeKeys.values.Count]; + int i = 0; + foreach (ConfigNode.Value v in settingsNodeKeys.values) + _skipKeys[i++] = v.value; + } + else + _skipKeys = new string[0]; + + if (settingsNodePrefixes != null) + { + _skipPrefixes = new string[settingsNodePrefixes.values.Count]; + int i = 0; + foreach (ConfigNode.Value v in settingsNodePrefixes.values) + _skipPrefixes[i++] = v.value; + } + else + _skipPrefixes = new string[0]; + + _valid = _skipKeys.Length > 0 || _skipPrefixes.Length > 0; + + patches.Add(new PatchInfo( + PatchMethodType.Prefix, + AccessTools.Method(typeof(ConfigNode), nameof(PreFormatConfig)), + this)); + } + + // This will fail if nested, so we cache off the old writeLinks. + private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List __result) + { + __result = _PreFormatConfig(cfgData); +#if DEBUG_CONFIGNODE_PERF + var old = OldPreFormat(cfgData); + string str = string.Empty; + if (__result.Count != old.Count) + { + str = $"\nMismatch in length! Ours {__result.Count}, old {old.Count}"; + for (int i = 0; i < old.Count; ++i) + { + int rL = __result[i].Length; + int oL = old[i].Length; + str += $"\nLine {i}:"; + for (int j = 0; j < Math.Max(rL, oL); ++j) + { + str += "\n"; + str += j < rL ? __result[i][j] : "<>"; + str += " |||| "; + str += j < oL ? old[i][j] : "<>"; + } + } + } + else + { + for (int i = 0; i < old.Count; ++i) + { + int rL = __result[i].Length; + int oL = old[i].Length; + if (rL != oL) + { + str += $"\nLength mismatch on line {i}. Dumping:"; + for (int j = 0; j < Math.Max(rL, oL); ++j) + { + str += "\n"; + str += j < rL ? __result[i][j] : "<>"; + str += " |||| "; + str += j < oL ? old[i][j] : "<>"; + } + break; + } + for (int j = 0; j < rL; ++j) + { + if (__result[i][j] != old[i][j]) + { + str += $"\nValue mismatch on line {i}, value {j}:\n{__result[i][j]} |||| {old[i][j]}"; + } + } + } + } + if (str != string.Empty) + { + Debug.Log("$$$$ mismatch:" + str); + __result = old; + } +#endif + return false; + } + + private static void AddKVP(List output, string line, int keyStart, int keyLast, int valueStart, int valueLast) + { + var pair = new string[2]; + if (keyLast < keyStart) + pair[0] = string.Empty; + else + pair[0] = line.Substring(keyStart, keyLast - keyStart + 1); + + if (valueLast < valueStart) + pair[1] = string.Empty; + else + pair[1] = line.Substring(valueStart, valueLast - valueStart + 1); + + output.Add(pair); + } + + private static unsafe void ProcessValueRaw(List output, string line, char* pszLine, int start, int lineLen, int idxKeyStart, int idxKeyLast) + { + int idxValueStart = start; + for (; idxValueStart < lineLen; ++idxValueStart) + { + if (!char.IsWhiteSpace(pszLine[idxValueStart])) + break; + } + + int idxValueLast; + if (idxValueStart == lineLen) + { + // Empty value + idxValueLast = -1; + } + else + { + idxValueLast = lineLen - 1; + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + } + + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); + } + + private static unsafe void ProcessValue(List output, string line, char* pszLine, int start, int lineLen, int idxKeyStart, int idxKeyLast) + { + int idxValueStart = start; + + for (; idxValueStart < lineLen; ++idxValueStart) + { + char c = pszLine[idxValueStart]; + if (c == '{' || c == '}') + { + // There is nothing left of the brace, so add an empty value. + AddKVP(output, line, idxKeyStart, idxKeyLast, 0, -1); + // next, add the brace. + output.Add(new string[1] { c.ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxValueStart + 1, lineLen); + return; + } + + if (idxValueStart < lineLen - 1 && c == '/' && pszLine[idxValueStart + 1] == '/') + { + lineLen = idxValueStart; // ignore whatever follows + break; + } + + if (!char.IsWhiteSpace(c)) + break; + } + + int idxValueLast; + if (idxValueStart == lineLen) + { + // Empty value + idxValueLast = -1; + } + else + { + idxValueLast = idxValueStart; + for (; idxValueLast < lineLen - 1; ++idxValueLast) + { + char c = pszLine[idxValueLast]; + if (c == '{' || c == '}') + { + // Welp, we found the end of the value. + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast - 1); + // next, add the brace. + output.Add(new string[1] { c.ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxValueStart + 1, lineLen); + return; + } + + if (c == '/' && pszLine[idxValueLast + 1] == '/') + { + --idxValueLast; + break; + } + } + + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + } + + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); + } + + private static unsafe void ProcessLine(List output, string line, char* pszLine, int start, int lineLen) + { + if (start == lineLen) + return; + + // Find first nonwhitespace char + int idxKeyStart = start; + for (; idxKeyStart < lineLen; ++idxKeyStart) + { + char c = pszLine[idxKeyStart]; + if (!char.IsWhiteSpace(c)) + break; + } + + // If we didn't find any non-whitespace, or it's only a comment, bail + if (idxKeyStart == lineLen + || (idxKeyStart < lineLen - 1 && pszLine[idxKeyStart] == '/' && pszLine[idxKeyStart + 1] == '/')) + return; + + // Find equals, if it exists. We'll divert if we find a brace, and truncate if we find a comment. + int idxEquals = idxKeyStart; + int idxKeyLast = idxEquals; + for (; idxEquals < lineLen; ++idxEquals) + { + char c = pszLine[idxEquals]; + if (c == '=') + break; + + if (c == '{' || c == '}') + { + // First, process what's left of the brace, using our known first-non-whitespace char + ProcessLine(output, line, pszLine, idxKeyStart, idxEquals); + // next, add the brace. + output.Add(new string[1] { pszLine[idxEquals].ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxEquals + 1, lineLen); + return; + } + + if (idxEquals < lineLen - 1 && c == '/' && pszLine[idxEquals + 1] == '/') + { + lineLen = idxEquals; // ignore whatever follows + break; + } + if (!char.IsWhiteSpace(c)) + idxKeyLast = idxEquals; + } + if (idxEquals == lineLen) + { + // this is a nonempty line with no equals, and we've already stripped the comment and any braces. + // We know the last non-WS character index, too, so let's just add it and return. + output.Add(new string[1] { line.Substring(idxKeyStart, idxKeyLast - idxKeyStart + 1) }); + return; + } + else + { + // See if we should skip further processing. + int keyLen = idxKeyLast - idxKeyStart + 1; + if (keyLen > 0) + { + foreach (string s in _skipKeys) + { + if (keyLen != s.Length) + continue; + + bool match = true; + fixed (char* pszComp = s) + { + for (int j = 0; j < keyLen; ++j) + { + if (pszLine[idxKeyStart + j] != pszComp[j]) + { + match = false; + break; + } + } + } + + if (match) + { +#if DEBUG + Debug.Log("$$$ Skipped processing key " + s); +#endif + ProcessValueRaw(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + return; + } + } + foreach (string s in _skipPrefixes) + { + if (keyLen < s.Length) + continue; + + bool match = true; + int sLen = s.Length; + fixed (char* pszComp = s) + { + for (int j = 0; j < sLen; ++j) + { + if (pszLine[idxKeyStart + j] != pszComp[j]) + { + match = false; + break; + } + } + } + + if (match) + { +#if DEBUG + Debug.Log($"$$$ Skipped processing key matching {s} (full key is {line.Substring(idxKeyStart, idxKeyLast - idxKeyStart + 1)})"); +#endif + ProcessValueRaw(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + return; + } + } + } + ProcessValue(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + } + } + + private static unsafe List _PreFormatConfig(string[] cfgData) + { + if (cfgData != null && cfgData.Length >= 1) + { + int lineCount = cfgData.Length; + List output = new List(lineCount); + + for (int i = 0; i < lineCount; ++i) + { + fixed (char* pszLine = cfgData[i]) + { + ProcessLine(output, cfgData[i], pszLine, 0, cfgData[i].Length); + } + } + + return output; + } + Debug.LogError("Error: Empty part config file"); + return null; + } + + private static List OldPreFormat(string[] cfgData) + { + if (cfgData != null && cfgData.Length >= 1) + { + List list = new List(cfgData); + int num = list.Count; + while (--num >= 0) + { + list[num] = list[num]; + int num2; + if ((num2 = list[num].IndexOf("//")) != -1) + { + if (num2 == 0) + { + list.RemoveAt(num); + continue; + } + list[num] = list[num].Remove(num2); + } + list[num] = list[num].Trim(); + if (list[num].Length == 0) + { + list.RemoveAt(num); + } + else if ((num2 = list[num].IndexOf("}", 0)) != -1 && (num2 != 0 || list[num].Length != 1)) + { + if (num2 > 0) + { + list.Insert(num, list[num].Substring(0, num2)); + num++; + list[num] = list[num].Substring(num2); + num2 = 0; + } + if (num2 < list[num].Length - 1) + { + list.Insert(num + 1, list[num].Substring(num2 + 1)); + list[num] = "}"; + num += 2; + } + } + else if ((num2 = list[num].IndexOf("{", 0)) != -1 && (num2 != 0 || list[num].Length != 1)) + { + if (num2 > 0) + { + list.Insert(num, list[num].Substring(0, num2)); + num++; + list[num] = list[num].Substring(num2); + num2 = 0; + } + if (num2 < list[num].Length - 1) + { + list.Insert(num + 1, list[num].Substring(num2 + 1)); + list[num] = "{"; + num += 2; + } + } + } + List list2 = new List(list.Count); + int i = 0; + for (int count = list.Count; i < count; i++) + { + string[] array = CustomEqualSplit(list[i]); + if (array != null && array.Length != 0) + { + list2.Add(array); + } + } + return list2; + } + Debug.LogError("Error: Empty part config file"); + return null; + } + } +} From e198485b3630827acaec4de752e12bb8ccbc9839 Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 01:12:47 -0700 Subject: [PATCH 2/8] Improve logging, add @siimav's parallel processing bit (seems to be strictly slower even for 1m+ rows, tho) --- .../Performance/ConfigNodePerf.cs | 85 +++++++++++++++++-- KSPCommunityFixes/Properties/AssemblyInfo.cs | 4 +- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 2795be5..2ff70ae 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -1,22 +1,24 @@ -// - Add support for IConfigNode serialization -// - Add support for Guid serialization - +//#define DEBUG_CONFIGNODE_PERF using HarmonyLib; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading.Tasks; using UnityEngine; using static ConfigNode; -using Random = System.Random; +using UniLinq; +using System.Runtime.CompilerServices; namespace KSPCommunityFixes.Modding { class ConfigNodePerf : BasePatch { - static string[] _skipKeys; - static string[] _skipPrefixes; + public static string[] _skipKeys; + public static string[] _skipPrefixes; static bool _valid = false; + const int _MinLinesForParallel = 20000; protected override Version VersionMin => new Version(1, 8, 0); @@ -56,9 +58,17 @@ protected override void ApplyPatches(List patches) // This will fail if nested, so we cache off the old writeLinks. private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List __result) { +#if DEBUG_CONFIGNODE_PERF + var sw = System.Diagnostics.Stopwatch.StartNew(); +#endif __result = _PreFormatConfig(cfgData); #if DEBUG_CONFIGNODE_PERF + var ourTime = sw.ElapsedMilliseconds; + sw.Restart(); var old = OldPreFormat(cfgData); + var oldTime = sw.ElapsedMilliseconds; + Debug.Log($"%%% Ours: {ourTime}, old: {oldTime}"); + string str = string.Empty; if (__result.Count != old.Count) { @@ -113,6 +123,7 @@ private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void AddKVP(List output, string line, int keyStart, int keyLast, int valueStart, int valueLast) { var pair = new string[2]; @@ -129,6 +140,7 @@ private static void AddKVP(List output, string line, int keyStart, int output.Add(pair); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe void ProcessValueRaw(List output, string line, char* pszLine, int start, int lineLen, int idxKeyStart, int idxKeyLast) { int idxValueStart = start; @@ -351,16 +363,44 @@ private static unsafe List _PreFormatConfig(string[] cfgData) { int lineCount = cfgData.Length; List output = new List(lineCount); +#if CONFIGNODE_PERF_USE_PARALLEL + if (lineCount < _MinLinesForParallel) + { +#endif + for (int i = 0; i < lineCount; ++i) + { + fixed (char* pszLine = cfgData[i]) + { + ProcessLine(output, cfgData[i], pszLine, 0, cfgData[i].Length); + } + } + return output; +#if CONFIGNODE_PERF_USE_PARALLEL + } - for (int i = 0; i < lineCount; ++i) + var listOfLists = new ConcurrentBag>>(); + Parallel.ForEach(Partitioner.Create(0, lineCount), range => { - fixed (char* pszLine = cfgData[i]) + int span = range.Item2 - range.Item1; + List tmpList = new List(span); + + for (int i = range.Item1; i < range.Item2; ++i) { - ProcessLine(output, cfgData[i], pszLine, 0, cfgData[i].Length); + fixed (char* pszLine = cfgData[i]) + { + ProcessLine(tmpList, cfgData[i], pszLine, 0, cfgData[i].Length); + } } + listOfLists.Add(new Tuple>(range.Item1, tmpList)); + }); + + foreach (var tuple in listOfLists.OrderBy(l => l.Item1)) + { + output.AddRange(tuple.Item2); } return output; +#endif } Debug.LogError("Error: Empty part config file"); return null; @@ -439,4 +479,31 @@ private static List OldPreFormat(string[] cfgData) return null; } } + +#if DEBUG_CONFIGNODE_PERF + [KSPAddon(KSPAddon.Startup.MainMenu, true)] + public class ConfigNodePerfTestser : MonoBehaviour + { + public void Awake() + { + // Insert your own tests here. + + //Debug.Log("(((((Loading craft, 30k lines"); + //ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); + //Debug.Log("Loading save, 1.2m lines"); + //ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); + //Debug.Log("Loading save, 364k lines w/ Principia"); + //ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + + string keys = "With keys:"; + foreach (var s in ConfigNodePerf._skipKeys) + keys += " " + s; + keys += " and prefixes"; + foreach (var s in ConfigNodePerf._skipPrefixes) + keys += " " + s; + Debug.Log(keys); + Debug.Log("end"); + } + } +#endif } diff --git a/KSPCommunityFixes/Properties/AssemblyInfo.cs b/KSPCommunityFixes/Properties/AssemblyInfo.cs index d949120..130bf90 100644 --- a/KSPCommunityFixes/Properties/AssemblyInfo.cs +++ b/KSPCommunityFixes/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("1.20.4.0")] -[assembly: AssemblyFileVersion("1.20.4.0")] +[assembly: AssemblyVersion("1.21.0.0")] +[assembly: AssemblyFileVersion("1.21.0.0")] From e4ffba6636639b959ba0142b8978b31db7712a5e Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 01:51:00 -0700 Subject: [PATCH 3/8] Patch writing, too, for a slight improvement in perf --- .../Performance/ConfigNodePerf.cs | 81 ++++++++++++++++++- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 2ff70ae..886e4d7 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -10,6 +10,7 @@ using static ConfigNode; using UniLinq; using System.Runtime.CompilerServices; +using System.IO; namespace KSPCommunityFixes.Modding { @@ -53,6 +54,11 @@ protected override void ApplyPatches(List patches) PatchMethodType.Prefix, AccessTools.Method(typeof(ConfigNode), nameof(PreFormatConfig)), this)); + + patches.Add(new PatchInfo( + PatchMethodType.Prefix, + AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.WriteNode)), + this)); } // This will fail if nested, so we cache off the old writeLinks. @@ -123,6 +129,55 @@ private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List return false; } + private static bool ConfigNode_WriteNode_Prefix(ConfigNode __instance, StreamWriter sw) + { + _WriteRootNode(__instance, sw); + return false; + } + + private static void _WriteRootNode(ConfigNode __instance, StreamWriter sw) + { + string line; + for (int i = 0, count = __instance.values.Count; i < count; ++i) + { + Value value = __instance.values[i]; + line = value.name + " = " + value.value; + if (!string.IsNullOrEmpty(value.comment)) + line += $" // {value.comment}"; + sw.WriteLine(line); + } + for (int i = 0, count = __instance.nodes.Count; i < count; ++i) + { + _WriteNodeString(__instance.nodes[i], sw, string.Empty); + } + } + + private static void _WriteNodeString(ConfigNode __instance, StreamWriter sw, string indent) + { + string line = __instance.name; + if (!string.IsNullOrEmpty(indent)) + line = indent + line; + if (!string.IsNullOrEmpty(__instance.comment)) + line += $" // {__instance.comment}"; + sw.WriteLine(line); + sw.WriteLine(indent + "{"); + string newIndent = indent + "\t"; + for (int i = 0, count = __instance.values.Count; i < count; ++i) + { + Value value = __instance.values[i]; + line = newIndent + value.name + " = " + value.value; + if (!string.IsNullOrEmpty(value.comment)) + line += $" // {value.comment}"; + sw.WriteLine(line); + } + for (int i = 0, count = __instance.nodes.Count; i < count; ++i) + { + _WriteNodeString(__instance.nodes[i], sw, newIndent); + } + sw.WriteLine(indent + "}"); + sw.Flush(); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void AddKVP(List output, string line, int keyStart, int keyLast, int valueStart, int valueLast) { @@ -489,11 +544,31 @@ public void Awake() // Insert your own tests here. //Debug.Log("(((((Loading craft, 30k lines"); - //ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); + //var craft = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); //Debug.Log("Loading save, 1.2m lines"); - //ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); + //var bigSave = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); //Debug.Log("Loading save, 364k lines w/ Principia"); - //ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + //var princSave = ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + + //var sw = System.Diagnostics.Stopwatch.StartNew(); + //bigSave.Save("c:\\temp\\t1.cfg"); + //var ours = sw.ElapsedMilliseconds; + //sw.Restart(); + //StreamWriter streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + //bigSave.WriteRootNode(streamWriter); + //streamWriter.Close(); + //var old = sw.ElapsedMilliseconds; + //Debug.Log($"%%% Big save Write perf: ours: {ours}, old: {old}"); + + //sw.Restart(); + //princSave.Save("c:\\temp\\t1.cfg"); + //ours = sw.ElapsedMilliseconds; + //sw.Restart(); + //streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + //princSave.WriteRootNode(streamWriter); + //streamWriter.Close(); + //old = sw.ElapsedMilliseconds; + //Debug.Log($"%%% Princ save Write perf: ours: {ours}, old: {old}"); string keys = "With keys:"; foreach (var s in ConfigNodePerf._skipKeys) From 522f4fe94d78db4187a61f3035bfc5c1b0b7af11 Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 06:08:19 -0700 Subject: [PATCH 4/8] Fix a lingering issue with trailing braces --- .../Performance/ConfigNodePerf.cs | 63 ++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 886e4d7..280f3df 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -19,7 +19,7 @@ class ConfigNodePerf : BasePatch public static string[] _skipKeys; public static string[] _skipPrefixes; static bool _valid = false; - const int _MinLinesForParallel = 20000; + const int _MinLinesForParallel = 100000; protected override Version VersionMin => new Version(1, 8, 0); @@ -79,10 +79,27 @@ private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List if (__result.Count != old.Count) { str = $"\nMismatch in length! Ours {__result.Count}, old {old.Count}"; - for (int i = 0; i < old.Count; ++i) + for (int i = 0; i < Math.Max(__result.Count, old.Count); ++i) { - int rL = __result[i].Length; - int oL = old[i].Length; + string[] rArr = null; + string[] oldArr = null; + if (i == __result.Count) + { + oldArr = old[i]; + rArr = new string[0]; + } + else if (i == old.Count) + { + rArr = __result[i]; + oldArr = new string[0]; + } + else + { + rArr = __result[i]; + oldArr = old[i]; + } + int rL = rArr.Length; + int oL = oldArr.Length; str += $"\nLine {i}:"; for (int j = 0; j < Math.Max(rL, oL); ++j) { @@ -261,25 +278,39 @@ private static unsafe void ProcessValue(List output, string line, char else { idxValueLast = idxValueStart; - for (; idxValueLast < lineLen - 1; ++idxValueLast) + for (; idxValueLast < lineLen; ++idxValueLast) { char c = pszLine[idxValueLast]; if (c == '{' || c == '}') { + int idxBrace = idxValueLast; + + --idxValueLast; // to get one back from the brace + // remove ws + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + // Welp, we found the end of the value. - AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast - 1); + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); // next, add the brace. output.Add(new string[1] { c.ToString() }); // hopefully this is faster than substring // finally, process the rest of the line. - ProcessLine(output, line, pszLine, idxValueStart + 1, lineLen); + ProcessLine(output, line, pszLine, idxBrace + 1, lineLen); return; } - if (c == '/' && pszLine[idxValueLast + 1] == '/') + if (idxValueLast < lineLen - 1 && c == '/' && pszLine[idxValueLast + 1] == '/') { --idxValueLast; break; } + if (idxValueLast == lineLen - 1) + { + break; + } } for (; idxValueLast > idxValueStart; --idxValueLast) @@ -424,6 +455,7 @@ private static unsafe List _PreFormatConfig(string[] cfgData) #endif for (int i = 0; i < lineCount; ++i) { + string line = cfgData[i]; fixed (char* pszLine = cfgData[i]) { ProcessLine(output, cfgData[i], pszLine, 0, cfgData[i].Length); @@ -550,6 +582,21 @@ public void Awake() //Debug.Log("Loading save, 364k lines w/ Principia"); //var princSave = ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + //Debug.Log("((((Loading tree-parts"); + //ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Tree\\TREE-Parts.cfg"); + + //Debug.Log("Loading Downrange"); + //ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Contracts\\Sounding Rockets\\DistanceSoundingDifficult.cfg"); + + //Debug.Log("Loading dictionary"); + //ConfigNode.Load("C:\\Games\\R112\\GameData\\Squad\\Localization\\dictionary.cfg"); + + //Debug.Log("Loading gravity-model"); + //ConfigNode.Load("C:\\Games\\R112\\GameData\\Principia\\real_solar_system\\gravity_model.cfg"); + + //Debug.Log("Loading mj loc fr"); + //ConfigNode.Load("C:\\Games\\R112\\GameData\\MechJeb2\\Localization\\fr-fr.cfg"); + //var sw = System.Diagnostics.Stopwatch.StartNew(); //bigSave.Save("c:\\temp\\t1.cfg"); //var ours = sw.ElapsedMilliseconds; From 3e1fa2a100d45533209c1d61c315cb623eaf3ca5 Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 12:05:06 -0700 Subject: [PATCH 5/8] Patch on startup, not after MM. Skipping keys will only happen after MM, but they're really just there for sfs loading, can't trust what's in gamedatabase enough to skip parsing there. --- .../Performance/ConfigNodePerf.cs | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 280f3df..707d8b0 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -11,20 +11,49 @@ using UniLinq; using System.Runtime.CompilerServices; using System.IO; +using System.Collections; -namespace KSPCommunityFixes.Modding +namespace KSPCommunityFixes.Performance { - class ConfigNodePerf : BasePatch + [KSPAddon(KSPAddon.Startup.Instantly, true)] + public class ConfigNodePerf : MonoBehaviour { - public static string[] _skipKeys; - public static string[] _skipPrefixes; +#if DEBUG_CONFIGNODE_PERF + public +#endif + static string[] _skipKeys; + +#if DEBUG_CONFIGNODE_PERF + public +#endif + static string[] _skipPrefixes; static bool _valid = false; const int _MinLinesForParallel = 100000; - protected override Version VersionMin => new Version(1, 8, 0); + public void Awake() + { + GameObject.DontDestroyOnLoad(this); + + Harmony harmony = new Harmony("ConfigNodePerf"); - protected override void ApplyPatches(List patches) + harmony.Patch( + AccessTools.Method(typeof(ConfigNode), nameof(PreFormatConfig)), + new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_PreFormatConfig_Prefix)))); + + harmony.Patch( + AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.WriteNode)), + new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_WriteNode_Prefix)))); + } + + public void ModuleManagerPostLoad() + { + StartCoroutine(LoadRoutine()); + } + + public IEnumerator LoadRoutine() { + yield return null; + ConfigNode settingsNodeKeys = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_KEYS"); ConfigNode settingsNodePrefixes = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS"); @@ -49,19 +78,8 @@ protected override void ApplyPatches(List patches) _skipPrefixes = new string[0]; _valid = _skipKeys.Length > 0 || _skipPrefixes.Length > 0; - - patches.Add(new PatchInfo( - PatchMethodType.Prefix, - AccessTools.Method(typeof(ConfigNode), nameof(PreFormatConfig)), - this)); - - patches.Add(new PatchInfo( - PatchMethodType.Prefix, - AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.WriteNode)), - this)); } - // This will fail if nested, so we cache off the old writeLinks. private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List __result) { #if DEBUG_CONFIGNODE_PERF @@ -381,7 +399,7 @@ private static unsafe void ProcessLine(List output, string line, char* { // See if we should skip further processing. int keyLen = idxKeyLast - idxKeyStart + 1; - if (keyLen > 0) + if (_valid && keyLen > 0) { foreach (string s in _skipKeys) { From fbca6aba10c2003318367568a5abbefa4814115c Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 12:11:25 -0700 Subject: [PATCH 6/8] Update readme and Settings --- GameData/KSPCommunityFixes/Settings.cfg | 6 ++++-- README.md | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/GameData/KSPCommunityFixes/Settings.cfg b/GameData/KSPCommunityFixes/Settings.cfg index 1854cc0..bbeee53 100644 --- a/GameData/KSPCommunityFixes/Settings.cfg +++ b/GameData/KSPCommunityFixes/Settings.cfg @@ -287,14 +287,16 @@ KSP_COMMUNITY_FIXES // and every time you delete a vessel in the Tracking Station FewerSaves = false - ConfigNodePerf = false + // These settings will make ConfigNode's loader not bother doing + // Extended checking during parsing for keys named ths CONFIGNODE_PERF_SKIP_PROCESSING_KEYS { item = serialized_plugin } + // or for keys that start with this. CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS { - item = __skipProc__ + item = **skipProc** } // ########################## diff --git a/README.md b/README.md index c6ec970..b99017d 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ User options are available from the "ESC" in-game settings menu :
Date: Thu, 25 Aug 2022 20:50:04 -0700 Subject: [PATCH 7/8] Big improvements to save perf by not writing lines / flushing / using a puny buffer. --- .../Performance/ConfigNodePerf.cs | 154 +++++++++++------- 1 file changed, 94 insertions(+), 60 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 707d8b0..1e2efdb 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -29,6 +29,8 @@ public class ConfigNodePerf : MonoBehaviour static string[] _skipPrefixes; static bool _valid = false; const int _MinLinesForParallel = 100000; + static readonly System.Text.UTF8Encoding _UTF8NoBOM = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + static readonly string _Newline = Environment.NewLine; public void Awake() { @@ -41,8 +43,8 @@ public void Awake() new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_PreFormatConfig_Prefix)))); harmony.Patch( - AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.WriteNode)), - new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_WriteNode_Prefix)))); + AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.Save), new Type[] { typeof(string), typeof(string) }), + new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_Save_Prefix)))); } public void ModuleManagerPostLoad() @@ -164,22 +166,40 @@ private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List return false; } - private static bool ConfigNode_WriteNode_Prefix(ConfigNode __instance, StreamWriter sw) + private static bool ConfigNode_Save_Prefix(ConfigNode __instance, string fileFullName, string header, ref bool __result) { - _WriteRootNode(__instance, sw); + __result = Save(__instance, fileFullName, header); return false; } + private static bool Save(ConfigNode __instance, string fileFullName, string header) + { + StreamWriter sw = new StreamWriter(File.Open(fileFullName, FileMode.Create), _UTF8NoBOM, 65536); + if (!string.IsNullOrEmpty(header)) + { + sw.Write("// "); + sw.Write(header); + sw.Write(_Newline); + } + _WriteRootNode(__instance, sw); + sw.Close(); + return true; + } + private static void _WriteRootNode(ConfigNode __instance, StreamWriter sw) { - string line; for (int i = 0, count = __instance.values.Count; i < count; ++i) { Value value = __instance.values[i]; - line = value.name + " = " + value.value; + sw.Write(value.name); + sw.Write(" = "); + sw.Write(value.value); if (!string.IsNullOrEmpty(value.comment)) - line += $" // {value.comment}"; - sw.WriteLine(line); + { + sw.Write(" // "); + sw.Write(value.comment); + } + sw.Write(_Newline); } for (int i = 0, count = __instance.nodes.Count; i < count; ++i) { @@ -189,28 +209,42 @@ private static void _WriteRootNode(ConfigNode __instance, StreamWriter sw) private static void _WriteNodeString(ConfigNode __instance, StreamWriter sw, string indent) { - string line = __instance.name; - if (!string.IsNullOrEmpty(indent)) - line = indent + line; + sw.Write(indent); + sw.Write(__instance.name); if (!string.IsNullOrEmpty(__instance.comment)) - line += $" // {__instance.comment}"; - sw.WriteLine(line); - sw.WriteLine(indent + "{"); + { + sw.Write(" // "); + sw.Write(__instance.comment); + } + sw.Write(_Newline); + + sw.Write(indent); + sw.Write("{"); + sw.Write(_Newline); + string newIndent = indent + "\t"; + for (int i = 0, count = __instance.values.Count; i < count; ++i) { Value value = __instance.values[i]; - line = newIndent + value.name + " = " + value.value; + sw.Write(newIndent); + sw.Write(value.name); + sw.Write(" = "); + sw.Write(value.value); if (!string.IsNullOrEmpty(value.comment)) - line += $" // {value.comment}"; - sw.WriteLine(line); + { + sw.Write(" // "); + sw.Write(value.comment); + } + sw.Write(_Newline); } for (int i = 0, count = __instance.nodes.Count; i < count; ++i) { _WriteNodeString(__instance.nodes[i], sw, newIndent); } - sw.WriteLine(indent + "}"); - sw.Flush(); + sw.Write(indent); + sw.Write("}"); + sw.Write(_Newline); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -593,47 +627,47 @@ public void Awake() { // Insert your own tests here. - //Debug.Log("(((((Loading craft, 30k lines"); - //var craft = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); - //Debug.Log("Loading save, 1.2m lines"); - //var bigSave = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); - //Debug.Log("Loading save, 364k lines w/ Principia"); - //var princSave = ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); - - //Debug.Log("((((Loading tree-parts"); - //ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Tree\\TREE-Parts.cfg"); - - //Debug.Log("Loading Downrange"); - //ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Contracts\\Sounding Rockets\\DistanceSoundingDifficult.cfg"); - - //Debug.Log("Loading dictionary"); - //ConfigNode.Load("C:\\Games\\R112\\GameData\\Squad\\Localization\\dictionary.cfg"); - - //Debug.Log("Loading gravity-model"); - //ConfigNode.Load("C:\\Games\\R112\\GameData\\Principia\\real_solar_system\\gravity_model.cfg"); - - //Debug.Log("Loading mj loc fr"); - //ConfigNode.Load("C:\\Games\\R112\\GameData\\MechJeb2\\Localization\\fr-fr.cfg"); - - //var sw = System.Diagnostics.Stopwatch.StartNew(); - //bigSave.Save("c:\\temp\\t1.cfg"); - //var ours = sw.ElapsedMilliseconds; - //sw.Restart(); - //StreamWriter streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); - //bigSave.WriteRootNode(streamWriter); - //streamWriter.Close(); - //var old = sw.ElapsedMilliseconds; - //Debug.Log($"%%% Big save Write perf: ours: {ours}, old: {old}"); - - //sw.Restart(); - //princSave.Save("c:\\temp\\t1.cfg"); - //ours = sw.ElapsedMilliseconds; - //sw.Restart(); - //streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); - //princSave.WriteRootNode(streamWriter); - //streamWriter.Close(); - //old = sw.ElapsedMilliseconds; - //Debug.Log($"%%% Princ save Write perf: ours: {ours}, old: {old}"); + Debug.Log("(((((Loading craft, 30k lines"); + var craft = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); + Debug.Log("Loading save, 1.2m lines"); + var bigSave = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); + Debug.Log("Loading save, 364k lines w/ Principia"); + var princSave = ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + + Debug.Log("((((Loading tree-parts"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Tree\\TREE-Parts.cfg"); + + Debug.Log("Loading Downrange"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Contracts\\Sounding Rockets\\DistanceSoundingDifficult.cfg"); + + Debug.Log("Loading dictionary"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\Squad\\Localization\\dictionary.cfg"); + + Debug.Log("Loading gravity-model"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\Principia\\real_solar_system\\gravity_model.cfg"); + + Debug.Log("Loading mj loc fr"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\MechJeb2\\Localization\\fr-fr.cfg"); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + bigSave.Save("c:\\temp\\t1.cfg"); + var ours = sw.ElapsedMilliseconds; + sw.Restart(); + StreamWriter streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + bigSave.WriteRootNode(streamWriter); + streamWriter.Close(); + var old = sw.ElapsedMilliseconds; + Debug.Log($"%%% Big save Write perf: ours: {ours}, old: {old}"); + + sw.Restart(); + princSave.Save("c:\\temp\\t1.cfg"); + ours = sw.ElapsedMilliseconds; + sw.Restart(); + streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + princSave.WriteRootNode(streamWriter); + streamWriter.Close(); + old = sw.ElapsedMilliseconds; + Debug.Log($"%%% Princ save Write perf: ours: {ours}, old: {old}"); string keys = "With keys:"; foreach (var s in ConfigNodePerf._skipKeys) From 1062b3905bbb1885f0223c7cd37567c213a5c74e Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 20:51:48 -0700 Subject: [PATCH 8/8] missed a newline --- KSPCommunityFixes/Performance/ConfigNodePerf.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index 1e2efdb..2ed6962 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -180,6 +180,7 @@ private static bool Save(ConfigNode __instance, string fileFullName, string head sw.Write("// "); sw.Write(header); sw.Write(_Newline); + sw.Write(_Newline); } _WriteRootNode(__instance, sw); sw.Close();