From fb90bee5874b534a1ddccf309ff3a6266860fb25 Mon Sep 17 00:00:00 2001 From: NathanKell Date: Thu, 25 Aug 2022 21:02:06 -0700 Subject: [PATCH] Improve ConfigNode load/save perf (#88) * Rewrite PreFormatConfig to iterate through strings as little as possible. * Improve logging, add @siimav's parallel processing bit (seems to be strictly slower even for 1m+ rows, tho) * Patch writing, too, for a slight improvement in perf * Fix a lingering issue with trailing braces * Patch on startup, not after MM. Skipping keys will only happen after MM, but they're really just there for sfs loading, can't trust what's in gamedatabase enough to skip parsing there. * Update readme and Settings * Big improvements to save perf by not writing lines / flushing / using a puny buffer. * missed a newline --- GameData/KSPCommunityFixes/Settings.cfg | 12 + KSPCommunityFixes/KSPCommunityFixes.csproj | 1 + .../Performance/ConfigNodePerf.cs | 684 ++++++++++++++++++ KSPCommunityFixes/Properties/AssemblyInfo.cs | 4 +- README.md | 2 + 5 files changed, 701 insertions(+), 2 deletions(-) create mode 100644 KSPCommunityFixes/Performance/ConfigNodePerf.cs diff --git a/GameData/KSPCommunityFixes/Settings.cfg b/GameData/KSPCommunityFixes/Settings.cfg index 81bec14..bbeee53 100644 --- a/GameData/KSPCommunityFixes/Settings.cfg +++ b/GameData/KSPCommunityFixes/Settings.cfg @@ -287,6 +287,18 @@ KSP_COMMUNITY_FIXES // and every time you delete a vessel in the Tracking Station FewerSaves = false + // These settings will make ConfigNode's loader not bother doing + // Extended checking during parsing for keys named ths + CONFIGNODE_PERF_SKIP_PROCESSING_KEYS + { + item = serialized_plugin + } + // or for keys that start with this. + CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS + { + item = **skipProc** + } + // ########################## // Modding // ########################## diff --git a/KSPCommunityFixes/KSPCommunityFixes.csproj b/KSPCommunityFixes/KSPCommunityFixes.csproj index 6b10612..8f22b5f 100644 --- a/KSPCommunityFixes/KSPCommunityFixes.csproj +++ b/KSPCommunityFixes/KSPCommunityFixes.csproj @@ -129,6 +129,7 @@ + diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs new file mode 100644 index 0000000..2ed6962 --- /dev/null +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -0,0 +1,684 @@ +//#define DEBUG_CONFIGNODE_PERF +using HarmonyLib; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using UnityEngine; +using static ConfigNode; +using UniLinq; +using System.Runtime.CompilerServices; +using System.IO; +using System.Collections; + +namespace KSPCommunityFixes.Performance +{ + [KSPAddon(KSPAddon.Startup.Instantly, true)] + public class ConfigNodePerf : MonoBehaviour + { +#if DEBUG_CONFIGNODE_PERF + public +#endif + static string[] _skipKeys; + +#if DEBUG_CONFIGNODE_PERF + public +#endif + static string[] _skipPrefixes; + static bool _valid = false; + const int _MinLinesForParallel = 100000; + static readonly System.Text.UTF8Encoding _UTF8NoBOM = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + static readonly string _Newline = Environment.NewLine; + + public void Awake() + { + GameObject.DontDestroyOnLoad(this); + + Harmony harmony = new Harmony("ConfigNodePerf"); + + harmony.Patch( + AccessTools.Method(typeof(ConfigNode), nameof(PreFormatConfig)), + new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_PreFormatConfig_Prefix)))); + + harmony.Patch( + AccessTools.Method(typeof(ConfigNode), nameof(ConfigNode.Save), new Type[] { typeof(string), typeof(string) }), + new HarmonyMethod(AccessTools.Method(typeof(ConfigNodePerf), nameof(ConfigNodePerf.ConfigNode_Save_Prefix)))); + } + + public void ModuleManagerPostLoad() + { + StartCoroutine(LoadRoutine()); + } + + public IEnumerator LoadRoutine() + { + yield return null; + + ConfigNode settingsNodeKeys = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_KEYS"); + ConfigNode settingsNodePrefixes = KSPCommunityFixes.SettingsNode.GetNode("CONFIGNODE_PERF_SKIP_PROCESSING_SUBSTRINGS"); + + if (settingsNodeKeys != null) + { + _skipKeys = new string[settingsNodeKeys.values.Count]; + int i = 0; + foreach (ConfigNode.Value v in settingsNodeKeys.values) + _skipKeys[i++] = v.value; + } + else + _skipKeys = new string[0]; + + if (settingsNodePrefixes != null) + { + _skipPrefixes = new string[settingsNodePrefixes.values.Count]; + int i = 0; + foreach (ConfigNode.Value v in settingsNodePrefixes.values) + _skipPrefixes[i++] = v.value; + } + else + _skipPrefixes = new string[0]; + + _valid = _skipKeys.Length > 0 || _skipPrefixes.Length > 0; + } + + private static bool ConfigNode_PreFormatConfig_Prefix(string[] cfgData, ref List __result) + { +#if DEBUG_CONFIGNODE_PERF + var sw = System.Diagnostics.Stopwatch.StartNew(); +#endif + __result = _PreFormatConfig(cfgData); +#if DEBUG_CONFIGNODE_PERF + var ourTime = sw.ElapsedMilliseconds; + sw.Restart(); + var old = OldPreFormat(cfgData); + var oldTime = sw.ElapsedMilliseconds; + Debug.Log($"%%% Ours: {ourTime}, old: {oldTime}"); + + string str = string.Empty; + if (__result.Count != old.Count) + { + str = $"\nMismatch in length! Ours {__result.Count}, old {old.Count}"; + for (int i = 0; i < Math.Max(__result.Count, old.Count); ++i) + { + string[] rArr = null; + string[] oldArr = null; + if (i == __result.Count) + { + oldArr = old[i]; + rArr = new string[0]; + } + else if (i == old.Count) + { + rArr = __result[i]; + oldArr = new string[0]; + } + else + { + rArr = __result[i]; + oldArr = old[i]; + } + int rL = rArr.Length; + int oL = oldArr.Length; + str += $"\nLine {i}:"; + for (int j = 0; j < Math.Max(rL, oL); ++j) + { + str += "\n"; + str += j < rL ? __result[i][j] : "<>"; + str += " |||| "; + str += j < oL ? old[i][j] : "<>"; + } + } + } + else + { + for (int i = 0; i < old.Count; ++i) + { + int rL = __result[i].Length; + int oL = old[i].Length; + if (rL != oL) + { + str += $"\nLength mismatch on line {i}. Dumping:"; + for (int j = 0; j < Math.Max(rL, oL); ++j) + { + str += "\n"; + str += j < rL ? __result[i][j] : "<>"; + str += " |||| "; + str += j < oL ? old[i][j] : "<>"; + } + break; + } + for (int j = 0; j < rL; ++j) + { + if (__result[i][j] != old[i][j]) + { + str += $"\nValue mismatch on line {i}, value {j}:\n{__result[i][j]} |||| {old[i][j]}"; + } + } + } + } + if (str != string.Empty) + { + Debug.Log("$$$$ mismatch:" + str); + __result = old; + } +#endif + return false; + } + + private static bool ConfigNode_Save_Prefix(ConfigNode __instance, string fileFullName, string header, ref bool __result) + { + __result = Save(__instance, fileFullName, header); + return false; + } + + private static bool Save(ConfigNode __instance, string fileFullName, string header) + { + StreamWriter sw = new StreamWriter(File.Open(fileFullName, FileMode.Create), _UTF8NoBOM, 65536); + if (!string.IsNullOrEmpty(header)) + { + sw.Write("// "); + sw.Write(header); + sw.Write(_Newline); + sw.Write(_Newline); + } + _WriteRootNode(__instance, sw); + sw.Close(); + return true; + } + + private static void _WriteRootNode(ConfigNode __instance, StreamWriter sw) + { + for (int i = 0, count = __instance.values.Count; i < count; ++i) + { + Value value = __instance.values[i]; + sw.Write(value.name); + sw.Write(" = "); + sw.Write(value.value); + if (!string.IsNullOrEmpty(value.comment)) + { + sw.Write(" // "); + sw.Write(value.comment); + } + sw.Write(_Newline); + } + for (int i = 0, count = __instance.nodes.Count; i < count; ++i) + { + _WriteNodeString(__instance.nodes[i], sw, string.Empty); + } + } + + private static void _WriteNodeString(ConfigNode __instance, StreamWriter sw, string indent) + { + sw.Write(indent); + sw.Write(__instance.name); + if (!string.IsNullOrEmpty(__instance.comment)) + { + sw.Write(" // "); + sw.Write(__instance.comment); + } + sw.Write(_Newline); + + sw.Write(indent); + sw.Write("{"); + sw.Write(_Newline); + + string newIndent = indent + "\t"; + + for (int i = 0, count = __instance.values.Count; i < count; ++i) + { + Value value = __instance.values[i]; + sw.Write(newIndent); + sw.Write(value.name); + sw.Write(" = "); + sw.Write(value.value); + if (!string.IsNullOrEmpty(value.comment)) + { + sw.Write(" // "); + sw.Write(value.comment); + } + sw.Write(_Newline); + } + for (int i = 0, count = __instance.nodes.Count; i < count; ++i) + { + _WriteNodeString(__instance.nodes[i], sw, newIndent); + } + sw.Write(indent); + sw.Write("}"); + sw.Write(_Newline); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void AddKVP(List output, string line, int keyStart, int keyLast, int valueStart, int valueLast) + { + var pair = new string[2]; + if (keyLast < keyStart) + pair[0] = string.Empty; + else + pair[0] = line.Substring(keyStart, keyLast - keyStart + 1); + + if (valueLast < valueStart) + pair[1] = string.Empty; + else + pair[1] = line.Substring(valueStart, valueLast - valueStart + 1); + + output.Add(pair); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void ProcessValueRaw(List output, string line, char* pszLine, int start, int lineLen, int idxKeyStart, int idxKeyLast) + { + int idxValueStart = start; + for (; idxValueStart < lineLen; ++idxValueStart) + { + if (!char.IsWhiteSpace(pszLine[idxValueStart])) + break; + } + + int idxValueLast; + if (idxValueStart == lineLen) + { + // Empty value + idxValueLast = -1; + } + else + { + idxValueLast = lineLen - 1; + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + } + + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); + } + + private static unsafe void ProcessValue(List output, string line, char* pszLine, int start, int lineLen, int idxKeyStart, int idxKeyLast) + { + int idxValueStart = start; + + for (; idxValueStart < lineLen; ++idxValueStart) + { + char c = pszLine[idxValueStart]; + if (c == '{' || c == '}') + { + // There is nothing left of the brace, so add an empty value. + AddKVP(output, line, idxKeyStart, idxKeyLast, 0, -1); + // next, add the brace. + output.Add(new string[1] { c.ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxValueStart + 1, lineLen); + return; + } + + if (idxValueStart < lineLen - 1 && c == '/' && pszLine[idxValueStart + 1] == '/') + { + lineLen = idxValueStart; // ignore whatever follows + break; + } + + if (!char.IsWhiteSpace(c)) + break; + } + + int idxValueLast; + if (idxValueStart == lineLen) + { + // Empty value + idxValueLast = -1; + } + else + { + idxValueLast = idxValueStart; + for (; idxValueLast < lineLen; ++idxValueLast) + { + char c = pszLine[idxValueLast]; + if (c == '{' || c == '}') + { + int idxBrace = idxValueLast; + + --idxValueLast; // to get one back from the brace + // remove ws + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + + // Welp, we found the end of the value. + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); + // next, add the brace. + output.Add(new string[1] { c.ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxBrace + 1, lineLen); + return; + } + + if (idxValueLast < lineLen - 1 && c == '/' && pszLine[idxValueLast + 1] == '/') + { + --idxValueLast; + break; + } + if (idxValueLast == lineLen - 1) + { + break; + } + } + + for (; idxValueLast > idxValueStart; --idxValueLast) + { + if (!char.IsWhiteSpace(pszLine[idxValueLast])) + break; + } + } + + AddKVP(output, line, idxKeyStart, idxKeyLast, idxValueStart, idxValueLast); + } + + private static unsafe void ProcessLine(List output, string line, char* pszLine, int start, int lineLen) + { + if (start == lineLen) + return; + + // Find first nonwhitespace char + int idxKeyStart = start; + for (; idxKeyStart < lineLen; ++idxKeyStart) + { + char c = pszLine[idxKeyStart]; + if (!char.IsWhiteSpace(c)) + break; + } + + // If we didn't find any non-whitespace, or it's only a comment, bail + if (idxKeyStart == lineLen + || (idxKeyStart < lineLen - 1 && pszLine[idxKeyStart] == '/' && pszLine[idxKeyStart + 1] == '/')) + return; + + // Find equals, if it exists. We'll divert if we find a brace, and truncate if we find a comment. + int idxEquals = idxKeyStart; + int idxKeyLast = idxEquals; + for (; idxEquals < lineLen; ++idxEquals) + { + char c = pszLine[idxEquals]; + if (c == '=') + break; + + if (c == '{' || c == '}') + { + // First, process what's left of the brace, using our known first-non-whitespace char + ProcessLine(output, line, pszLine, idxKeyStart, idxEquals); + // next, add the brace. + output.Add(new string[1] { pszLine[idxEquals].ToString() }); // hopefully this is faster than substring + // finally, process the rest of the line. + ProcessLine(output, line, pszLine, idxEquals + 1, lineLen); + return; + } + + if (idxEquals < lineLen - 1 && c == '/' && pszLine[idxEquals + 1] == '/') + { + lineLen = idxEquals; // ignore whatever follows + break; + } + if (!char.IsWhiteSpace(c)) + idxKeyLast = idxEquals; + } + if (idxEquals == lineLen) + { + // this is a nonempty line with no equals, and we've already stripped the comment and any braces. + // We know the last non-WS character index, too, so let's just add it and return. + output.Add(new string[1] { line.Substring(idxKeyStart, idxKeyLast - idxKeyStart + 1) }); + return; + } + else + { + // See if we should skip further processing. + int keyLen = idxKeyLast - idxKeyStart + 1; + if (_valid && keyLen > 0) + { + foreach (string s in _skipKeys) + { + if (keyLen != s.Length) + continue; + + bool match = true; + fixed (char* pszComp = s) + { + for (int j = 0; j < keyLen; ++j) + { + if (pszLine[idxKeyStart + j] != pszComp[j]) + { + match = false; + break; + } + } + } + + if (match) + { +#if DEBUG + Debug.Log("$$$ Skipped processing key " + s); +#endif + ProcessValueRaw(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + return; + } + } + foreach (string s in _skipPrefixes) + { + if (keyLen < s.Length) + continue; + + bool match = true; + int sLen = s.Length; + fixed (char* pszComp = s) + { + for (int j = 0; j < sLen; ++j) + { + if (pszLine[idxKeyStart + j] != pszComp[j]) + { + match = false; + break; + } + } + } + + if (match) + { +#if DEBUG + Debug.Log($"$$$ Skipped processing key matching {s} (full key is {line.Substring(idxKeyStart, idxKeyLast - idxKeyStart + 1)})"); +#endif + ProcessValueRaw(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + return; + } + } + } + ProcessValue(output, line, pszLine, idxEquals + 1, lineLen, idxKeyStart, idxKeyLast); + } + } + + private static unsafe List _PreFormatConfig(string[] cfgData) + { + if (cfgData != null && cfgData.Length >= 1) + { + int lineCount = cfgData.Length; + List output = new List(lineCount); +#if CONFIGNODE_PERF_USE_PARALLEL + if (lineCount < _MinLinesForParallel) + { +#endif + for (int i = 0; i < lineCount; ++i) + { + string line = cfgData[i]; + fixed (char* pszLine = cfgData[i]) + { + ProcessLine(output, cfgData[i], pszLine, 0, cfgData[i].Length); + } + } + return output; +#if CONFIGNODE_PERF_USE_PARALLEL + } + + var listOfLists = new ConcurrentBag>>(); + Parallel.ForEach(Partitioner.Create(0, lineCount), range => + { + int span = range.Item2 - range.Item1; + List tmpList = new List(span); + + for (int i = range.Item1; i < range.Item2; ++i) + { + fixed (char* pszLine = cfgData[i]) + { + ProcessLine(tmpList, cfgData[i], pszLine, 0, cfgData[i].Length); + } + } + listOfLists.Add(new Tuple>(range.Item1, tmpList)); + }); + + foreach (var tuple in listOfLists.OrderBy(l => l.Item1)) + { + output.AddRange(tuple.Item2); + } + + return output; +#endif + } + Debug.LogError("Error: Empty part config file"); + return null; + } + + private static List OldPreFormat(string[] cfgData) + { + if (cfgData != null && cfgData.Length >= 1) + { + List list = new List(cfgData); + int num = list.Count; + while (--num >= 0) + { + list[num] = list[num]; + int num2; + if ((num2 = list[num].IndexOf("//")) != -1) + { + if (num2 == 0) + { + list.RemoveAt(num); + continue; + } + list[num] = list[num].Remove(num2); + } + list[num] = list[num].Trim(); + if (list[num].Length == 0) + { + list.RemoveAt(num); + } + else if ((num2 = list[num].IndexOf("}", 0)) != -1 && (num2 != 0 || list[num].Length != 1)) + { + if (num2 > 0) + { + list.Insert(num, list[num].Substring(0, num2)); + num++; + list[num] = list[num].Substring(num2); + num2 = 0; + } + if (num2 < list[num].Length - 1) + { + list.Insert(num + 1, list[num].Substring(num2 + 1)); + list[num] = "}"; + num += 2; + } + } + else if ((num2 = list[num].IndexOf("{", 0)) != -1 && (num2 != 0 || list[num].Length != 1)) + { + if (num2 > 0) + { + list.Insert(num, list[num].Substring(0, num2)); + num++; + list[num] = list[num].Substring(num2); + num2 = 0; + } + if (num2 < list[num].Length - 1) + { + list.Insert(num + 1, list[num].Substring(num2 + 1)); + list[num] = "{"; + num += 2; + } + } + } + List list2 = new List(list.Count); + int i = 0; + for (int count = list.Count; i < count; i++) + { + string[] array = CustomEqualSplit(list[i]); + if (array != null && array.Length != 0) + { + list2.Add(array); + } + } + return list2; + } + Debug.LogError("Error: Empty part config file"); + return null; + } + } + +#if DEBUG_CONFIGNODE_PERF + [KSPAddon(KSPAddon.Startup.MainMenu, true)] + public class ConfigNodePerfTestser : MonoBehaviour + { + public void Awake() + { + // Insert your own tests here. + + Debug.Log("(((((Loading craft, 30k lines"); + var craft = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\Ships\\VAB\\Dolphin Lunar Orbital.craft"); + Debug.Log("Loading save, 1.2m lines"); + var bigSave = ConfigNode.Load("C:\\Games\\R112\\saves\\Hard\\persistent.sfs"); + Debug.Log("Loading save, 364k lines w/ Principia"); + var princSave = ConfigNode.Load("C:\\Games\\R112\\saves\\lcdev\\persistent.sfs"); + + Debug.Log("((((Loading tree-parts"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Tree\\TREE-Parts.cfg"); + + Debug.Log("Loading Downrange"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\RP-0\\Contracts\\Sounding Rockets\\DistanceSoundingDifficult.cfg"); + + Debug.Log("Loading dictionary"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\Squad\\Localization\\dictionary.cfg"); + + Debug.Log("Loading gravity-model"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\Principia\\real_solar_system\\gravity_model.cfg"); + + Debug.Log("Loading mj loc fr"); + ConfigNode.Load("C:\\Games\\R112\\GameData\\MechJeb2\\Localization\\fr-fr.cfg"); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + bigSave.Save("c:\\temp\\t1.cfg"); + var ours = sw.ElapsedMilliseconds; + sw.Restart(); + StreamWriter streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + bigSave.WriteRootNode(streamWriter); + streamWriter.Close(); + var old = sw.ElapsedMilliseconds; + Debug.Log($"%%% Big save Write perf: ours: {ours}, old: {old}"); + + sw.Restart(); + princSave.Save("c:\\temp\\t1.cfg"); + ours = sw.ElapsedMilliseconds; + sw.Restart(); + streamWriter = new StreamWriter(File.Open("c:\\temp\\t2.cfg", FileMode.Create)); + princSave.WriteRootNode(streamWriter); + streamWriter.Close(); + old = sw.ElapsedMilliseconds; + Debug.Log($"%%% Princ save Write perf: ours: {ours}, old: {old}"); + + string keys = "With keys:"; + foreach (var s in ConfigNodePerf._skipKeys) + keys += " " + s; + keys += " and prefixes"; + foreach (var s in ConfigNodePerf._skipPrefixes) + keys += " " + s; + Debug.Log(keys); + Debug.Log("end"); + } + } +#endif +} diff --git a/KSPCommunityFixes/Properties/AssemblyInfo.cs b/KSPCommunityFixes/Properties/AssemblyInfo.cs index d949120..130bf90 100644 --- a/KSPCommunityFixes/Properties/AssemblyInfo.cs +++ b/KSPCommunityFixes/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("1.20.4.0")] -[assembly: AssemblyFileVersion("1.20.4.0")] +[assembly: AssemblyVersion("1.21.0.0")] +[assembly: AssemblyFileVersion("1.21.0.0")] diff --git a/README.md b/README.md index c6ec970..b99017d 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ User options are available from the "ESC" in-game settings menu :