Cache remote build map, fetch in registry refresh

Cmdline/Main.cs

@@ -49,6 +49,18 @@ public static int Main(string[] args)
             }
 
             Logging.Initialize();
+            // We need to load the game instance manager before parsing the args,
+            // which is too late for debug flags if we want them active during the instance loading
+            if (args.Contains("--debug"))
+            {
+                LogManager.GetRepository().Threshold = Level.Debug;
+                log.Info("Debug logging enabled");
+            }
+            else if (args.Contains("--verbose"))
+            {
+                LogManager.GetRepository().Threshold = Level.Info;
+                log.Info("Verbose logging enabled");
+            }
             log.Info("CKAN started.");
 
             // Force-allow TLS 1.2 for HTTPS URLs, because GitHub requires it.

Core/Games/KerbalSpaceProgram/GameVersionProviders/IKspBuildMap.cs

@@ -9,6 +9,9 @@ public interface IKspBuildMap
 
         List<GameVersion> KnownVersions { get; }
 
+        /// <summary>
+        /// Download the build map from the server to the cache
+        /// </summary>
         void Refresh();
     }
 }

Core/Games/KerbalSpaceProgram/GameVersionProviders/KspBuildMap.cs

@@ -1,9 +1,12 @@
 using System;
+using System.Linq;
 using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
+
 using log4net;
 using Newtonsoft.Json;
+
 using CKAN.Versioning;
 using CKAN.Configuration;
 
@@ -30,6 +33,9 @@ public sealed class KspBuildMap : IKspBuildMap
         // TODO: Need a way for the client to configure this
         private static readonly Uri BuildMapUri =
             new Uri("https://raw.githubusercontent.com/KSP-CKAN/CKAN-meta/master/builds.json");
+        private static readonly string cachedBuildMapPath = Path.Combine(
+            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+            "CKAN", "builds.json");
 
         private static readonly ILog Log = LogManager.GetLogger(typeof(KspBuildMap));
 
@@ -53,12 +59,9 @@ public List<GameVersion> KnownVersions
             get
             {
                 EnsureBuildMap();
-                List<GameVersion> knownVersions = new List<GameVersion>();
-                foreach (var version in _jBuilds.Builds)
-                {
-                    knownVersions.Add(GameVersion.Parse(version.Value));
-                }
-                return knownVersions;
+                return _jBuilds.Builds
+                    .Select(b => GameVersion.Parse(b.Value))
+                    .ToList();
             }
         }
 
@@ -75,21 +78,24 @@ private void EnsureBuildMap()
                 {
                     if (ReferenceEquals(_jBuilds, null))
                     {
-                        Refresh();
+                        // Check for a cached copy of the remote build map
+                        if (TrySetCachedBuildMap())   return;
+                        // If that doesn't exist, use the copy from when we were compiled
+                        if (TrySetEmbeddedBuildMap()) return;
+                        Log.Warn("Could not load build map from cached or embedded copy");
                     }
                 }
             }
         }
 
         /// <summary>
-        /// Load a build map
+        /// Download the build map from the server to the cache
         /// </summary>
         public void Refresh()
         {
-            if (TrySetRemoteBuildMap())   return;
-            if (TrySetEmbeddedBuildMap()) return;
-
-            Log.Warn("Could not refresh the build map from any source");
+            Log.Debug("Refreshing build map from server");
+            if (TrySetRemoteBuildMap()) return;
+            Log.Warn("Could not refresh the build map from remote server");
         }
 
         private bool TrySetBuildMap(string buildMapJson)
@@ -106,20 +112,28 @@ private bool TrySetBuildMap(string buildMapJson)
                 return false;
             }
         }
+
+        private bool TrySetCachedBuildMap()
+        {
+            try
+            {
+                Log.Debug("Getting cached build map");
+                return TrySetBuildMap(File.ReadAllText(cachedBuildMapPath));
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
         private bool TrySetRemoteBuildMap()
         {
             try
             {
+                Log.Debug("Getting remote build map");
                 var json = Net.DownloadText(BuildMapUri);
-
-                if (TrySetBuildMap(json))
-                {
-                    return true;
-                }
-                else
-                {
-                    return false;
-                }
+                File.WriteAllText(cachedBuildMapPath, json);
+                return TrySetBuildMap(json);
             }
             catch (Exception e)
             {
@@ -133,14 +147,15 @@ private bool TrySetEmbeddedBuildMap()
         {
             try
             {
-                var resourceStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("CKAN.builds.json");
+                Log.Debug("Getting embedded build map");
+                var resourceStream = Assembly.GetExecutingAssembly()
+                    .GetManifestResourceStream("CKAN.builds.json");
 
                 if (resourceStream != null)
                 {
                     using (var reader = new StreamReader(resourceStream))
                     {
-                        TrySetBuildMap(reader.ReadToEnd());
-                        return true;
+                        return TrySetBuildMap(reader.ReadToEnd());
                     }
                 }
                 else

Core/Net/Repo.cs

@@ -37,6 +37,11 @@ public static class Repo
         public static RepoUpdateResult UpdateAllRepositories(RegistryManager registry_manager, GameInstance ksp, NetModuleCache cache, IUser user)
         {
             SortedDictionary<string, Repository> sortedRepositories = registry_manager.registry.Repositories;
+
+            // First get latest copy of the game versions data
+            user.RaiseMessage(Properties.Resources.NetRepoUpdatingBuildMap);
+            ServiceLocator.Container.Resolve<IKspBuildMap>().Refresh();
+
             user.RaiseProgress(Properties.Resources.NetRepoCheckingForUpdates, 0);
             if (sortedRepositories.Values.All(repo => !string.IsNullOrEmpty(repo.last_server_etag) && repo.last_server_etag == Net.CurrentETag(repo.uri)))
             {

Core/Properties/Resources.resx

@@ -136,6 +136,7 @@ Install the `mono-complete` package or equivalent for your operating system.</va
   <data name="NetModuleCacheNotValidZIP" xml:space="preserve"><value>{0}: {1} is not a valid ZIP file: {2}</value></data>
   <data name="NetModuleCacheMismatchSHA1" xml:space="preserve"><value>{0}: {1} has SHA1 {2}, should be {3}</value></data>
   <data name="NetModuleCacheMismatchSHA256" xml:space="preserve"><value>{0}: {1} has SHA256 {2}, should be {3}</value></data>
+  <data name="NetRepoUpdatingBuildMap" xml:space="preserve"><value>Refreshing game version data</value></data>
   <data name="NetRepoCheckingForUpdates" xml:space="preserve"><value>Checking for updates</value></data>
   <data name="NetRepoAlreadyUpToDate" xml:space="preserve"><value>Already up to date</value></data>
   <data name="NetRepoNoChanges" xml:space="preserve"><value>No changes since last update</value></data>