Accept header and infrastructure for auth tokens

Core/Net/AutoUpdate.cs

@@ -144,10 +144,14 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null)
             // download updater app and new ckan.exe
             string updaterFilename = System.IO.Path.GetTempPath() + Guid.NewGuid().ToString() + ".exe";
             string ckanFilename = System.IO.Path.GetTempPath() + Guid.NewGuid().ToString() + ".exe";
-            Net.DownloadWithProgress(new[]{
-                new Net.DownloadTarget(fetchedUpdaterUrl.Item1, updaterFilename, fetchedUpdaterUrl.Item2),
-                new Net.DownloadTarget(fetchedCkanUrl.Item1, ckanFilename, fetchedCkanUrl.Item2),
-            }, user);
+            Net.DownloadWithProgress(
+                new[]
+                {
+                    new Net.DownloadTarget(fetchedUpdaterUrl.Item1, updaterFilename, fetchedUpdaterUrl.Item2),
+                    new Net.DownloadTarget(fetchedCkanUrl.Item1,    ckanFilename,    fetchedCkanUrl.Item2),
+                },
+                user
+            );
 
             // run updater
             SetExecutable(updaterFilename);

Core/Net/Net.cs

@@ -17,6 +17,7 @@ namespace CKAN
 
     public class Net
     {
+        // The user agent that we report to web sites
         public static string UserAgentString = "Mozilla/4.0 (compatible; CKAN)";
 
         private static readonly ILog Log = LogManager.GetLogger(typeof (Net));
@@ -112,20 +113,19 @@ public static string Download(string url, string filename = null, IUser user = n
 
         public class DownloadTarget
         {
-            public Uri uri { get; private set; }
+            public Uri    url      { get; private set; }
             public string filename { get; private set; }
-            public long size { get; private set; }
+            public long   size     { get; private set; }
+            public string mimeType { get; private set; }
 
-            public DownloadTarget(Uri uri, string filename = null, long size = 0)
+            public DownloadTarget(Uri url, string filename = null, long size = 0, string mimeType = "")
             {
-                if (filename == null)
-                {
-                    filename = FileTransaction.GetTempFileName();
-                }
-
-                this.uri = uri;
-                this.filename = filename;
-                this.size = size;
+                this.url      = url;
+                this.filename = string.IsNullOrEmpty(filename)
+                    ? FileTransaction.GetTempFileName()
+                    : filename;
+                this.size     = size;
+                this.mimeType = mimeType;
             }
         }
 
@@ -150,10 +150,10 @@ public static void DownloadWithProgress(ICollection<DownloadTarget> downloadTarg
                     if (filenames == null || urls == null) return;
                     for (var i = 0; i < Math.Min(urls.Length, filenames.Length); i++)
                     {
-                        File.Move(filenames[i], downloadTargets.First(p => p.uri == urls[i]).filename);
+                        File.Move(filenames[i], downloadTargets.First(p => p.url == urls[i]).filename);
                     }
                 }
-            }.DownloadAndWait(downloadTargets.ToDictionary(p => p.uri, p => p.size));
+            }.DownloadAndWait(downloadTargets);
         }
 
         public static string DownloadText(Uri url)

Core/Net/NetAsyncDownloader.cs

@@ -31,15 +31,31 @@ private class NetAsyncDownloaderDownloadPart
             public Exception error;
             public int lastProgressUpdateSize;
 
-            public NetAsyncDownloaderDownloadPart(Uri url, long expectedSize, string path = null)
+            public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target, string path = null)
             {
-                this.url = url;
+                this.url  = target.url;
                 this.path = path ?? Path.GetTempFileName();
-                bytesLeft = expectedSize;
-                size = expectedSize;
+                size = bytesLeft = target.size;
                 lastProgressUpdateTime = DateTime.Now;
 
                 agent.Headers.Add("User-Agent", Net.UserAgentString);
+
+                // Tell the server what kind of files we want
+                if (!string.IsNullOrEmpty(target.mimeType))
+                {
+                    log.InfoFormat("Setting MIME type {0}", target.mimeType);
+                    agent.Headers.Add("Accept", target.mimeType);
+                }
+
+                // Check whether to use an auth token for this host
+                string token;
+                if (Win32Registry.TryGetAuthToken(this.url.Host, out token)
+                        && !string.IsNullOrEmpty(token))
+                {
+                    log.InfoFormat("Using auth token for {0}", this.url.Host);
+                    // Send our auth token to the GitHub API (or whoever else needs one)
+                    agent.Headers.Add("Authentication", $"token {token}");
+                }
             }
         }
 
@@ -73,14 +89,14 @@ public NetAsyncDownloader(IUser user)
 
         /// <summary>
         /// Downloads our files, returning an array of filenames that we're writing to.
-        /// The sole argument is a collection of KeyValuePair(s) containing the download URL and the expected download size
+        /// The sole argument is a collection of DownloadTargets.
         /// The .onCompleted delegate will be called on completion.
         /// </summary>
-        private void Download(ICollection<KeyValuePair<Uri, long>> urls)
+        private void Download(ICollection<Net.DownloadTarget> targets)
         {
-            foreach (var download in urls.Select(url => new NetAsyncDownloaderDownloadPart(url.Key, url.Value)))
+            foreach (Net.DownloadTarget target in targets)
             {
-                downloads.Add(download);
+                downloads.Add(new NetAsyncDownloaderDownloadPart(target));
             }
 
             // adding chicken bits
@@ -239,7 +255,7 @@ private void CurlWatchThread(int index, CurlEasy easy, FileStream stream)
             }
         }
 
-        public void DownloadAndWait(ICollection<KeyValuePair<Uri, long>> urls)
+        public void DownloadAndWait(ICollection<Net.DownloadTarget> urls)
         {
             // Start the download!
             Download(urls);

Core/Net/NetAsyncModulesDownloader.cs

@@ -21,6 +21,7 @@ public IUser User
 
         private          List<CkanModule>   modules;
         private readonly NetAsyncDownloader downloader;
+        private const    string             defaultMimeType = "application/octet-stream";
 
         /// <summary>
         /// Returns a perfectly boring NetAsyncModulesDownloader.
@@ -50,13 +51,20 @@ public void DownloadModules(NetModuleCache cache, IEnumerable<CkanModule> module
                 (_uris, paths, errors) =>
                     ModuleDownloadsComplete(cache, _uris, paths, errors);
 
-            // retrieve the expected download size for each mod
-            List<KeyValuePair<Uri, long>> downloads_with_size = unique_downloads
-                .Select(item => new KeyValuePair<Uri, long>(item.Key, item.Value.download_size))
-                .ToList();
-
-            // Start the download!
-            downloader.DownloadAndWait(downloads_with_size);
+            // Start the downloads!
+            downloader.DownloadAndWait(
+                unique_downloads.Select(item => new Net.DownloadTarget(
+                    item.Key,
+                    // Use a temp file name
+                    null,
+                    item.Value.download_size,
+                    // Send the MIME type to use for the Accept header
+                    // The GitHub API requires this to include application/octet-stream
+                    string.IsNullOrEmpty(item.Value.download_content_type)
+                        ? defaultMimeType
+                        : $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"
+                )).ToList()
+            );
         }
 
         /// <summary>

Core/Registry/IRegistryQuerier.cs

@@ -116,7 +116,7 @@ public interface IRegistryQuerier
     /// Helpers for <see cref="IRegistryQuerier"/>
     /// </summary>
     public static class IRegistryQuerierHelpers
-{
+    {
         /// <summary>
         /// Helper to call <see cref="IRegistryQuerier.GetModuleByVersion(string, Version)"/>
         /// </summary>

Core/Registry/Registry.cs

@@ -242,7 +242,7 @@ public static Registry Empty()
                 new Dictionary<string, AvailableModule>(),
                 new Dictionary<string, string>(),
                 new SortedDictionary<string, Repository>()
-                );
+            );
         }
 
         #endregion

Core/Types/CkanModule.cs

@@ -175,6 +175,9 @@ public class CkanModule : IEquatable<CkanModule>
         [JsonProperty("download_hash", NullValueHandling = NullValueHandling.Ignore)]
         public DownloadHashesDescriptor download_hash;
 
+        [JsonProperty("download_content_type", NullValueHandling = NullValueHandling.Ignore)]
+        public string download_content_type;
+
         [JsonProperty("identifier", Required = Required.Always)]
         public string identifier;
 

Core/Win32Registry.cs

@@ -1,12 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using Microsoft.Win32;
 
 namespace CKAN
 {
     public interface IWin32Registry
     {
-
         string AutoStartInstance { get; set; }
         void SetRegistryToInstances(SortedList<string, KSP> instances, string autoStartInstance);
         IEnumerable<Tuple<string, string>> GetInstances();
@@ -16,11 +16,15 @@ public interface IWin32Registry
 
     public class Win32Registry : IWin32Registry
     {
-        private static readonly string CKAN_KEY = @"HKEY_CURRENT_USER\Software\CKAN";
+        private const           string CKAN_KEY           = @"HKEY_CURRENT_USER\Software\CKAN";
+        private static readonly string CKAN_KEY_NO_PREFIX = StripPrefixKey(CKAN_KEY);
+
+        private const           string authTokenKey         = CKAN_KEY + @"\AuthTokens";
+        private static readonly string authTokenKeyNoPrefix = StripPrefixKey(authTokenKey);
 
         public Win32Registry()
         {
-            ConstructKey();
+            ConstructKey(CKAN_KEY_NO_PREFIX);
         }
         private int InstanceCount
         {
@@ -43,11 +47,11 @@ public void SetRegistryToInstances(SortedList<string, KSP> instances, string aut
         {
             SetAutoStartInstance(autoStartInstance ?? string.Empty);
             SetNumberOfInstances(instances.Count);
-            
+
             foreach (var instance in instances.Select((instance,i)=>
                 new {number=i,name=instance.Key,path=instance.Value}))
-            {                
-                SetInstanceKeysTo(instance.number, instance.name, instance.path);                
+            {
+                SetInstanceKeysTo(instance.number, instance.name, instance.path);
             }
         }
 
@@ -66,12 +70,67 @@ public void SetKSPBuilds(string buildMap)
             SetRegistryValue(@"KSPBuilds", buildMap);
         }
 
-        private void ConstructKey()
+        /// <summary>
+        /// Look for an auth token in the registry.
+        /// </summary>
+        /// <param name="host">Host for which to find a token</param>
+        /// <param name="token">Value of the token returned in parameter</param>
+        /// <returns>
+        /// True if found, false otherwise
+        /// </returns>
+        public static bool TryGetAuthToken(string host, out string token)
+        {
+            try
+            {
+                token = Microsoft.Win32.Registry.GetValue(authTokenKey, host, null) as string;
+                return !string.IsNullOrEmpty(token);
+            }
+            catch
+            {
+                // If GetValue threw SecurityException, IOException, or ArgumentException,
+                // just report failure.
+                token = "";
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Get the hosts that have auth tokens stored in the registry
+        /// </summary>
+        /// <returns>
+        /// Strings that are values of the auth token registry key
+        /// </returns>
+        public static IEnumerable<string> GetAuthTokenHosts()
         {
-            var key = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(@"Software\CKAN");
+            RegistryKey key = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(authTokenKeyNoPrefix);
+            return key?.GetValueNames();
+        }
+
+        /// <summary>
+        /// Set an auth token in the registry
+        /// </summary>
+        /// <param name="host">Host for which to set the token</param>
+        /// <param name="token">Token to set</param>
+        public static void SetAuthToken(string host, string token)
+        {
+            ConstructKey(authTokenKeyNoPrefix);
+            Microsoft.Win32.Registry.SetValue(authTokenKey, host, token);
+        }
+
+        private static string StripPrefixKey(string keyname)
+        {
+            int firstBackslash = keyname.IndexOf(@"\");
+            return firstBackslash < 0
+                ? keyname
+                : keyname.Substring(1 + firstBackslash);
+        }
+
+        private static void ConstructKey(string whichKey)
+        {
+            RegistryKey key = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(whichKey);
             if (key == null)
             {
-                Microsoft.Win32.Registry.CurrentUser.CreateSubKey(@"Software\CKAN");
+                Microsoft.Win32.Registry.CurrentUser.CreateSubKey(whichKey);
             }
         }
 
@@ -86,10 +145,10 @@ private void SetNumberOfInstances(int count)
         }
 
         private void SetInstanceKeysTo(int instanceIndex, string name, KSP ksp)
-        {            
+        {
             SetRegistryValue(@"KSPInstanceName_" + instanceIndex, name);
             SetRegistryValue(@"KSPInstancePath_" + instanceIndex, ksp.GameDir());
-        }        
+        }
 
         private static void SetRegistryValue<T>(string key, T value)
         {
@@ -101,4 +160,4 @@ private static T GetRegistryValue<T>(string key, T defaultValue)
             return (T)Microsoft.Win32.Registry.GetValue(CKAN_KEY, key, defaultValue);
         }
     }
-}
+}

GUI/MainModInfo.cs

@@ -407,9 +407,9 @@ private void CacheMod(object sender, DoWorkEventArgs e)
             Main.Instance.ResetProgress();
             Main.Instance.ClearLog();
 
-            NetAsyncModulesDownloader dowloader = new NetAsyncModulesDownloader(Main.Instance.currentUser);
+            NetAsyncModulesDownloader downloader = new NetAsyncModulesDownloader(Main.Instance.currentUser);
 
-            dowloader.DownloadModules(Main.Instance.CurrentInstance.Cache, new List<CkanModule> { (CkanModule)e.Argument });
+            downloader.DownloadModules(Main.Instance.CurrentInstance.Cache, new List<CkanModule> { (CkanModule)e.Argument });
             e.Result = e.Argument;
         }
 

Tests/Core/Net/NetAsyncModulesDownloader.cs

@@ -75,10 +75,7 @@ public void SingleDownload()
             log.InfoFormat("Downloading kOS from {0}",kOS.download);
 
             // Download our module.
-            async.DownloadModules(
-                ksp.KSP.Cache,
-                modules
-            );
+            async.DownloadModules(ksp.KSP.Cache, modules);
 
             // Assert that we have it, and it passes zip validation.
             Assert.IsTrue(cache.IsCachedZip(kOS));