-
Notifications
You must be signed in to change notification settings - Fork 37
/
Program.cs
152 lines (129 loc) · 5.51 KB
/
Program.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using ICSharpCode.SharpZipLib.GZip;
using ICSharpCode.SharpZipLib.Tar;
using Mono.Options;
namespace UploadIndexStage1
{
class Program
{
static async Task Main(string[] args)
{
string sourceFolder = null;
string repoName = null;
string clientId = null;
string storageAccount = null;
string blobContainer = null;
var options = new OptionSet
{
{"i=", "The source folder", i => sourceFolder = i},
{"n=", "The repo name", n => repoName = n},
{"c=", "The Azure Client ID (optional)", c => clientId = c},
{"s=", "The destination storage account name or URL", s => storageAccount = s},
{"b=", "The destination storage account container", b => blobContainer = b},
};
List<string> extra = options.Parse(args);
if (extra.Any())
{
Fatal($"Unexpected argument {extra.First()}");
}
if (string.IsNullOrEmpty(sourceFolder))
{
Fatal("Missing argument -i");
}
if (string.IsNullOrEmpty(repoName))
{
Fatal("Missing argument -n");
}
if (string.IsNullOrEmpty(storageAccount))
{
Fatal("Missing argument -s");
}
if (string.IsNullOrEmpty(blobContainer))
{
Fatal("Missing argument -b");
}
if (!storageAccount.StartsWith("https://"))
{
storageAccount = "https://" + storageAccount + ".blob.core.windows.net";
}
DefaultAzureCredential credential;
if (string.IsNullOrEmpty(clientId) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ARM_CLIENT_ID")))
{
clientId = Environment.GetEnvironmentVariable("ARM_CLIENT_ID");
System.Console.WriteLine("Found client ID in environment variable; using it");
}
if (string.IsNullOrEmpty(clientId))
{
credential = new DefaultAzureCredential();
System.Console.WriteLine("Trying to use managed identity without default identity");
}
else
{
credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = clientId });
System.Console.WriteLine("Trying to use managed identity with client id: " + clientId);
}
BlobServiceClient blobServiceClient = new(
new Uri(storageAccount),
credential);
var containerClient = blobServiceClient.GetBlobContainerClient(blobContainer);
string newBlobName = $"{repoName}/{DateTime.UtcNow:O}.tar.gz";
BlobClient newBlobClient = containerClient.GetBlobClient(newBlobName);
Console.WriteLine($"Uploading folder {sourceFolder} to blob {new UriBuilder(newBlobClient.Uri) {Fragment = "", Query = ""}.Uri.AbsoluteUri}");
await using (var outputFileStream = new MemoryStream())
{
await using (var gzoStream = new GZipOutputStream(outputFileStream){ IsStreamOwner = false })
using (var tarArchive = TarArchive.CreateOutputTarArchive(gzoStream, Encoding.UTF8))
{
string sourceRoot = Path.GetFullPath(sourceFolder).Replace('\\', '/').TrimEnd('/');
void AddEntry(string path)
{
string normalizedPath = Path.GetFullPath(path).Replace('\\', '/');
var e = TarEntry.CreateEntryFromFile(path);
e.Name = normalizedPath.Substring(sourceRoot.Length).TrimStart('/');
Console.WriteLine($"Adding {path} as {e.Name}");
tarArchive.WriteEntry(e, false);
}
void AddFolder(string path)
{
AddEntry(path);
foreach (string file in Directory.GetFiles(path))
{
AddEntry(file);
}
foreach (string dir in Directory.GetDirectories(path))
{
AddFolder(dir);
}
}
AddFolder(sourceRoot);
}
outputFileStream.Position = 0;
await newBlobClient.UploadAsync(outputFileStream);
}
Console.WriteLine("Cleaning up old blobs");
List<BlobItem> blobs = containerClient.GetBlobs(prefix: repoName + "/").ToList();
List<BlobItem> toDelete = blobs.OrderByDescending(b => b.Name).Skip(10).ToList();
foreach (BlobItem d in toDelete)
{
Console.WriteLine($"Deleting blob {d.Name}");
await containerClient.DeleteBlobAsync(d.Name);
}
Console.WriteLine("Finished.");
}
[DoesNotReturn]
private static void Fatal(string msg)
{
Console.Error.WriteLine($"fatal: {msg}");
Environment.Exit(-1);
}
}
}