tools: Allow a generator output directory to be specified

This also adjusts the post-generation steps to make sure they work without any current handwritten production code.
(In other words, we should be able to generate into an entirely empty directory, and the result should be the same as is in the repo, for all the files that end up being generated.)

There's still work to do around project and solution files.

generateapis.sh

@@ -16,6 +16,13 @@ else
   declare -r GOOGLEAPIS="$PWD/googleapis"
 fi
 
+if [[ "$GENERATOR_OUTPUT" != "" ]]
+then
+  declare -r GENERATOR_OUTPUT_DIR="$GENERATOR_OUTPUT"
+else
+  declare -r GENERATOR_OUTPUT_DIR="$PWD"
+fi
+
 fetch_github_repos() {
   # We assume that if the directory has been explicitly specified, we don't need
   # to fetch it.
@@ -47,5 +54,6 @@ export GRPC_PLUGIN
 export PROTOBUF_TOOLS_ROOT
 export GAPIC_PLUGIN
 export PROTOC
+export GENERATOR_OUTPUT_DIR
 
 dotnet run --project tools/Google.Cloud.Tools.ReleaseManager -- generate-apis $*

generator-input/tweaks/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.GenerateClient/Program.cs

@@ -261,7 +261,7 @@ private static async Task<int> Main(string[] args)
 
             try
             {
-                var resultPath = Path.Combine(Path.GetDirectoryName(projectFile), $"{userClientName}.cs");
+                var resultPath = Path.Combine(Path.GetDirectoryName(projectFile), $"{userClientName}.g.cs");
                 File.WriteAllText(resultPath, resultText);
             }
             catch (Exception e)

generator-input/tweaks/Google.Cloud.Bigtable.V2/postgeneration.sh

@@ -2,9 +2,7 @@
 
 set -e
 
-# TODO: Use the right output directory
-GENERATOR_OUTPUT=../../..
-BIGTABLE_OUTPUT=$GENERATOR_OUTPUT/apis/Google.Cloud.Bigtable.V2
+BIGTABLE_OUTPUT=$GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Bigtable.V2
 
 # Generate BigtableClient
 dotnet run --project Google.Cloud.Bigtable.V2.GenerateClient -- \

generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/EnumContainer.cs

@@ -41,7 +41,7 @@ public EnumContainer(MessageDescriptor descriptor, IEnumerable<EnumDescriptor> e
             NestedContainers = messages.Select(message => new EnumContainer(message, message.EnumTypes, message.NestedTypes)).ToList();
         }
 
-        public void Generate(TextWriter writer, string indentation, System.Type computeEnumHelpersType)
+        public void Generate(TextWriter writer, string indentation)
         {
             if (Descriptor is object)
             {
@@ -60,29 +60,29 @@ public void Generate(TextWriter writer, string indentation, System.Type computeE
             foreach (var enumDescriptor in Enums.OrderBy(d => d.ClrType.Name, StringComparer.Ordinal))
             {
                 MaybeWriteLineBetweenElements(writer, ref first);
-                GenerateEnumConstants(writer, enumDescriptor, moreIndentation, computeEnumHelpersType);
+                GenerateEnumConstants(writer, enumDescriptor, moreIndentation);
             }
             foreach (var container in NestedContainers.Where(nc => nc.ShouldGenerate).OrderBy(nc => nc.Descriptor.Name, StringComparer.Ordinal))
             {
                 MaybeWriteLineBetweenElements(writer, ref first);
-                container.Generate(writer, moreIndentation, computeEnumHelpersType);
+                container.Generate(writer, moreIndentation);
             }
             writer.WriteLine($"{indentation}}}");
         }
 
-        private static void GenerateEnumConstants(TextWriter writer, EnumDescriptor descriptor, string indentation, System.Type computeEnumHelpersType)
+        private static void GenerateEnumConstants(TextWriter writer, EnumDescriptor descriptor, string indentation)
         {
             var enumType = descriptor.ClrType;
             writer.WriteLine($"{indentation}/// <summary>Constants for wire representations of the <see cref=\"global::{enumType.FullName.Replace("+", ".")}\"/> enum.</summary>");
             writer.WriteLine($"{indentation}public static class {descriptor.ClrType.Name}");
             writer.WriteLine($"{indentation}{{");
             string moreIndentation = indentation + "    ";
-            var formatMethod = computeEnumHelpersType.GetMethod("Format").MakeGenericMethod(enumType);
             bool first = true;
             foreach (var field in enumType.GetFields(BindingFlags.Static | BindingFlags.Public))
             {
                 MaybeWriteLineBetweenElements(writer, ref first);
-                var wireValue = formatMethod.Invoke(null, new object[] { field.GetValue(null) });
+                // This is effectively what ComputeEnumHelpers.Format does (with lots of caching etc).
+                var wireValue = field.GetCustomAttributes<OriginalNameAttribute>().Single().Name;
                 // TODO: It would be nice to generate the comment for the enum value as well, but that's quite tricky.
                 writer.WriteLine($"{moreIndentation}/// <summary>Wire representation of <see cref=\"global::{enumType.FullName.Replace("+", ".")}.{field.Name}\"/>.</summary>");
                 string maybeUnderscore = NeedsUnderscore(field.Name) ? "_" : "";

generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/Program.cs

@@ -55,7 +55,7 @@ private static int Main(string[] args)
 
 namespace Google.Cloud.Compute.V1
 {{");
-            rootContainer.Generate(writer, "    ", asm.GetType("Google.Cloud.Compute.V1.ComputeEnumHelpers"));
+            rootContainer.Generate(writer, "    ");
             writer.WriteLine("}");
             return 0;
         }

generator-input/tweaks/Google.Cloud.Compute.V1/postgeneration.sh

@@ -2,16 +2,33 @@
 
 set -e
 
-# TODO: Use the appropriate output directory
-
-GENERATOR_OUTPUT=../../..
-COMPUTE_OUTPUT=$GENERATOR_OUTPUT/apis/Google.Cloud.Compute.V1
+COMPUTE_OUTPUT=$GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Compute.V1
 
 TFM=netstandard2.0
 
 mkdir -p /tmp/compute
+
+# Workarounds to get the generated code to compile.
+# The project doesn't include Google.LongRunning due to the way DIREGAPIC works.
+dotnet add $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.csproj \
+    package Google.LongRunning
+# We have a currently-handwritten class which is required by the generated
+# code. We could eventually generate it, but it's not clear how much value that would have.
+cat << EOF > $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/OperationWorkaround.g.cs
+using lro = Google.LongRunning;
+namespace Google.Cloud.Compute.V1;
+partial class Operation
+{
+    internal Google.LongRunning.Operation ToLroResponse(string name) => null;
+}
+EOF
+
+# Build the code so we can access the descriptor
 dotnet build $COMPUTE_OUTPUT/Google.Cloud.Compute.V1 -f $TFM -o /tmp/compute
 
+# Remove the temporary file.
+rm $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/OperationWorkaround.g.cs
+
 # Generate enum constants
 dotnet run --project Google.Cloud.Compute.V1.EnumConstantGenerator \
     /tmp/compute/Google.Cloud.Compute.V1.dll \

generator-input/tweaks/Google.Cloud.DevTools.ContainerAnalysis.V1/postgeneration.sh

@@ -2,8 +2,7 @@
 
 set -e
 
-# TODO: Use the right output directory
-cd ../../../apis/Google.Cloud.DevTools.ContainerAnalysis.V1
+cd $GENERATOR_OUTPUT_DIR/apis/Google.Cloud.DevTools.ContainerAnalysis.V1
 
 # Add the Grafeas.V1 surface to the metadata so that
 # the REST transport will work. This API is *very* unusual

generator-input/tweaks/Google.Cloud.Logging.V2/postgeneration.sh

@@ -2,8 +2,7 @@
 
 set -e
 
-# TODO: Use the right output directory
-cd ../../../apis/Google.Cloud.Logging.V2
+cd $GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Logging.V2
 
 # Fix the gRPC generated code (for the grpc::Method instances) so it refers to the right server-side RPC
 sed -i 's/"UpdateBucketLongRunning"/"UpdateBucketAsync"/g' Google.Cloud.Logging.V2/LoggingConfigGrpc.g.cs

generator-input/tweaks/Google.LongRunning/postgeneration.sh

@@ -5,13 +5,12 @@ set -e
 # Generate an additional proto file containing annotations for
 # LROs. Ideally this would be in a common protos directory (or
 # google.longrunning) but it's too late to fix that now.
-cd ../../..
-source toolversions.sh
-install_protoc
+# We expect that protoc is already installed, and that all the
+# environment variables used below are present.
 declare -r CORE_PROTOS_ROOT=$PROTOBUF_TOOLS_ROOT/tools
 
 $PROTOC \
-  --csharp_out=apis/Google.LongRunning/Google.LongRunning \
+  --csharp_out=$GENERATOR_OUTPUT_DIR/apis/Google.LongRunning/Google.LongRunning \
   --csharp_opt=base_namespace=Google.Cloud,file_extension=.g.cs \
   -I $GOOGLEAPIS \
   -I $CORE_PROTOS_ROOT \

generator-input/tweaks/Grafeas.V1/postgeneration.sh

@@ -2,5 +2,4 @@
 
 set -e
 
-# TODO: Use the specified output directory
-dotnet run --project Grafeas.V1.FixGeneratedCode ../../../apis/Grafeas.V1
+dotnet run --project Grafeas.V1.FixGeneratedCode $GENERATOR_OUTPUT_DIR/apis/Grafeas.V1

tools/Google.Cloud.Tools.Common/DirectoryLayout.cs

@@ -57,18 +57,19 @@ public static DirectoryLayout ForHelpSnippets()
                 );
         }
 
-        public static DirectoryLayout ForApi(string api)
-        {
-            var root = DetermineRootDirectory();
-            return new DirectoryLayout(
-                source: Path.Combine(root, "apis", api),
-                docsOutput: Path.Combine(root, "docs", "output", api),
-                metadata: Path.Combine(root, "docs", "output", api, "obj", "api"),
-                snippetOutput: Path.Combine(root, "docs", "output", api, "obj", "snippets"),
-                docsSource: Path.Combine(root, "apis", api, "docs"),
-                tweaks: Path.Combine(root, "generator-input", "tweaks", api)
+        public static DirectoryLayout ForApi(string api, string outputRoot) =>
+            new DirectoryLayout(
+                source: Path.Combine(outputRoot, "apis", api),
+                docsOutput: Path.Combine(outputRoot, "docs", "output", api),
+                metadata: Path.Combine(outputRoot, "docs", "output", api, "obj", "api"),
+                snippetOutput: Path.Combine(outputRoot, "docs", "output", api, "obj", "snippets"),
+                docsSource: Path.Combine(outputRoot, "apis", api, "docs"),
+                // TODO: Maybe accept the root directory for inputs as another parameter?
+                // We don't need to do this at the moment.
+                tweaks: Path.Combine(DetermineRootDirectory(), "generator-input", "tweaks", api)
                 );
-        }
+
+        public static DirectoryLayout ForApi(string api) => ForApi(api, DetermineRootDirectory());
 
         /// <summary>
         /// Find the root directory of the project. We expect this to contain "apis" and "LICENSE".

tools/Google.Cloud.Tools.ReleaseManager/GenerateApisCommand.cs

@@ -35,13 +35,15 @@ internal class GenerateApisCommand : ICommand
     private const string GapicGeneratorEnvironmentVariable = "GAPIC_PLUGIN";
     private const string GrpcGeneratorEnvironmentVariable = "GRPC_PLUGIN";
     private const string GoogleApisDirectoryEnvironmentVariable = "GOOGLEAPIS";
+    private const string GeneratorOutputDirectoryEnvironmentVariable = "GENERATOR_OUTPUT_DIR";
     private const string TempDir = "tmp";
 
     private readonly string protocBinary;
     private readonly string gapicGeneratorBinary;
     private readonly string grpcGeneratorBinary;
     private readonly string googleApisDirectory;
     private readonly string protobufToolsRootDirectory;
+    private readonly string generatorOutputDirectory;
 
     public string Description => "Generates APIs (used by generateapis.sh)";
 
@@ -57,6 +59,7 @@ public GenerateApisCommand()
         grpcGeneratorBinary = Environment.GetEnvironmentVariable(GrpcGeneratorEnvironmentVariable);
         googleApisDirectory = Environment.GetEnvironmentVariable(GoogleApisDirectoryEnvironmentVariable);
         protobufToolsRootDirectory = Environment.GetEnvironmentVariable(ProtobufToolsRootEnvironmentVariable);
+        generatorOutputDirectory = Environment.GetEnvironmentVariable(GeneratorOutputDirectoryEnvironmentVariable);
     }
 
     public int Execute(string[] args)
@@ -107,7 +110,7 @@ public int Execute(string[] args)
 
     private void Generate(ApiMetadata api)
     {
-        var layout = DirectoryLayout.ForApi(api.Id);
+        var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory);
         Console.WriteLine($"{DateTime.UtcNow:yyyy-MM-dd'T'HH:mm:ss.fff}Z Generating {api.Id}");
         MaybeRunScript("pregeneration.sh");
         switch (api.Generator)
@@ -172,7 +175,7 @@ void ValidateNamespaces()
 
     private void GenerateGapicApi(ApiMetadata api)
     {
-        var layout = DirectoryLayout.ForApi(api.Id);
+        var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory);
         string productionDirectory = Path.Combine(layout.SourceDirectory, api.Id);
         Directory.CreateDirectory(productionDirectory);
         DeleteGeneratedFiles(productionDirectory);
@@ -260,7 +263,7 @@ private void GenerateGapicApi(ApiMetadata api)
 
     private void GenerateProtoApi(ApiMetadata api)
     {
-        var layout = DirectoryLayout.ForApi(api.Id);
+        var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory);
         string productionDirectory = Path.Combine(layout.SourceDirectory, api.Id);
         Directory.CreateDirectory(productionDirectory);
         DeleteGeneratedFiles(productionDirectory);
@@ -316,6 +319,7 @@ private void ValidateEnvironment()
         ValidateFile(grpcGeneratorBinary, "gRPC generator");
         ValidateDirectory(googleApisDirectory, "googleapis");
         ValidateDirectory(protobufToolsRootDirectory, "protobuf tools root");
+        ValidateDirectory(generatorOutputDirectory, "generator output");
         // This will throw if we can't detect bash.
         GetBashExecutable();
 
@@ -355,6 +359,7 @@ private static string GetBashExecutable() =>
     private int ExecuteForUnconfigured(string[] args)
     {
         // TODO: Maybe have another way of specifying this, eventually.
+        // (Potentially just use generatorOutputDirectory.)
         var root = DirectoryLayout.DetermineRootDirectory();
 
         var outputRoot = Path.Combine(root, "unconfigured-generation");