From 97053f76deb8b500272f7941d7afae1d74172548 Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Tue, 3 Dec 2024 17:22:29 +0000 Subject: [PATCH] tools: Allow a generator output directory to be specified This also adjusts the post-generation steps to make sure they work without any current handwritten production code. (In other words, we should be able to generate into an entirely empty directory, and the result should be the same as is in the repo, for all the files that end up being generated.) There's still work to do around project and solution files. --- generateapis.sh | 8 ++++++ .../Program.cs | 2 +- .../postgeneration.sh | 4 +-- .../EnumContainer.cs | 12 ++++----- .../Program.cs | 2 +- .../Google.Cloud.Compute.V1/postgeneration.sh | 25 ++++++++++++++++--- .../postgeneration.sh | 3 +-- .../Google.Cloud.Logging.V2/postgeneration.sh | 3 +-- .../Google.LongRunning/postgeneration.sh | 7 +++--- .../tweaks/Grafeas.V1/postgeneration.sh | 3 +-- .../DirectoryLayout.cs | 23 +++++++++-------- .../GenerateApisCommand.cs | 11 +++++--- 12 files changed, 64 insertions(+), 39 deletions(-) diff --git a/generateapis.sh b/generateapis.sh index 46a2d6ea7854..26ba18cce45d 100755 --- a/generateapis.sh +++ b/generateapis.sh @@ -16,6 +16,13 @@ else declare -r GOOGLEAPIS="$PWD/googleapis" fi +if [[ "$GENERATOR_OUTPUT" != "" ]] +then + declare -r GENERATOR_OUTPUT_DIR="$GENERATOR_OUTPUT" +else + declare -r GENERATOR_OUTPUT_DIR="$PWD" +fi + fetch_github_repos() { # We assume that if the directory has been explicitly specified, we don't need # to fetch it. @@ -47,5 +54,6 @@ export GRPC_PLUGIN export PROTOBUF_TOOLS_ROOT export GAPIC_PLUGIN export PROTOC +export GENERATOR_OUTPUT_DIR dotnet run --project tools/Google.Cloud.Tools.ReleaseManager -- generate-apis $* diff --git a/generator-input/tweaks/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.GenerateClient/Program.cs b/generator-input/tweaks/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.GenerateClient/Program.cs index 0996f46bb2af..d7ff578667ed 100644 --- a/generator-input/tweaks/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.GenerateClient/Program.cs +++ b/generator-input/tweaks/Google.Cloud.Bigtable.V2/Google.Cloud.Bigtable.V2.GenerateClient/Program.cs @@ -261,7 +261,7 @@ private static async Task Main(string[] args) try { - var resultPath = Path.Combine(Path.GetDirectoryName(projectFile), $"{userClientName}.cs"); + var resultPath = Path.Combine(Path.GetDirectoryName(projectFile), $"{userClientName}.g.cs"); File.WriteAllText(resultPath, resultText); } catch (Exception e) diff --git a/generator-input/tweaks/Google.Cloud.Bigtable.V2/postgeneration.sh b/generator-input/tweaks/Google.Cloud.Bigtable.V2/postgeneration.sh index 098c630fb706..a596349682d7 100644 --- a/generator-input/tweaks/Google.Cloud.Bigtable.V2/postgeneration.sh +++ b/generator-input/tweaks/Google.Cloud.Bigtable.V2/postgeneration.sh @@ -2,9 +2,7 @@ set -e -# TODO: Use the right output directory -GENERATOR_OUTPUT=../../.. -BIGTABLE_OUTPUT=$GENERATOR_OUTPUT/apis/Google.Cloud.Bigtable.V2 +BIGTABLE_OUTPUT=$GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Bigtable.V2 # Generate BigtableClient dotnet run --project Google.Cloud.Bigtable.V2.GenerateClient -- \ diff --git a/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/EnumContainer.cs b/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/EnumContainer.cs index d126ae218c02..e6366623f6a6 100644 --- a/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/EnumContainer.cs +++ b/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/EnumContainer.cs @@ -41,7 +41,7 @@ public EnumContainer(MessageDescriptor descriptor, IEnumerable e NestedContainers = messages.Select(message => new EnumContainer(message, message.EnumTypes, message.NestedTypes)).ToList(); } - public void Generate(TextWriter writer, string indentation, System.Type computeEnumHelpersType) + public void Generate(TextWriter writer, string indentation) { if (Descriptor is object) { @@ -60,29 +60,29 @@ public void Generate(TextWriter writer, string indentation, System.Type computeE foreach (var enumDescriptor in Enums.OrderBy(d => d.ClrType.Name, StringComparer.Ordinal)) { MaybeWriteLineBetweenElements(writer, ref first); - GenerateEnumConstants(writer, enumDescriptor, moreIndentation, computeEnumHelpersType); + GenerateEnumConstants(writer, enumDescriptor, moreIndentation); } foreach (var container in NestedContainers.Where(nc => nc.ShouldGenerate).OrderBy(nc => nc.Descriptor.Name, StringComparer.Ordinal)) { MaybeWriteLineBetweenElements(writer, ref first); - container.Generate(writer, moreIndentation, computeEnumHelpersType); + container.Generate(writer, moreIndentation); } writer.WriteLine($"{indentation}}}"); } - private static void GenerateEnumConstants(TextWriter writer, EnumDescriptor descriptor, string indentation, System.Type computeEnumHelpersType) + private static void GenerateEnumConstants(TextWriter writer, EnumDescriptor descriptor, string indentation) { var enumType = descriptor.ClrType; writer.WriteLine($"{indentation}/// Constants for wire representations of the enum."); writer.WriteLine($"{indentation}public static class {descriptor.ClrType.Name}"); writer.WriteLine($"{indentation}{{"); string moreIndentation = indentation + " "; - var formatMethod = computeEnumHelpersType.GetMethod("Format").MakeGenericMethod(enumType); bool first = true; foreach (var field in enumType.GetFields(BindingFlags.Static | BindingFlags.Public)) { MaybeWriteLineBetweenElements(writer, ref first); - var wireValue = formatMethod.Invoke(null, new object[] { field.GetValue(null) }); + // This is effectively what ComputeEnumHelpers.Format does (with lots of caching etc). + var wireValue = field.GetCustomAttributes().Single().Name; // TODO: It would be nice to generate the comment for the enum value as well, but that's quite tricky. writer.WriteLine($"{moreIndentation}/// Wire representation of ."); string maybeUnderscore = NeedsUnderscore(field.Name) ? "_" : ""; diff --git a/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/Program.cs b/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/Program.cs index 87ab2482b496..80cf86584d9f 100644 --- a/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/Program.cs +++ b/generator-input/tweaks/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.EnumConstantGenerator/Program.cs @@ -55,7 +55,7 @@ private static int Main(string[] args) namespace Google.Cloud.Compute.V1 {{"); - rootContainer.Generate(writer, " ", asm.GetType("Google.Cloud.Compute.V1.ComputeEnumHelpers")); + rootContainer.Generate(writer, " "); writer.WriteLine("}"); return 0; } diff --git a/generator-input/tweaks/Google.Cloud.Compute.V1/postgeneration.sh b/generator-input/tweaks/Google.Cloud.Compute.V1/postgeneration.sh index 5fc29082e419..fb65d397c613 100644 --- a/generator-input/tweaks/Google.Cloud.Compute.V1/postgeneration.sh +++ b/generator-input/tweaks/Google.Cloud.Compute.V1/postgeneration.sh @@ -2,16 +2,33 @@ set -e -# TODO: Use the appropriate output directory - -GENERATOR_OUTPUT=../../.. -COMPUTE_OUTPUT=$GENERATOR_OUTPUT/apis/Google.Cloud.Compute.V1 +COMPUTE_OUTPUT=$GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Compute.V1 TFM=netstandard2.0 mkdir -p /tmp/compute + +# Workarounds to get the generated code to compile. +# The project doesn't include Google.LongRunning due to the way DIREGAPIC works. +dotnet add $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/Google.Cloud.Compute.V1.csproj \ + package Google.LongRunning +# We have a currently-handwritten class which is required by the generated +# code. We could eventually generate it, but it's not clear how much value that would have. +cat << EOF > $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/OperationWorkaround.g.cs +using lro = Google.LongRunning; +namespace Google.Cloud.Compute.V1; +partial class Operation +{ + internal Google.LongRunning.Operation ToLroResponse(string name) => null; +} +EOF + +# Build the code so we can access the descriptor dotnet build $COMPUTE_OUTPUT/Google.Cloud.Compute.V1 -f $TFM -o /tmp/compute +# Remove the temporary file. +rm $COMPUTE_OUTPUT/Google.Cloud.Compute.V1/OperationWorkaround.g.cs + # Generate enum constants dotnet run --project Google.Cloud.Compute.V1.EnumConstantGenerator \ /tmp/compute/Google.Cloud.Compute.V1.dll \ diff --git a/generator-input/tweaks/Google.Cloud.DevTools.ContainerAnalysis.V1/postgeneration.sh b/generator-input/tweaks/Google.Cloud.DevTools.ContainerAnalysis.V1/postgeneration.sh index 84066046b56e..7bfdee38da94 100644 --- a/generator-input/tweaks/Google.Cloud.DevTools.ContainerAnalysis.V1/postgeneration.sh +++ b/generator-input/tweaks/Google.Cloud.DevTools.ContainerAnalysis.V1/postgeneration.sh @@ -2,8 +2,7 @@ set -e -# TODO: Use the right output directory -cd ../../../apis/Google.Cloud.DevTools.ContainerAnalysis.V1 +cd $GENERATOR_OUTPUT_DIR/apis/Google.Cloud.DevTools.ContainerAnalysis.V1 # Add the Grafeas.V1 surface to the metadata so that # the REST transport will work. This API is *very* unusual diff --git a/generator-input/tweaks/Google.Cloud.Logging.V2/postgeneration.sh b/generator-input/tweaks/Google.Cloud.Logging.V2/postgeneration.sh index 9f675b062937..3565d5ed6f5a 100644 --- a/generator-input/tweaks/Google.Cloud.Logging.V2/postgeneration.sh +++ b/generator-input/tweaks/Google.Cloud.Logging.V2/postgeneration.sh @@ -2,8 +2,7 @@ set -e -# TODO: Use the right output directory -cd ../../../apis/Google.Cloud.Logging.V2 +cd $GENERATOR_OUTPUT_DIR/apis/Google.Cloud.Logging.V2 # Fix the gRPC generated code (for the grpc::Method instances) so it refers to the right server-side RPC sed -i 's/"UpdateBucketLongRunning"/"UpdateBucketAsync"/g' Google.Cloud.Logging.V2/LoggingConfigGrpc.g.cs diff --git a/generator-input/tweaks/Google.LongRunning/postgeneration.sh b/generator-input/tweaks/Google.LongRunning/postgeneration.sh index 211bb09eef10..573d70a25a6b 100644 --- a/generator-input/tweaks/Google.LongRunning/postgeneration.sh +++ b/generator-input/tweaks/Google.LongRunning/postgeneration.sh @@ -5,13 +5,12 @@ set -e # Generate an additional proto file containing annotations for # LROs. Ideally this would be in a common protos directory (or # google.longrunning) but it's too late to fix that now. -cd ../../.. -source toolversions.sh -install_protoc +# We expect that protoc is already installed, and that all the +# environment variables used below are present. declare -r CORE_PROTOS_ROOT=$PROTOBUF_TOOLS_ROOT/tools $PROTOC \ - --csharp_out=apis/Google.LongRunning/Google.LongRunning \ + --csharp_out=$GENERATOR_OUTPUT_DIR/apis/Google.LongRunning/Google.LongRunning \ --csharp_opt=base_namespace=Google.Cloud,file_extension=.g.cs \ -I $GOOGLEAPIS \ -I $CORE_PROTOS_ROOT \ diff --git a/generator-input/tweaks/Grafeas.V1/postgeneration.sh b/generator-input/tweaks/Grafeas.V1/postgeneration.sh index 0928a9057783..938d5c125717 100644 --- a/generator-input/tweaks/Grafeas.V1/postgeneration.sh +++ b/generator-input/tweaks/Grafeas.V1/postgeneration.sh @@ -2,5 +2,4 @@ set -e -# TODO: Use the specified output directory -dotnet run --project Grafeas.V1.FixGeneratedCode ../../../apis/Grafeas.V1 +dotnet run --project Grafeas.V1.FixGeneratedCode $GENERATOR_OUTPUT_DIR/apis/Grafeas.V1 diff --git a/tools/Google.Cloud.Tools.Common/DirectoryLayout.cs b/tools/Google.Cloud.Tools.Common/DirectoryLayout.cs index 6675b6fe0ff0..6cb3c3e38e77 100644 --- a/tools/Google.Cloud.Tools.Common/DirectoryLayout.cs +++ b/tools/Google.Cloud.Tools.Common/DirectoryLayout.cs @@ -57,18 +57,19 @@ public static DirectoryLayout ForHelpSnippets() ); } - public static DirectoryLayout ForApi(string api) - { - var root = DetermineRootDirectory(); - return new DirectoryLayout( - source: Path.Combine(root, "apis", api), - docsOutput: Path.Combine(root, "docs", "output", api), - metadata: Path.Combine(root, "docs", "output", api, "obj", "api"), - snippetOutput: Path.Combine(root, "docs", "output", api, "obj", "snippets"), - docsSource: Path.Combine(root, "apis", api, "docs"), - tweaks: Path.Combine(root, "generator-input", "tweaks", api) + public static DirectoryLayout ForApi(string api, string outputRoot) => + new DirectoryLayout( + source: Path.Combine(outputRoot, "apis", api), + docsOutput: Path.Combine(outputRoot, "docs", "output", api), + metadata: Path.Combine(outputRoot, "docs", "output", api, "obj", "api"), + snippetOutput: Path.Combine(outputRoot, "docs", "output", api, "obj", "snippets"), + docsSource: Path.Combine(outputRoot, "apis", api, "docs"), + // TODO: Maybe accept the root directory for inputs as another parameter? + // We don't need to do this at the moment. + tweaks: Path.Combine(DetermineRootDirectory(), "generator-input", "tweaks", api) ); - } + + public static DirectoryLayout ForApi(string api) => ForApi(api, DetermineRootDirectory()); /// /// Find the root directory of the project. We expect this to contain "apis" and "LICENSE". diff --git a/tools/Google.Cloud.Tools.ReleaseManager/GenerateApisCommand.cs b/tools/Google.Cloud.Tools.ReleaseManager/GenerateApisCommand.cs index 33ba74304ddf..6da4e7e0f12d 100644 --- a/tools/Google.Cloud.Tools.ReleaseManager/GenerateApisCommand.cs +++ b/tools/Google.Cloud.Tools.ReleaseManager/GenerateApisCommand.cs @@ -35,6 +35,7 @@ internal class GenerateApisCommand : ICommand private const string GapicGeneratorEnvironmentVariable = "GAPIC_PLUGIN"; private const string GrpcGeneratorEnvironmentVariable = "GRPC_PLUGIN"; private const string GoogleApisDirectoryEnvironmentVariable = "GOOGLEAPIS"; + private const string GeneratorOutputDirectoryEnvironmentVariable = "GENERATOR_OUTPUT_DIR"; private const string TempDir = "tmp"; private readonly string protocBinary; @@ -42,6 +43,7 @@ internal class GenerateApisCommand : ICommand private readonly string grpcGeneratorBinary; private readonly string googleApisDirectory; private readonly string protobufToolsRootDirectory; + private readonly string generatorOutputDirectory; public string Description => "Generates APIs (used by generateapis.sh)"; @@ -57,6 +59,7 @@ public GenerateApisCommand() grpcGeneratorBinary = Environment.GetEnvironmentVariable(GrpcGeneratorEnvironmentVariable); googleApisDirectory = Environment.GetEnvironmentVariable(GoogleApisDirectoryEnvironmentVariable); protobufToolsRootDirectory = Environment.GetEnvironmentVariable(ProtobufToolsRootEnvironmentVariable); + generatorOutputDirectory = Environment.GetEnvironmentVariable(GeneratorOutputDirectoryEnvironmentVariable); } public int Execute(string[] args) @@ -107,7 +110,7 @@ public int Execute(string[] args) private void Generate(ApiMetadata api) { - var layout = DirectoryLayout.ForApi(api.Id); + var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory); Console.WriteLine($"{DateTime.UtcNow:yyyy-MM-dd'T'HH:mm:ss.fff}Z Generating {api.Id}"); MaybeRunScript("pregeneration.sh"); switch (api.Generator) @@ -172,7 +175,7 @@ void ValidateNamespaces() private void GenerateGapicApi(ApiMetadata api) { - var layout = DirectoryLayout.ForApi(api.Id); + var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory); string productionDirectory = Path.Combine(layout.SourceDirectory, api.Id); Directory.CreateDirectory(productionDirectory); DeleteGeneratedFiles(productionDirectory); @@ -260,7 +263,7 @@ private void GenerateGapicApi(ApiMetadata api) private void GenerateProtoApi(ApiMetadata api) { - var layout = DirectoryLayout.ForApi(api.Id); + var layout = DirectoryLayout.ForApi(api.Id, generatorOutputDirectory); string productionDirectory = Path.Combine(layout.SourceDirectory, api.Id); Directory.CreateDirectory(productionDirectory); DeleteGeneratedFiles(productionDirectory); @@ -316,6 +319,7 @@ private void ValidateEnvironment() ValidateFile(grpcGeneratorBinary, "gRPC generator"); ValidateDirectory(googleApisDirectory, "googleapis"); ValidateDirectory(protobufToolsRootDirectory, "protobuf tools root"); + ValidateDirectory(generatorOutputDirectory, "generator output"); // This will throw if we can't detect bash. GetBashExecutable(); @@ -355,6 +359,7 @@ private static string GetBashExecutable() => private int ExecuteForUnconfigured(string[] args) { // TODO: Maybe have another way of specifying this, eventually. + // (Potentially just use generatorOutputDirectory.) var root = DirectoryLayout.DetermineRootDirectory(); var outputRoot = Path.Combine(root, "unconfigured-generation");