Merge pull request #139 from pulumi/vl/CompressSchemaMetadata

Gzip schema and metadata to avoid ballooning memory requirements

provider/cmd/pulumi-gen-azurerm/main.go

@@ -15,6 +15,8 @@
 package main
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -96,8 +98,14 @@ func emitSchema(pkgSpec schema.PackageSpec, outDir string) error {
 }
 
 func emitMetadata(metadata *provider.AzureApiMetadata, outDir string) error {
-	raw, err := json.Marshal(metadata)
+	compressedMeta := bytes.Buffer{}
+	compressedWriter := gzip.NewWriter(&compressedMeta)
+	err := json.NewEncoder(compressedWriter).Encode(metadata)
 	if err != nil {
+		return errors.Wrap(err, "marshaling metadata")
+	}
+
+	if err = compressedWriter.Close(); err != nil {
 		return err
 	}
 
@@ -108,7 +116,7 @@ func emitMetadata(metadata *provider.AzureApiMetadata, outDir string) error {
 
 	err = ioutil.WriteFile("./provider/cmd/pulumi-resource-azurerm/metadata.go", []byte(fmt.Sprintf(`package main
 var azureApiResources = %#v
-`, raw)), 0600)
+`, compressedMeta.Bytes())), 0600)
 	if err != nil {
 		return err
 	}
@@ -120,14 +128,19 @@ func emitSchemaBytes(pkgSpec schema.PackageSpec, version string) error {
 	// Ensure the spec is stamped with a version.
 	pkgSpec.Version = version
 
-	bytes, err := json.Marshal(pkgSpec)
+	compressedSchema := bytes.Buffer{}
+	compressedWriter := gzip.NewWriter(&compressedSchema)
+	err := json.NewEncoder(compressedWriter).Encode(pkgSpec)
 	if err != nil {
+		return errors.Wrap(err, "marshaling metadata")
+	}
+	if err = compressedWriter.Close(); err != nil {
 		return err
 	}
 
 	return ioutil.WriteFile("./provider/cmd/pulumi-resource-azurerm/schema.go", []byte(fmt.Sprintf(`package main
 var pulumiSchema = %#v
-`, bytes)), 0600)
+`, compressedSchema.Bytes())), 0600)
 }
 
 func generate(ppkg *schema.Package, language string) (map[string][]byte, error) {

provider/pkg/provider/provider.go

@@ -15,6 +15,8 @@
 package provider
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -66,21 +68,40 @@ func makeProvider(host *provider.HostClient, name, version string, schemaBytes [
 	// Set a long timeout of 2 hours for now.
 	client.PollingDuration = 120 * time.Minute
 
-	var resourceMap *AzureApiMetadata
-	err := json.Unmarshal(azureApiResourcesBytes, &resourceMap)
+	var resourceMap AzureApiMetadata
+
+	uncompressed, err := gzip.NewReader(bytes.NewReader(azureApiResourcesBytes))
 	if err != nil {
+		return nil, errors.Wrap(err, "expand compressed metadata")
+	}
+	if err = json.NewDecoder(uncompressed).Decode(&resourceMap); err != nil {
 		return nil, errors.Wrap(err, "unmarshalling resource map")
 	}
+	if err = uncompressed.Close(); err != nil {
+		return nil, errors.Wrap(err, "closing uncompress stream for metadata")
+	}
+
+	uncompressed, err = gzip.NewReader(bytes.NewReader(schemaBytes))
+	if err != nil {
+		return nil, errors.Wrap(err, "expand compressed schema")
+	}
+
+	buf := bytes.Buffer{}
+	buf.ReadFrom(uncompressed)
+
+	if err = uncompressed.Close(); err != nil {
+		return nil, errors.Wrap(err, "closing uncompress stream for schema")
+	}
 
 	// Return the new provider
 	return &azurermProvider{
 		host:        host,
 		name:        name,
 		version:     version,
 		client:      client,
-		resourceMap: resourceMap,
+		resourceMap: &resourceMap,
 		config:      map[string]string{},
-		schemaBytes: schemaBytes,
+		schemaBytes: buf.Bytes(),
 	}, nil
 }