Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gzip schema and metadata to avoid ballooning memory requirements #139

Merged
merged 2 commits into from
Aug 26, 2020
Merged

Gzip schema and metadata to avoid ballooning memory requirements #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0aaf4d0
Gzip schema and metadata to avoid ballooning memory requirements
Aug 26, 2020
1f4defb
Formatting
Aug 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions provider/cmd/pulumi-gen-azurerm/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package main

import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io/ioutil"
Expand Down Expand Up @@ -96,8 +98,14 @@ func emitSchema(pkgSpec schema.PackageSpec, outDir string) error {
}

func emitMetadata(metadata *provider.AzureApiMetadata, outDir string) error {
raw, err := json.Marshal(metadata)
compressedMeta := bytes.Buffer{}
compressedWriter := gzip.NewWriter(&compressedMeta)
err := json.NewEncoder(compressedWriter).Encode(metadata)
if err != nil {
return errors.Wrap(err, "marshaling metadata")
}

if err = compressedWriter.Close(); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if we return the error above? Should we close in defer instead?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was initially considering a defered close but a the close actually performs a final flush so we need it to be flushed before writing the serialized content. Also, if we return early, GC should be able to reclaim the buffers created earlier.

return err
}

Expand All @@ -108,7 +116,7 @@ func emitMetadata(metadata *provider.AzureApiMetadata, outDir string) error {

err = ioutil.WriteFile("./provider/cmd/pulumi-resource-azurerm/metadata.go", []byte(fmt.Sprintf(`package main
var azureApiResources = %#v
`, raw)), 0600)
`, compressedMeta.Bytes())), 0600)
if err != nil {
return err
}
Expand All @@ -120,14 +128,19 @@ func emitSchemaBytes(pkgSpec schema.PackageSpec, version string) error {
// Ensure the spec is stamped with a version.
pkgSpec.Version = version

bytes, err := json.Marshal(pkgSpec)
compressedSchema := bytes.Buffer{}
compressedWriter := gzip.NewWriter(&compressedSchema)
err := json.NewEncoder(compressedWriter).Encode(pkgSpec)
if err != nil {
return errors.Wrap(err, "marshaling metadata")
}
if err = compressedWriter.Close(); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to make a helper method here somewhere in utils? I don't have a string opinion though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can consider it as a subsequent step if this seems like a sufficient workaround. I have a feeling we might have to do some more maneuvering before a release here.

return err
}

return ioutil.WriteFile("./provider/cmd/pulumi-resource-azurerm/schema.go", []byte(fmt.Sprintf(`package main
var pulumiSchema = %#v
`, bytes)), 0600)
`, compressedSchema.Bytes())), 0600)
}

func generate(ppkg *schema.Package, language string) (map[string][]byte, error) {
Expand Down
29 changes: 25 additions & 4 deletions provider/pkg/provider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package provider

import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -66,21 +68,40 @@ func makeProvider(host *provider.HostClient, name, version string, schemaBytes [
// Set a long timeout of 2 hours for now.
client.PollingDuration = 120 * time.Minute

var resourceMap *AzureApiMetadata
err := json.Unmarshal(azureApiResourcesBytes, &resourceMap)
var resourceMap AzureApiMetadata

uncompressed, err := gzip.NewReader(bytes.NewReader(azureApiResourcesBytes))
if err != nil {
return nil, errors.Wrap(err, "expand compressed metadata")
}
if err = json.NewDecoder(uncompressed).Decode(&resourceMap); err != nil {
return nil, errors.Wrap(err, "unmarshalling resource map")
}
if err = uncompressed.Close(); err != nil {
return nil, errors.Wrap(err, "closing uncompress stream for metadata")
}

uncompressed, err = gzip.NewReader(bytes.NewReader(schemaBytes))
if err != nil {
return nil, errors.Wrap(err, "expand compressed schema")
}

buf := bytes.Buffer{}
buf.ReadFrom(uncompressed)

if err = uncompressed.Close(); err != nil {
return nil, errors.Wrap(err, "closing uncompress stream for schema")
}

// Return the new provider
return &azurermProvider{
host: host,
name: name,
version: version,
client: client,
resourceMap: resourceMap,
resourceMap: &resourceMap,
config: map[string]string{},
schemaBytes: schemaBytes,
schemaBytes: buf.Bytes(),
}, nil
}

Expand Down