diff --git a/ignition/provider.go b/ignition/provider.go index 91b79576..921cf611 100644 --- a/ignition/provider.go +++ b/ignition/provider.go @@ -6,12 +6,15 @@ import ( "encoding/json" "fmt" "log" + "os" + "path/filepath" "sync" "github.com/coreos/ignition/config/v2_1/types" "github.com/coreos/ignition/config/validate/report" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/terraform" + "github.com/peterbourgon/diskv" ) // globalCache keeps the instances of the internal types of ignition generated @@ -46,10 +49,22 @@ func Provider() terraform.ResourceProvider { "ignition_user": dataSourceUser(), "ignition_group": dataSourceGroup(), }, + ConfigureFunc: providerConfigure, } } +func providerConfigure(d *schema.ResourceData) (interface{}, error) { + basePath := filepath.Join(os.TempDir(), "terraform-provider-ignition") + + globalCache.d = diskv.New(diskv.Options{ + BasePath: basePath, + CacheSizeMax: 100 * 1024 * 1024, // 100MB + }) + return nil, nil +} + type cache struct { + // quick retrieval without marshal/unmarshal overhead disks map[string]*types.Disk arrays map[string]*types.Raid filesystems map[string]*types.Filesystem @@ -61,6 +76,9 @@ type cache struct { users map[string]*types.PasswdUser groups map[string]*types.PasswdGroup + // cached retrieval from disk and marshal/unmarshal overhead + d *diskv.Diskv + sync.Mutex } @@ -68,105 +86,296 @@ func (c *cache) addDisk(g *types.Disk) string { c.Lock() defer c.Unlock() - id := id(g) + id, raw := id(g) c.disks[id] = g - + c.d.Write(keyFromType("disks", id), raw) return id } +func (c *cache) getDisk(id string) (*types.Disk, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.disks[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("disks", id)) + if err != nil { + return nil, err + } + out := &types.Disk{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.disks[id] = out + return out, nil +} + func (c *cache) addRaid(r *types.Raid) string { c.Lock() defer c.Unlock() - id := id(r) + id, raw := id(r) c.arrays[id] = r - + c.d.Write(keyFromType("arrays", id), raw) return id } +func (c *cache) getRaid(id string) (*types.Raid, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.arrays[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("arrays", id)) + if err != nil { + return nil, err + } + out := &types.Raid{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.arrays[id] = out + return out, nil +} + func (c *cache) addFilesystem(f *types.Filesystem) string { c.Lock() defer c.Unlock() - id := id(f) + id, raw := id(f) c.filesystems[id] = f - + c.d.Write(keyFromType("fs", id), raw) return id } +func (c *cache) getFilesystem(id string) (*types.Filesystem, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.filesystems[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("fs", id)) + if err != nil { + return nil, err + } + out := &types.Filesystem{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.filesystems[id] = out + return out, nil +} + func (c *cache) addFile(f *types.File) string { c.Lock() defer c.Unlock() - id := id(f) + id, raw := id(f) c.files[id] = f - + c.d.Write(keyFromType("files", id), raw) return id } +func (c *cache) getFile(id string) (*types.File, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.files[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("files", id)) + if err != nil { + return nil, err + } + out := &types.File{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.files[id] = out + return out, nil +} + func (c *cache) addDirectory(d *types.Directory) string { c.Lock() defer c.Unlock() - id := id(d) + id, raw := id(d) c.directories[id] = d - + c.d.Write(keyFromType("dirs", id), raw) return id } +func (c *cache) getDirectory(id string) (*types.Directory, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.directories[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("dirs", id)) + if err != nil { + return nil, err + } + out := &types.Directory{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.directories[id] = out + return out, nil +} + func (c *cache) addLink(l *types.Link) string { c.Lock() defer c.Unlock() - id := id(l) + id, raw := id(l) c.links[id] = l - + c.d.Write(keyFromType("links", id), raw) return id } +func (c *cache) getLink(id string) (*types.Link, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.links[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("links", id)) + if err != nil { + return nil, err + } + out := &types.Link{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.links[id] = out + return out, nil +} + func (c *cache) addSystemdUnit(u *types.Unit) string { c.Lock() defer c.Unlock() - id := id(u) + id, raw := id(u) c.systemdUnits[id] = u - + c.d.Write(keyFromType("sunits", id), raw) return id } +func (c *cache) getSystemdUnit(id string) (*types.Unit, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.systemdUnits[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("sunits", id)) + if err != nil { + return nil, err + } + out := &types.Unit{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.systemdUnits[id] = out + return out, nil +} + func (c *cache) addNetworkdUnit(u *types.Networkdunit) string { c.Lock() defer c.Unlock() - id := id(u) + id, raw := id(u) c.networkdUnits[id] = u - + c.d.Write(keyFromType("nunits", id), raw) return id } +func (c *cache) getNetworkdunit(id string) (*types.Networkdunit, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.networkdUnits[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("nunits", id)) + if err != nil { + return nil, err + } + out := &types.Networkdunit{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.networkdUnits[id] = out + return out, nil +} + func (c *cache) addUser(u *types.PasswdUser) string { c.Lock() defer c.Unlock() - id := id(u) + id, raw := id(u) c.users[id] = u - + c.d.Write(keyFromType("users", id), raw) return id } +func (c *cache) getUser(id string) (*types.PasswdUser, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.users[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("users", id)) + if err != nil { + return nil, err + } + out := &types.PasswdUser{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.users[id] = out + return out, nil +} + func (c *cache) addGroup(g *types.PasswdGroup) string { c.Lock() defer c.Unlock() - id := id(g) + id, raw := id(g) c.groups[id] = g - + c.d.Write(keyFromType("groups", id), raw) return id } -func id(input interface{}) string { +func (c *cache) getGroup(id string) (*types.PasswdGroup, error) { + c.Lock() + defer c.Unlock() + + if v, ok := c.groups[id]; ok { + return v, nil + } + raw, err := c.d.Read(keyFromType("groups", id)) + if err != nil { + return nil, err + } + out := &types.PasswdGroup{} + if err := json.Unmarshal(raw, out); err != nil { + return nil, err + } + c.groups[id] = out + return out, nil +} + +// id returns the hashed and raw identifier for the input +func id(input interface{}) (string, []byte) { b, _ := json.Marshal(input) - return hash(string(b)) + return hash(string(b)), b } func hash(s string) string { @@ -174,6 +383,8 @@ func hash(s string) string { return hex.EncodeToString(sha[:]) } +func keyFromType(t string, id string) string { return fmt.Sprintf("%s-%s", t, id) } + func castSliceInterface(i []interface{}) []string { var o []string for _, value := range i { diff --git a/ignition/resource_ignition_config.go b/ignition/resource_ignition_config.go index 643ba79a..0e2553a5 100644 --- a/ignition/resource_ignition_config.go +++ b/ignition/resource_ignition_config.go @@ -217,9 +217,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - d, ok := c.disks[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid disk %q, unknown disk id", id) + d, err := c.getDisk(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid disk %q, failed to get disk id: %v", id, err) } storage.Disks = append(storage.Disks, *d) @@ -229,9 +229,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - a, ok := c.arrays[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid raid %q, unknown raid id", id) + a, err := c.getRaid(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid raid %q, failed to get disk id: %v", id, err) } storage.Raid = append(storage.Raid, *a) @@ -241,9 +241,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - f, ok := c.filesystems[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid filesystem %q, unknown filesystem id", id) + f, err := c.getFilesystem(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid filesystem %q, failed to get filesystem id: %v", id, err) } storage.Filesystems = append(storage.Filesystems, *f) @@ -253,9 +253,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - f, ok := c.files[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid file %q, unknown file id", id) + f, err := c.getFile(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid file %q, failed to get file id: %v", id, err) } storage.Files = append(storage.Files, *f) @@ -265,9 +265,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - f, ok := c.directories[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid file %q, unknown directory id", id) + f, err := c.getDirectory(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid file %q, failed to get directory id: %v", id, err) } storage.Directories = append(storage.Directories, *f) @@ -277,9 +277,9 @@ func buildStorage(d *schema.ResourceData, c *cache) (types.Storage, error) { if id == nil { continue } - f, ok := c.links[id.(string)] - if !ok { - return storage, fmt.Errorf("invalid file %q, unknown link id", id) + f, err := c.getLink(id.(string)) + if err != nil { + return storage, fmt.Errorf("invalid file %q, failed to get link id: %v", id, err) } storage.Links = append(storage.Links, *f) @@ -297,9 +297,9 @@ func buildSystemd(d *schema.ResourceData, c *cache) (types.Systemd, error) { continue } - u, ok := c.systemdUnits[id.(string)] - if !ok { - return systemd, fmt.Errorf("invalid systemd unit %q, unknown systemd unit id", id) + u, err := c.getSystemdUnit(id.(string)) + if err != nil { + return systemd, fmt.Errorf("invalid systemd unit %q, failed to get systemd unit id: %v", id, err) } systemd.Units = append(systemd.Units, *u) @@ -317,9 +317,9 @@ func buildNetworkd(d *schema.ResourceData, c *cache) (types.Networkd, error) { continue } - u, ok := c.networkdUnits[id.(string)] - if !ok { - return networkd, fmt.Errorf("invalid networkd unit %q, unknown networkd unit id", id) + u, err := c.getNetworkdunit(id.(string)) + if err != nil { + return networkd, fmt.Errorf("invalid networkd unit %q, failed to get networkd unit id: %v", id, err) } networkd.Units = append(networkd.Units, *u) @@ -336,9 +336,9 @@ func buildPasswd(d *schema.ResourceData, c *cache) (types.Passwd, error) { continue } - u, ok := c.users[id.(string)] - if !ok { - return passwd, fmt.Errorf("invalid user %q, unknown user id", id) + u, err := c.getUser(id.(string)) + if err != nil { + return passwd, fmt.Errorf("invalid user %q, failed to get user id: %v", id, err) } passwd.Users = append(passwd.Users, *u) @@ -349,9 +349,9 @@ func buildPasswd(d *schema.ResourceData, c *cache) (types.Passwd, error) { continue } - g, ok := c.groups[id.(string)] - if !ok { - return passwd, fmt.Errorf("invalid group %q, unknown group id", id) + g, err := c.getGroup(id.(string)) + if err != nil { + return passwd, fmt.Errorf("invalid group %q, failed to get group id: %v", id, err) } passwd.Groups = append(passwd.Groups, *g)