Ensure Directories are watchable with etcd

etcd v2.0+ with API v2 cannot watch on a regular key, we must ensure
that the initial steps involve creating a 'key' under a directory at
'path'. This is a workaround to recursively create the path and make
sure that 'path' is considered as a watchable directory.

These steps are necessary unless 'libkv' includes and supports the
new etcd API v3 which removes the directory/key distinction to allow
for a better abstraction.

Fixes: moby#392
See: docker/libkv#20

Signed-off-by: Alexandre Beslic <abronan@docker.com>

store.go

@@ -152,7 +152,7 @@ func (c *controller) watchNetworks() error {
 	c.Unlock()
 
 	networkKey := datastore.Key(datastore.NetworkKeyPrefix)
-	if err := ensureKeys(networkKey, cs); err != nil {
+	if err := ensureWatchable(networkKey, cs); err != nil {
 		return fmt.Errorf("failed to ensure if the network keys are valid and present in store: %v", err)
 	}
 	nwPairs, err := cs.KVStore().WatchTree(networkKey, nil)
@@ -209,7 +209,7 @@ func (n *network) watchEndpoints() error {
 	n.Unlock()
 
 	endpointKey := datastore.Key(tmp.KeyPrefix()...)
-	if err := ensureKeys(endpointKey, cs); err != nil {
+	if err := ensureWatchable(endpointKey, cs); err != nil {
 		return fmt.Errorf("failed to ensure if the endpoint keys are valid and present in store: %v", err)
 	}
 	epPairs, err := cs.KVStore().WatchTree(endpointKey, stopCh)
@@ -330,15 +330,51 @@ func (c *controller) processEndpointUpdate(ep *endpoint) bool {
 	return false
 }
 
-func ensureKeys(key string, cs datastore.DataStore) error {
-	exists, err := cs.KVStore().Exists(key)
+// ensureWatchable ensures that the key named 'path'
+// is watchable by the backend store client.
+//
+// FIXME because etcd cannot watch on a regular key,
+// we create a key under a directory structure first
+// as a workaround. We then delete this key. This
+// ensures that the directory gets created and is
+// considered as such.
+func ensureWatchable(path string, cs datastore.DataStore) error {
+	// If the path exists, it must be watchable
+	// from a precedent creation
+	exists, err := cs.KVStore().Exists(path)
 	if err != nil {
-		return err
+		return fmt.Errorf("error trying to verify if key %v exists in store", path)
 	}
 	if exists {
 		return nil
 	}
-	return cs.KVStore().Put(key, []byte{}, nil)
+
+	// If not, we create a key under 'path' to create
+	// the directories recursively, we then delete
+	// the key as this was just a placeholder
+	initKey := path + "init"
+	if err = cs.KVStore().Put(initKey, []byte{}, nil); err != nil {
+		return fmt.Errorf("cannot initialize key %v to be watchable", initKey)
+	}
+	if err = cs.KVStore().Delete(initKey); err != nil {
+		return fmt.Errorf("cannot delete %v after initializing key %v to be watchable", initKey, path)
+	}
+
+	// If 'path' still does not exist, this means that
+	// the store backend used does not have any notion
+	// of key and directory. We need to create a
+	// standalone key to watch
+	exists, err = cs.KVStore().Exists(path)
+	if err != nil {
+		return fmt.Errorf("error trying to verify if key %v exists in store", path)
+	}
+	if !exists {
+		if err = cs.KVStore().Put(path, []byte{}, nil); err != nil {
+			return fmt.Errorf("cannot initialize key %v to be watchable", initKey)
+		}
+	}
+
+	return nil
 }
 
 func (c *controller) getLocalStoreConfig(cfg *config.Config) *config.DatastoreCfg {