ovsdb: Add support for online schema conversion.

With this change, "ovsdb-client convert" can be used to convert a database
from one schema to another without taking the database offline.

This can be useful to minimize downtime for a database during a software
upgrade.

Signed-off-by: Ben Pfaff <blp@ovn.org>
Acked-by: Justin Pettit <jpettit@ovn.org>

Documentation/ref/ovsdb-server.7.rst

@@ -414,6 +414,41 @@ The reply is always the same::
     "error": null
     "id": same "id" as request
 
+4.1.17 Schema Conversion
+------------------------
+
+Open vSwitch 2.9 adds a new JSON-RPC request to convert an online database from
+one schema to another.  The request contains the following members::
+
+    "method": "convert"
+    "params": [<db-name>, <database-schema>]
+    "id": <nonnull-json-value>
+
+Upon receipt, the server converts database <db-name> to schema
+<database-schema>.  The schema's name must be <db-name>.  The conversion is
+atomic, consistent, isolated, and durable.  The data in the database must be
+valid when interpreted under <database-schema>, with only one exception: data
+for tables and columns that do not exist in the new schema are ignored.
+Columns that exist in <database-schema> but not in the database are set to
+their default values.  All of the new schema's constraints apply in full.
+
+If the conversion is successful, the server notifies clients that use the
+``set_db_change_aware`` RPC introduced in Open vSwitch 2.9 and cancels their
+outstanding transactions and monitors.  The server disconnects other clients,
+enabling them to notice the change when they reconnect.  The server sends the
+following reply::
+
+    "result": {}
+    "error": null
+    "id": same "id" as request
+
+If the conversion fails, then the server sends an error reply in the following
+form::
+
+    "result": null
+    "error": [<error>]
+    "id": same "id" as request
+
 5.1 Notation
 ------------
 

Documentation/ref/ovsdb.7.rst

@@ -367,10 +367,17 @@ active-backup database, first stop the database server or servers, then use
 ``ovsdb-tool convert`` to convert it to the new schema, and then restart the
 database server.
 
+OVSDB also supports online database schema conversion.
+To convert a database online, use ``ovsdb-client convert``.
+The conversion is atomic, consistent, isolated, and durable.  ``ovsdb-server``
+disconnects any clients connected when the conversion takes place (except
+clients that use the ``set_db_change_aware`` Open vSwitch extension RPC).  Upon
+reconnection, clients will discover that the schema has changed.
+
 Schema versions and checksums (see Schemas_ above) can give hints about whether
 a database needs to be converted to a new schema.  If there is any question,
-though, the ``needs-conversion`` command on ``ovsdb-tool`` can provide a
-definitive answer.
+though, the ``needs-conversion`` command on ``ovsdb-tool`` and ``ovsdb-client``
+can provide a definitive answer.
 
 Working with Database History
 -----------------------------

NEWS

@@ -26,6 +26,8 @@ v2.9.0 - 19 Feb 2018
      * New high-level documentation in ovsdb(7).
      * New file format documentation for developers in ovsdb(5).
      * Protocol documentation moved from ovsdb-server(1) to ovsdb-server(7).
+     * ovsdb-server now supports online schema conversion via
+       "ovsdb-client convert".
      * ovsdb-server now always hosts a built-in database named _Server.  See
        ovsdb-server(5) for more details.
      * ovsdb-client: New "get-schema-cksum" and "query" commands.

lib/ovsdb-data.c

@@ -1684,6 +1684,19 @@ ovsdb_datum_from_smap(struct ovsdb_datum *datum, const struct smap *smap)
     ovsdb_datum_sort_unique(datum, OVSDB_TYPE_STRING, OVSDB_TYPE_STRING);
 }
 
+struct ovsdb_error * OVS_WARN_UNUSED_RESULT
+ovsdb_datum_convert(struct ovsdb_datum *dst,
+                    const struct ovsdb_type *dst_type,
+                    const struct ovsdb_datum *src,
+                    const struct ovsdb_type *src_type)
+{
+    struct json *json = ovsdb_datum_to_json(src, src_type);
+    struct ovsdb_error *error = ovsdb_datum_from_json(dst, dst_type, json,
+                                                      NULL);
+    json_destroy(json);
+    return error;
+}
+
 static uint32_t
 hash_atoms(enum ovsdb_atomic_type type, const union ovsdb_atom *atoms,
            unsigned int n, uint32_t basis)

lib/ovsdb-data.h

@@ -192,6 +192,12 @@ void ovsdb_datum_to_bare(const struct ovsdb_datum *,
 
 void ovsdb_datum_from_smap(struct ovsdb_datum *, const struct smap *);
 
+struct ovsdb_error *ovsdb_datum_convert(struct ovsdb_datum *dst,
+                                        const struct ovsdb_type *dst_type,
+                                        const struct ovsdb_datum *src,
+                                        const struct ovsdb_type *src_type)
+    OVS_WARN_UNUSED_RESULT;
+
 /* Comparison. */
 uint32_t ovsdb_datum_hash(const struct ovsdb_datum *,
                           const struct ovsdb_type *, uint32_t basis);

ovsdb/file.c

@@ -570,23 +570,31 @@ ovsdb_file_txn_annotate(struct json *json, const char *comment)
     return json;
 }
 
-struct ovsdb_error *
-ovsdb_file_commit(struct ovsdb_file *file,
-                  const struct ovsdb_txn *txn, bool durable)
+/* Returns 'txn' transformed into the JSON format that is used in OVSDB files.
+ * (But the caller must use ovsdb_file_txn_annotate() to add the _comment and
+ * _date members.)  If 'txn' doesn't actually change anything, returns NULL */
+static struct json *
+ovsdb_file_txn_to_json(const struct ovsdb_txn *txn)
 {
     struct ovsdb_file_txn ftxn;
-    struct ovsdb_error *error;
-    long long int current_time;
 
     ovsdb_file_txn_init(&ftxn);
     ovsdb_txn_for_each_change(txn, ovsdb_file_change_cb, &ftxn);
-    if (!ftxn.json) {
+    return ftxn.json;
+}
+
+struct ovsdb_error *
+ovsdb_file_commit(struct ovsdb_file *file,
+                  const struct ovsdb_txn *txn, bool durable)
+{
+    struct json *txn_json = ovsdb_file_txn_to_json(txn);
+    if (!txn_json) {
         /* Nothing to commit. */
         return NULL;
     }
 
-    error = ovsdb_file_txn_commit(ftxn.json, ovsdb_txn_get_comment(txn),
-                                  durable, file->log);
+    struct ovsdb_error *error = ovsdb_file_txn_commit(
+        txn_json, ovsdb_txn_get_comment(txn), durable, file->log);
     if (error) {
         return error;
     }
@@ -599,7 +607,7 @@ ovsdb_file_commit(struct ovsdb_file *file,
      * size of the previous snapshot, then compact the database. However, if
      * it has been over COMPACT_MAX_MSEC ms since the last compaction, the
      * database size is not taken into account. */
-    current_time = time_msec();
+    long long int current_time = time_msec();
     if (current_time >= file->next_compact
         && file->n_transactions >= 100
         && (current_time - file->last_compact >= COMPACT_MAX_MSEC
@@ -857,3 +865,115 @@ ovsdb_file_txn_commit(struct json *json, const char *comment,
 
     return NULL;
 }
+
+static struct ovsdb_error * OVS_WARN_UNUSED_RESULT
+ovsdb_convert_table(struct ovsdb_txn *txn,
+                    const struct ovsdb_table *src_table,
+                    struct ovsdb_table *dst_table)
+{
+    const struct ovsdb_row *src_row;
+    HMAP_FOR_EACH (src_row, hmap_node, &src_table->rows) {
+        struct ovsdb_row *dst_row = ovsdb_row_create(dst_table);
+        *ovsdb_row_get_uuid_rw(dst_row) = *ovsdb_row_get_uuid(src_row);
+
+        struct shash_node *node;
+        SHASH_FOR_EACH (node, &src_table->schema->columns) {
+            const struct ovsdb_column *src_column = node->data;
+            if (src_column->index == OVSDB_COL_UUID ||
+                src_column->index == OVSDB_COL_VERSION) {
+                continue;
+            }
+
+            const struct ovsdb_column *dst_column
+                = shash_find_data(&dst_table->schema->columns,
+                                  src_column->name);
+            if (!dst_column) {
+                continue;
+            }
+
+            struct ovsdb_error *error = ovsdb_datum_convert(
+                &dst_row->fields[dst_column->index], &dst_column->type,
+                &src_row->fields[src_column->index], &src_column->type);
+            if (error) {
+                ovsdb_row_destroy(dst_row);
+                return error;
+            }
+        }
+
+        ovsdb_txn_row_insert(txn, dst_row);
+    }
+    return NULL;
+}
+
+struct ovsdb_error * OVS_WARN_UNUSED_RESULT
+ovsdb_file_convert(const struct ovsdb_file *file,
+                   const struct ovsdb_schema *new_schema)
+{
+    struct ovsdb *new_db = ovsdb_create(ovsdb_schema_clone(new_schema));
+    struct ovsdb_txn *txn = ovsdb_txn_create(new_db);
+    struct ovsdb_error *error = NULL;
+
+    struct shash_node *node;
+    SHASH_FOR_EACH (node, &file->db->tables) {
+        const char *table_name = node->name;
+        const struct ovsdb_table *src_table = node->data;
+        struct ovsdb_table *dst_table = shash_find_data(&new_db->tables,
+                                                        table_name);
+        if (!dst_table) {
+            continue;
+        }
+
+        error = ovsdb_convert_table(txn, src_table, dst_table);
+        if (error) {
+            goto error;
+        }
+    }
+
+    error = ovsdb_txn_start_commit(txn);
+    if (error) {
+        goto error;
+    }
+
+    struct ovsdb_log *new;
+    error = ovsdb_log_replace_start(file->log, &new);
+    if (error) {
+        goto error;
+    }
+
+    /* Write schema. */
+    struct json *schema_json = ovsdb_schema_to_json(new_schema);
+    error = ovsdb_log_write(new, schema_json);
+    json_destroy(schema_json);
+    if (error) {
+        goto error;
+    }
+
+    /* Write data. */
+    struct json *txn_json = ovsdb_file_txn_to_json(txn);
+    if (txn_json) {
+        error = ovsdb_log_write(new, txn_json);
+        json_destroy(txn_json);
+        if (error) {
+            goto error;
+        }
+    }
+
+    error = ovsdb_log_replace_commit(file->log, new);
+    if (error) {
+        goto error;
+    }
+
+    error = ovsdb_txn_finish_commit(txn, true);
+    ovs_assert(!error);         /* Can't happen. */
+
+    ovsdb_replace(file->db, new_db);
+
+    return NULL;
+
+error:
+    ovsdb_destroy(new_db);
+    if (txn) {
+        ovsdb_txn_abort(txn);
+    }
+    return error;
+}

ovsdb/file.h

@@ -51,4 +51,8 @@ void ovsdb_file_destroy(struct ovsdb_file *);
 
 struct json *ovsdb_file_txn_annotate(struct json *, const char *comment);
 
+struct ovsdb_error *ovsdb_file_convert(const struct ovsdb_file *,
+                                       const struct ovsdb_schema *)
+    OVS_WARN_UNUSED_RESULT;
+
 #endif /* ovsdb/file.h */