plugins/sql: handle sub-objects.

The only one is `option_will_fund`, so flatten it that into the same
table.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

doc/lightning-sql.7.md

@@ -58,6 +58,8 @@ TABLES
 [comment]: # (GENERATE-DOC-START)
 Note that most column names reflect the JSON field names, with the exception of `index` which is renamed to `idx` to avoid using an SQL keyword.
 
+Sub-arrays are represented in their own tables, and sub-objects are flattened with `_`.
+
 The following tables are currently supported:
 - channels indexed by `short_channel_id` (see lightning-listchannels(7))
   - `source` (type `pubkey`, sqltype `BLOB`)
@@ -128,6 +130,12 @@ The following tables are currently supported:
     - `arrindex` (index within array, sqltype `INTEGER`)
     - `type` (type `string`, sqltype `TEXT`)
     - `port` (type `u16`, sqltype `INTEGER`)
+  - `option_will_fund_lease_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)
+  - `option_will_fund_lease_fee_basis` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)
+  - `option_will_fund_funding_weight` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)
+  - `option_will_fund_channel_fee_max_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)
+  - `option_will_fund_channel_fee_max_proportional_thousandths` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)
+  - `option_will_fund_compact_lease` (type `hex`, sqltype `BLOB`, from JSON object `option_will_fund`)
 
 - offers indexed by `offer_id` (see lightning-listoffers(7))
   - `offer_id` (type `hex`, sqltype `BLOB`)
@@ -262,4 +270,4 @@ RESOURCES
 ---------
 
 Main web site: <https://github.com/ElementsProject/lightning>
-[comment]: # ( SHA256STAMP:9b553b45cf31bc46fb4158f2f913bfb3aab1c4c6ffcf2a9e59664f9a10ec1ac6)
+[comment]: # ( SHA256STAMP:03aee4b1270fce52618a6261f79308927e18df1f490df0da3972840d7f9df1cf)

plugins/sql.c

@@ -157,6 +157,8 @@ struct table_desc {
 	char *update_stmt;
 	/* If we are a subtable */
 	struct table_desc *parent;
+	/* Is this a sub object (otherwise, subarray if parent is true) */
+	bool is_subobject;
 	/* function to refresh it. */
 	struct command_result *(*refresh)(struct command *cmd,
 					  const struct table_desc *td,
@@ -409,11 +411,25 @@ static struct command_result *process_json_obj(struct command *cmd,
 		const struct column *col = &td->columns[i];
 		const jsmntok_t *coltok;
 
-		/* Handle sub-tables below: we need rowid! */
-		if (col->sub)
+		if (col->sub) {
+			struct command_result *ret;
+			/* Handle sub-tables below: we need rowid! */
+			if (!col->sub->is_subobject)
+				continue;
+
+			coltok = json_get_member(buf, t, col->jsonname);
+			ret = process_json_obj(cmd, buf, coltok, col->sub, row, rowid,
+					       sqloff, stmt);
+			if (ret)
+				return ret;
 			continue;
+		}
+
+		if (!t)
+			coltok = NULL;
+		else
+			coltok = json_get_member(buf, t, col->jsonname);
 
-		coltok = json_get_member(buf, t, col->jsonname);
 		if (!coltok)
 			sqlite3_bind_null(stmt, (*sqloff)++);
 		else {
@@ -494,6 +510,10 @@ static struct command_result *process_json_obj(struct command *cmd,
 		}
 	}
 
+	/* Sub objects get folded into parent's SQL */
+	if (td->parent && td->is_subobject)
+		return NULL;
+
 	err = sqlite3_step(stmt);
 	if (err != SQLITE_DONE) {
 		return command_fail(cmd, LIGHTNINGD,
@@ -512,7 +532,7 @@ static struct command_result *process_json_obj(struct command *cmd,
 		sqlite3_stmt *substmt;
 		size_t suboff = 1;
 
-		if (!col->sub)
+		if (!col->sub || col->sub->is_subobject)
 			continue;
 
 		/* We turned index -> idx, so translate back! */
@@ -1007,7 +1027,8 @@ static const char *finish_td(const tal_t *ctx, struct table_desc *td)
 	char *errmsg;
 	const char *sep = "";
 
-	if (!td)
+	/* subobject are separate at JSON level, folded at db level! */
+	if (!td || td->is_subobject)
 		return NULL;
 
 	if (ignore_column(td->name)) {
@@ -1031,9 +1052,22 @@ static const char *finish_td(const tal_t *ctx, struct table_desc *td)
 	for (size_t i = 0; i < tal_count(td->columns); i++) {
 		const struct column *col = &td->columns[i];
 
-		/* Has its own table. */
-		if (col->sub)
+		if (col->sub) {
+			/* sub-arrays are a completely separate table. */
+			if (!col->sub->is_subobject)
+				continue;
+			/* sub-objects are folded into this table. */
+			for (size_t j = 0; j < tal_count(col->sub->columns); j++) {
+				const struct column *subcol = &col->sub->columns[j];
+				tal_append_fmt(&td->update_stmt, "%s?", sep);
+				tal_append_fmt(&create_stmt, "%s%s %s",
+					       sep,
+					       subcol->dbname,
+					       fieldtypemap[subcol->ftype].sqltype);
+				sep = ",";
+			}
 			continue;
+		}
 		tal_append_fmt(&td->update_stmt, "%s?", sep);
 		tal_append_fmt(&create_stmt, "%s%s %s",
 			       sep,
@@ -1065,6 +1099,7 @@ static const char *db_column_name(const tal_t *ctx, const char *name)
 static const char *start_new_table_desc(const tal_t *ctx,
 					struct table_desc **tdp,
 					struct table_desc *parent,
+					bool is_subobject,
 					const char *name TAKES,
 					const char *arrname TAKES,
 					struct command_result *
@@ -1081,6 +1116,7 @@ static const char *start_new_table_desc(const tal_t *ctx,
 	*tdp = tal(NULL, struct table_desc);
 	(*tdp)->name = tal_strdup(*tdp, name);
 	(*tdp)->parent = parent;
+	(*tdp)->is_subobject = is_subobject;
 	(*tdp)->arrname = tal_strdup(*tdp, arrname);
 	(*tdp)->columns = tal_arr(*tdp, struct column, 0);
 	(*tdp)->refresh = refresh;
@@ -1111,7 +1147,7 @@ static const char *init_tablemap(const tal_t *ctx)
 			if (!streq(sub[2], "array"))
 				continue;
 
-			err = start_new_table_desc(ctx, &td, NULL, sub[0], sub[1],
+			err = start_new_table_desc(ctx, &td, NULL, false, sub[0], sub[1],
 						   streq(sub[0], "channels") ? channels_refresh
 						   : streq(sub[0], "nodes") ? nodes_refresh
 						   : default_refresh);
@@ -1120,12 +1156,16 @@ static const char *init_tablemap(const tal_t *ctx)
 			continue;
 		}
 
-		/* Sub array? */
-		if (num == 5 && streq(type, "array")) {
+		/* Sub array / object? */
+		if (num == 5 && (streq(type, "array") || streq(type, "object"))) {
 			assert(td);
-			err = start_new_table_desc(ctx, &subtd, td,
-						   take(tal_fmt(NULL, "%s_%s", sub[0], sub[3])),
-						   sub[3], NULL);
+			if (streq(type, "array"))
+				err = start_new_table_desc(ctx, &subtd, td, false,
+							   take(tal_fmt(NULL, "%s_%s", sub[0], sub[3])),
+							   sub[3], NULL);
+			else
+				err = start_new_table_desc(ctx, &subtd, td, true, sub[3], sub[3],
+							   NULL);
 			if (err)
 				return err;
 
@@ -1139,7 +1179,6 @@ static const char *init_tablemap(const tal_t *ctx)
 			continue;
 		}
 
-		/* FIXME: handle sub-objects! */
 		if (streq(type, "object") || streq(type, "array"))
 			continue;
 
@@ -1153,13 +1192,20 @@ static const char *init_tablemap(const tal_t *ctx)
 			continue;
 		}
 
-		/* Direct array of non-objects (e.g peers.netaddr) */
-		if (num == 6 && streq(sub[4], "[]")) {
+		if (num == 6) {
 			assert(subtd);
-			/* Make column name the field name, eg netaddr */
-			col.dbname = db_column_name(subtd->columns, sub[3]);
 			col.ftype = find_fieldtype(type);
 			col.sub = NULL;
+			/* Direct array of non-objects (e.g peers.netaddr) */
+			if (streq(sub[4], "[]")) {
+				/* Make column name the field name, eg netaddr */
+				col.dbname = db_column_name(subtd->columns, sub[3]);
+			} else {
+				/* Sub-objects (e.g nodes.option_will_fund) */
+				col.dbname = tal_fmt(subtd->columns, "%s_%s",
+						     sub[3], sub[4]);
+			}
+			col.jsonname = tal_strdup(subtd->columns, sub[4]);
 			tal_arr_expand(&subtd->columns, col);
 			continue;
 		}
@@ -1276,25 +1322,37 @@ static bool print_one_table(const char *member,
 
 	for (size_t i = 0; i < tal_count(td->columns); i++) {
 		const char *origin;
-		if (!streq(td->columns[i].dbname, td->columns[i].jsonname)) {
-			origin = tal_fmt(tmpctx, ", from JSON field `%s`",
-					 td->columns[i].jsonname);
-		} else
-			origin = "";
 		if (td->columns[i].sub) {
 			const struct table_desc *subtd = td->columns[i].sub;
-			printf("  - subtable `%s`\n", subtd->name);
-			printf("    - `row` (reference to `%s.rowid`, sqltype `INTEGER`)\n"
-			       "    - `arrindex` (index within array, sqltype `INTEGER`)\n",
-			       td->name);
-			for (size_t j = 0; j < tal_count(subtd->columns); j++) {
-				printf("    - `%s` (type `%s`, sqltype `%s`)\n",
-				       subtd->columns[j].dbname,
-				       fieldtypemap[subtd->columns[j].ftype].name,
-				       fieldtypemap[subtd->columns[j].ftype].sqltype);
+
+			if (!subtd->is_subobject) {
+				printf("  - subtable `%s`\n", subtd->name);
+				printf("    - `row` (reference to `%s.rowid`, sqltype `INTEGER`)\n"
+				       "    - `arrindex` (index within array, sqltype `INTEGER`)\n",
+				       td->name);
+				for (size_t j = 0; j < tal_count(subtd->columns); j++) {
+					printf("    - `%s` (type `%s`, sqltype `%s`)\n",
+					       subtd->columns[j].dbname,
+					       fieldtypemap[subtd->columns[j].ftype].name,
+					       fieldtypemap[subtd->columns[j].ftype].sqltype);
+				}
+			} else {
+				for (size_t j = 0; j < tal_count(subtd->columns); j++) {
+					printf("  - `%s` (type `%s`, sqltype `%s`, from JSON object `%s`)\n",
+					       subtd->columns[j].dbname,
+					       fieldtypemap[subtd->columns[j].ftype].name,
+					       fieldtypemap[subtd->columns[j].ftype].sqltype,
+					       td->columns[i].jsonname);
+				}
 			}
 			continue;
 		}
+
+		if (!streq(td->columns[i].dbname, td->columns[i].jsonname)) {
+			origin = tal_fmt(tmpctx, ", from JSON field `%s`",
+					 td->columns[i].jsonname);
+		} else
+			origin = "";
 		printf("  - `%s` (type `%s`, sqltype `%s`%s)\n",
 		       td->columns[i].dbname,
 		       fieldtypemap[td->columns[i].ftype].name,
@@ -1318,6 +1376,7 @@ int main(int argc, char *argv[])
 
 		printf("Note that most column names reflect the JSON field names, with the exception of `index` which is renamed to `%s` to avoid using an SQL keyword.\n\n",
 		       db_column_name(tmpctx, "index"));
+		printf("Sub-arrays are represented in their own tables, and sub-objects are flattened with `_`.\n\n");
 		printf("The following tables are currently supported:\n");
 		strmap_iterate(&tablemap, print_one_table, NULL);
 		common_shutdown();

tests/test_plugin.py

@@ -3239,9 +3239,16 @@ def test_block_added_notifications(node_factory, bitcoind):
 
 
 def test_sql(node_factory, bitcoind):
+    l2opts = {'lease-fee-basis': 50,
+              'lease-fee-base-sat': '2000msat',
+              'channel-fee-max-base-msat': '500sat',
+              'channel-fee-max-proportional-thousandths': 200,
+              'experimental-offers': None,
+              'sqlfilename': 'sql.sqlite3'}
     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-                                         opts={'experimental-offers': None,
-                                               'sqlfilename': 'sql.sqlite3'})
+                                         opts=[{'experimental-offers': None},
+                                               l2opts,
+                                               {'experimental-offers': None}])
 
     ret = l2.rpc.sql("SELECT * FROM forwards;")
     assert ret == {'rows': []}
@@ -3295,6 +3302,18 @@ def test_sql(node_factory, bitcoind):
                         {'name': 'color',
                          'type': 'hex'},
                         {'name': 'features',
+                         'type': 'hex'},
+                        {'name': 'option_will_fund_lease_fee_base_msat',
+                         'type': 'msat'},
+                        {'name': 'option_will_fund_lease_fee_basis',
+                         'type': 'u32'},
+                        {'name': 'option_will_fund_funding_weight',
+                         'type': 'u32'},
+                        {'name': 'option_will_fund_channel_fee_max_base_msat',
+                         'type': 'msat'},
+                        {'name': 'option_will_fund_channel_fee_max_proportional_thousandths',
+                         'type': 'u32'},
+                        {'name': 'option_will_fund_compact_lease',
                          'type': 'hex'}]},
         'forwards': {
             'indices': [['in_channel', 'in_htlc_id']],
@@ -3544,3 +3563,25 @@ def test_sql(node_factory, bitcoind):
     wait_for(lambda: len(l3.rpc.listchannels()['channels']) == 2)
     assert len(l3.rpc.sql("SELECT * FROM channels;")['rows']) == 2
     l3.daemon.wait_for_log("Deleting channel: {}".format(scid))
+
+    # Test subobject case (option_will_fund)
+    ret = l2.rpc.sqlsimple(['option_will_fund_lease_fee_base_msat',
+                            'option_will_fund_lease_fee_basis',
+                            'option_will_fund_funding_weight',
+                            'option_will_fund_channel_fee_max_base_msat',
+                            'option_will_fund_channel_fee_max_proportional_thousandths',
+                            'option_will_fund_compact_lease'],
+                           "FROM nodes WHERE HEX(nodeid) = '{}';".format(l2.info['id'].upper()))
+    optret = only_one(l2.rpc.listnodes(l2.info['id'])['nodes'])['option_will_fund']
+    assert len(only_one(ret['rows'])) == len(optret)
+    for k, v in optret.items():
+        assert ret['rows'][0]['option_will_fund_{}'.format(k)] == v
+
+    # Correctly handles missing object.
+    assert l2.rpc.sqlsimple(['option_will_fund_lease_fee_base_msat',
+                             'option_will_fund_lease_fee_basis',
+                             'option_will_fund_funding_weight',
+                             'option_will_fund_channel_fee_max_base_msat',
+                             'option_will_fund_channel_fee_max_proportional_thousandths',
+                             'option_will_fund_compact_lease'],
+                            "FROM nodes WHERE HEX(nodeid) = '{}';".format(l1.info['id'].upper())) == {'rows': [{}]}