Simple misc cleanups

Makefile

@@ -443,12 +443,15 @@ PYLN_PATH=$(shell pwd)/lightningd:$(PATH)
 check-pyln-%: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@(cd contrib/$(shell echo $@ | cut -b 7-) && PATH=$(PYLN_PATH) PYTHONPATH=$(PYTHONPATH) $(MAKE) check)
 
-check-python: check-pyln-client check-pyln-testing
+check-python: check-python-flake8 check-pytest-pyln-proto check-pyln-client check-pyln-testing
+
+check-python-flake8:
 	@# E501 line too long (N > 79 characters)
 	@# E731 do not assign a lambda expression, use a def
 	@# W503: line break before binary operator
 	@flake8 --ignore=E501,E731,W503 ${PYSRC}
 
+check-pytest-pyln-proto:
 	PATH=$(PYLN_PATH) PYTHONPATH=$(PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/
 
 check-includes: check-src-includes check-hdr-includes

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2502-gb45a3266
+CCAN version: init-2503-g56d5c41f

ccan/ccan/utf8/test/run-decode.c

@@ -117,7 +117,7 @@ test_unicode_scalar_value(void) {
   char src[4];
 
   /* Unicode scalar value [U+0000, U+007F] */
-  for (ord = 0x0000; ord <= 0x007F; ord++) {
+  for (ord = 0x0001; ord <= 0x007F; ord++) {
     encode_ord(ord, 1, src);
     TEST_UTF8(src, 1, ord ? 0 : ERANGE);
   }
@@ -255,7 +255,7 @@ test_continuations(void) {
 int
 main(int argc, char **argv)
 {
-  plan_tests(2190906);
+  plan_tests(2190906 - 1);
   test_unicode_scalar_value();
   test_surrogates();
   test_non_shortest_form();

ccan/ccan/utf8/utf8.c

@@ -63,6 +63,8 @@ bool utf8_decode(struct utf8_state *utf8_state, char c)
 		/* First character in sequence. */
 		if (((unsigned char)c & 0x80) == 0) {
 			/* ASCII, easy. */
+			if (c == 0)
+				goto bad_encoding;
 			utf8_state->total_len = 1;
 			utf8_state->c = c;
 			goto finished_decoding;

ccan/ccan/utf8/utf8.h

@@ -33,7 +33,7 @@ static inline void utf8_state_init(struct utf8_state *utf8_state)
  * Otherwise returns true, @utf8_state can be reused without initializeation,
  * and sets errno:
  * 0: success
- * EINVAL: bad encoding.
+ * EINVAL: bad encoding (including a NUL character).
  * EFBIG: not a minimal encoding.
  * ERANGE: encoding of invalid character.
  *

common/bolt11.c

@@ -173,20 +173,24 @@ static void decode_p(struct bolt11 *b11,
  * `d` (13): `data_length` variable.  Short description of purpose of payment
  * (UTF-8), e.g. '1 cup of coffee' or 'ナンセンス 1杯'
  */
-static void decode_d(struct bolt11 *b11,
-		     struct hash_u5 *hu5,
-		     u5 **data, size_t *data_len,
-		     size_t data_length, bool *have_d)
+static char *decode_d(struct bolt11 *b11,
+		      struct hash_u5 *hu5,
+		      u5 **data, size_t *data_len,
+		      size_t data_length, bool *have_d)
 {
-	if (*have_d) {
-		unknown_field(b11, hu5, data, data_len, 'd', data_length);
-		return;
-	}
+	u8 *desc;
+	if (*have_d)
+		return unknown_field(b11, hu5, data, data_len, 'd', data_length);
+
+	desc = tal_arr(NULL, u8, data_length * 5 / 8);
+	pull_bits_certain(hu5, data, data_len, desc, data_length*5, false);
 
-	b11->description = tal_arrz(b11, char, num_u8(data_length) + 1);
-	pull_bits_certain(hu5, data, data_len, (u8 *)b11->description,
-			  data_length*5, false);
 	*have_d = true;
+	b11->description = utf8_str(b11, take(desc), tal_bytelen(desc));
+	if (b11->description)
+		return NULL;
+
+	return tal_fmt(b11, "d: invalid utf8");
 }
 
 /* BOLT #11:
@@ -721,8 +725,8 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
 			break;
 
 		case 'd':
-			decode_d(b11, &hu5, &data, &data_len, data_length,
-				 &have_d);
+			problem = decode_d(b11, &hu5, &data, &data_len,
+					   data_length, &have_d);
 			break;
 
 		case 'h':

common/json.c

@@ -41,6 +41,28 @@ bool json_tok_streq(const char *buffer, const jsmntok_t *tok, const char *str)
 	return strncmp(buffer + tok->start, str, tok->end - tok->start) == 0;
 }
 
+bool json_tok_startswith(const char *buffer, const jsmntok_t *tok,
+			 const char *prefix)
+{
+	if (tok->type != JSMN_STRING)
+		return false;
+	if (tok->end - tok->start < strlen(prefix))
+		return false;
+	return memcmp(buffer + tok->start,
+		      prefix, strlen(prefix)) == 0;
+}
+
+bool json_tok_endswith(const char *buffer, const jsmntok_t *tok,
+		       const char *suffix)
+{
+	if (tok->type != JSMN_STRING)
+		return false;
+	if (tok->end - tok->start < strlen(suffix))
+		return false;
+	return memcmp(buffer + tok->end - strlen(suffix),
+		      suffix, strlen(suffix)) == 0;
+}
+
 char *json_strdup(const tal_t *ctx, const char *buffer, const jsmntok_t *tok)
 {
 	return tal_strndup(ctx, buffer + tok->start, tok->end - tok->start);
@@ -325,7 +347,7 @@ JSMN Result Validation Starts
  * This part of the code performs some filtering so
  * that at least some of the invalid JSON that
  * LIBJSMN accepts, will be rejected by
- * json_parse_input.
+ * json_parse_input.  It also checks that strings are valid UTF-8.
  */
 
 /*~ These functions are used in JSMN validation.
@@ -360,7 +382,8 @@ JSMN Result Validation Starts
  */
 /* Validate a *single* datum.  */
 static const jsmntok_t *
-validate_jsmn_datum(const jsmntok_t *p,
+validate_jsmn_datum(const char *buf,
+		    const jsmntok_t *p,
 		    const jsmntok_t *end,
 		    bool *valid);
 /*~ Validate a key-value pair.
@@ -384,12 +407,14 @@ validate_jsmn_datum(const jsmntok_t *p,
  * is to reject such improper arrays and objects.
  */
 static const jsmntok_t *
-validate_jsmn_keyvalue(const jsmntok_t *p,
+validate_jsmn_keyvalue(const char *buf,
+		       const jsmntok_t *p,
 		       const jsmntok_t *end,
 		       bool *valid);
 
 static const jsmntok_t *
-validate_jsmn_datum(const jsmntok_t *p,
+validate_jsmn_datum(const char *buf,
+		    const jsmntok_t *p,
 		    const jsmntok_t *end,
 		    bool *valid)
 {
@@ -402,8 +427,11 @@ validate_jsmn_datum(const jsmntok_t *p,
 	}
 
 	switch (p->type) {
-	case JSMN_UNDEFINED:
 	case JSMN_STRING:
+		if (!utf8_check(buf + p->start, p->end - p->start))
+			*valid = false;
+		/* Fall thru */
+	case JSMN_UNDEFINED:
 	case JSMN_PRIMITIVE:
 		/* These types should not have sub-datums.  */
 		if (p->size != 0)
@@ -418,7 +446,7 @@ validate_jsmn_datum(const jsmntok_t *p,
 		++p;
 		for (i = 0; i < sz; ++i) {
 			/* Arrays should only contain standard JSON datums.  */
-			p = validate_jsmn_datum(p, end, valid);
+			p = validate_jsmn_datum(buf, p, end, valid);
 			if (!*valid)
 				break;
 		}
@@ -430,7 +458,7 @@ validate_jsmn_datum(const jsmntok_t *p,
 		++p;
 		for (i = 0; i < sz; ++i) {
 			/* Objects should only contain key-value pairs.  */
-			p = validate_jsmn_keyvalue(p, end, valid);
+			p = validate_jsmn_keyvalue(buf, p, end, valid);
 			if (!*valid)
 				break;
 		}
@@ -445,7 +473,8 @@ validate_jsmn_datum(const jsmntok_t *p,
 }
 /* Key-value pairs *must* be strings with size 1.  */
 static inline const jsmntok_t *
-validate_jsmn_keyvalue(const jsmntok_t *p,
+validate_jsmn_keyvalue(const char *buf,
+		       const jsmntok_t *p,
 		       const jsmntok_t *end,
 		       bool *valid)
 {
@@ -472,13 +501,14 @@ validate_jsmn_keyvalue(const jsmntok_t *p,
 	 * incidentally rejects that non-standard
 	 * JSON.
 	 */
-	if (p->type != JSMN_STRING || p->size != 1) {
+	if (p->type != JSMN_STRING || p->size != 1
+	    || !utf8_check(buf + p->start, p->end - p->start)) {
 		*valid = false;
 		return p;
 	}
 
 	++p;
-	return validate_jsmn_datum(p, end, valid);
+	return validate_jsmn_datum(buf, p, end, valid);
 }
 
 /** validate_jsmn_parse_output
@@ -525,12 +555,13 @@ validate_jsmn_keyvalue(const jsmntok_t *p,
  * `jsmn_parse`, false otherwise.
  */
 static bool
-validate_jsmn_parse_output(const jsmntok_t *p, const jsmntok_t *end)
+validate_jsmn_parse_output(const char *buf,
+			   const jsmntok_t *p, const jsmntok_t *end)
 {
 	bool valid = true;
 
 	while (p < end && valid)
-		p = validate_jsmn_datum(p, end, &valid);
+		p = validate_jsmn_datum(buf, p, end, &valid);
 
 	return valid;
 }
@@ -583,7 +614,7 @@ bool json_parse_input(jsmn_parser *parser,
 	 * element. */
 	ret = json_next(*toks) - *toks;
 
-	if (!validate_jsmn_parse_output(*toks, *toks + ret))
+	if (!validate_jsmn_parse_output(input, *toks, *toks + ret))
 		return false;
 
 	/* Cut to length and return. */
@@ -753,13 +784,19 @@ void json_add_literal(struct json_stream *result, const char *fieldname,
 		memcpy(dest, literal, strlen(literal));
 }
 
-void json_add_string(struct json_stream *result, const char *fieldname, const char *value TAKES)
+void json_add_stringn(struct json_stream *result, const char *fieldname,
+		      const char *value TAKES, size_t value_len)
 {
-	json_add_member(result, fieldname, true, "%s", value);
+	json_add_member(result, fieldname, true, "%.*s", (int)value_len, value);
 	if (taken(value))
 		tal_free(value);
 }
 
+void json_add_string(struct json_stream *result, const char *fieldname, const char *value TAKES)
+{
+	json_add_stringn(result, fieldname, value, strlen(value));
+}
+
 void json_add_bool(struct json_stream *result, const char *fieldname, bool value)
 {
 	json_add_member(result, fieldname, false, value ? "true" : "false");

common/json.h

@@ -26,6 +26,14 @@ int json_tok_full_len(const jsmntok_t *t);
 /* Is this a string equal to str? */
 bool json_tok_streq(const char *buffer, const jsmntok_t *tok, const char *str);
 
+/* Does this string token start with prefix? */
+bool json_tok_startswith(const char *buffer, const jsmntok_t *tok,
+			 const char *prefix);
+
+/* Does this string token end with suffix? */
+bool json_tok_endswith(const char *buffer, const jsmntok_t *tok,
+		       const char *suffix);
+
 /* Allocate a tal string copy */
 char *json_strdup(const tal_t *ctx, const char *buffer, const jsmntok_t *tok);
 
@@ -155,6 +163,13 @@ const jsmntok_t *json_delve(const char *buffer,
  */
 void json_add_string(struct json_stream *result, const char *fieldname, const char *value);
 
+/* '"fieldname" : "value[:value_len]"' or '"value[:value_len]"' if
+ * fieldname is NULL.  Turns any non-printable chars into JSON
+ * escapes, but leaves existing escapes alone.
+ */
+void json_add_stringn(struct json_stream *result, const char *fieldname,
+		      const char *value TAKES, size_t value_len);
+
 /* '"fieldname" : "value"' or '"value"' if fieldname is NULL.  String must
  * already be JSON escaped as necessary. */
 void json_add_escaped_string(struct json_stream *result,

common/json_tok.c

@@ -313,41 +313,28 @@ struct command_result *param_feerate_val(struct command *cmd,
 					 const jsmntok_t *tok,
 					 u32 **feerate_per_kw)
 {
-	jsmntok_t base = *tok, suffix = *tok;
+	jsmntok_t base = *tok;
 	enum feerate_style style;
 	unsigned int num;
 
-	/* We have to split the number and suffix. */
-	suffix.start = suffix.end;
-	while (suffix.start > base.start && !isdigit(buffer[suffix.start-1])) {
-		suffix.start--;
-		base.end--;
-	}
+	if (json_tok_endswith(buffer, tok,
+			      feerate_style_name(FEERATE_PER_KBYTE))) {
+		style = FEERATE_PER_KBYTE;
+		base.end -= strlen(feerate_style_name(FEERATE_PER_KBYTE));
+	} else if (json_tok_endswith(buffer, tok,
+				     feerate_style_name(FEERATE_PER_KSIPA))) {
+		style = FEERATE_PER_KSIPA;
+		base.end -= strlen(feerate_style_name(FEERATE_PER_KSIPA));
+	} else
+		style = FEERATE_PER_KBYTE;
 
 	if (!json_to_number(buffer, &base, &num)) {
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
-				    "'%s' prefix should be an integer, not '%.*s'",
+				    "'%s' should be an integer with optional perkw/perkb, not '%.*s'",
 				    name, base.end - base.start,
 				    buffer + base.start);
 	}
 
-	if (suffix.end == suffix.start
-	    || json_tok_streq(buffer, &suffix,
-			      feerate_style_name(FEERATE_PER_KBYTE))) {
-		style = FEERATE_PER_KBYTE;
-	} else if (json_tok_streq(buffer, &suffix,
-				feerate_style_name(FEERATE_PER_KSIPA))) {
-		style = FEERATE_PER_KSIPA;
-	} else {
-		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
-				    "'%s' suffix should be '%s' or '%s', not '%.*s'",
-				    name,
-				    feerate_style_name(FEERATE_PER_KSIPA),
-				    feerate_style_name(FEERATE_PER_KBYTE),
-				    suffix.end - suffix.start,
-				    buffer + suffix.start);
-	}
-
 	*feerate_per_kw = tal(cmd, u32);
 	**feerate_per_kw = feerate_from_style(num, style);
 	if (**feerate_per_kw < FEERATE_FLOOR)