Merge branch 'ElementsProject:master' into test-current-feerate-assertion

Author: enaples

.github/workflows/ci.yaml

@@ -312,7 +312,7 @@ jobs:
       - name: Test
         env:
           SLOW_MACHINE: 1
-          PYTEST_PAR: 10
+          PYTEST_PAR: 4
           TEST_DEBUG: 1
           TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
@@ -430,7 +430,7 @@ jobs:
           COMPAT: 1
           CFG: ${{ matrix.CFG }}
           SLOW_MACHINE: 1
-          PYTEST_PAR: 10
+          PYTEST_PAR: 4
           TEST_DEBUG: 1
           TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
@@ -524,8 +524,9 @@ jobs:
         env:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
+          PYTEST_PAR: 2
         run: |
-          VALGRIND=1 uv run eatmydata pytest tests/ -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          VALGRIND=1 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -613,8 +614,10 @@ jobs:
         run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
 
       - name: Test
+        env:
+          PYTEST_PAR: 2
         run: |
-          uv run eatmydata pytest tests/ -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -737,7 +740,7 @@ jobs:
           COMPAT: 1
           CFG: ${{ matrix.CFG }}
           SLOW_MACHINE: 1
-          PYTEST_PAR: 10
+          PYTEST_PAR: 4
           TEST_DEBUG: 1
           TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}

Makefile

@@ -313,6 +313,10 @@ else
 LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
 endif
 
+ifeq ($(HAVE_FUNCTION_SECTIONS),1)
+LDLIBS += -Wl,--gc-sections
+endif
+
 # If we have the postgres client library we need to link against it as well
 ifeq ($(HAVE_POSTGRES),1)
 LDLIBS += $(POSTGRES_LDLIBS)
@@ -1152,7 +1156,11 @@ ccan-rune-rune.o: $(CCANDIR)/ccan/rune/rune.c
 ccan-rune-coding.o: $(CCANDIR)/ccan/rune/coding.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
-print-binary-sizes: $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS)
-	@find $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) -printf '%p\t%s\n'
-	@echo 'Total program size:	'`find $(ALL_PROGRAMS) -printf '%s\n' | awk '{TOTAL+= $$1} END {print TOTAL}'`
-	@echo 'Total tests size:	'`find $(ALL_TEST_PROGRAMS) -printf '%s\n' | awk '{TOTAL+= $$1} END {print TOTAL}'`
+canned-gossmap: devtools/gossmap-compress
+	DATE=`date +%Y-%m-%d` && devtools/gossmap-compress compress --output-node-map /tmp/gossip_store tests/data/gossip-store-$$DATE.compressed > tests/data/gossip-store-$$DATE-node-map && xz -9 tests/data/gossip-store-$$DATE-node-map && ls -l tests/data/gossip-store-$$DATE*
+
+print-binary-sizes: $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(BIN_PROGRAMS)
+	@echo User programs:
+	@size -t $(PKGLIBEXEC_PROGRAMS) $(filter-out tools/reckless,$(BIN_PROGRAMS)) $(PLUGINS)
+	@echo All programs:
+	@size -t $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) | tail -n1

common/bolt12.c

@@ -171,6 +171,7 @@ struct tlv_offer *offer_decode(const tal_t *ctx,
 	const u8 *data;
 	size_t dlen;
 	const struct tlv_field *badf;
+	const struct recurrence *recurr;
 
 	data = string_to_data(tmpctx, b12, b12len, "lno", &dlen, fail);
 	if (!data)
@@ -221,6 +222,16 @@ struct tlv_offer *offer_decode(const tal_t *ctx,
 		return tal_free(offer);
 	}
 
+	/* BOLT #12:
+	 *
+	 *   - if `offer_currency` is set and `offer_amount` is not set:
+	 *     - MUST NOT respond to the offer.
+	 */
+	if (offer->offer_currency && !offer->offer_amount) {
+		*fail = tal_strdup(ctx, "Offer contains a currency with no amount");
+		return tal_free(offer);
+	}
+
 	/* BOLT #12:
 	 *
 	 *   - if neither `offer_issuer_id` nor `offer_paths` are set:
@@ -242,6 +253,46 @@ struct tlv_offer *offer_decode(const tal_t *ctx,
 		}
 	}
 
+	/* BOLT-recurrence #12
+	 *   - if `offer_recurrence_optional` or `offer_recurrence_compulsory` are set:
+	 *     - if `time_unit` is not one of 0, 1, or 2:
+	 *        - MUST NOT respond to the offer.
+	 *     - if `period` is 0:
+	 *        - MUST NOT respond to the offer.
+	 *     - if `offer_recurrence_limit` is set and `max_period_index` is 0:
+	 *        - MUST NOT respond to the offer.
+	 */
+	recurr = offer_recurrence(offer);
+	if (recurr) {
+		if (recurr->time_unit != 0
+		    && recurr->time_unit != 1
+		    && recurr->time_unit != 2) {
+			*fail = tal_fmt(ctx, "Offer contains invalid recurrence time_unit %u", recurr->time_unit);
+			return tal_free(offer);
+		}
+		if (recurr->period == 0) {
+			*fail = tal_fmt(ctx, "Offer contains invalid recurrence period %u", recurr->period);
+			return tal_free(offer);
+		}
+		if (offer->offer_recurrence_limit && *offer->offer_recurrence_limit == 0) {
+			*fail = tal_fmt(ctx, "Offer contains invalid recurrence limit %u",
+					*offer->offer_recurrence_limit);
+			return tal_free(offer);
+		}
+	} else {
+		/* BOLT-recurrence #12
+		 *   - otherwise: (no recurrence):
+		 *     - if it `offer_recurrence_paywindow`, `offer_recurrence_limit` or `offer_recurrence_base` are set:
+		 *        - MUST NOT respond to the offer.
+		 */
+		if (offer->offer_recurrence_paywindow
+		    || offer->offer_recurrence_limit
+		    || offer->offer_recurrence_base) {
+			*fail = tal_strdup(ctx, "Offer contains recurrence fields but no recurrence");
+			return tal_free(offer);
+		}
+	}
+
 	return offer;
 }
 

common/configvar.c

@@ -4,6 +4,7 @@
 #include <ccan/tal/str/str.h>
 #include <common/configvar.h>
 #include <common/utils.h>
+#include <unistd.h>
 
 struct configvar *configvar_new(const tal_t *ctx,
 				enum configvar_src src,

common/gossmap.c

@@ -503,16 +503,14 @@ static struct gossmap_chan *add_channel(struct gossmap *map,
 	return chan;
 }
 
-/* Does not set hc->nodeidx! */
-static void fill_from_update(struct gossmap *map,
+/* Does not set hc->nodeidx!
+ * Returns false if it doesn't fit in our representation: this happens
+ * on the real network, as people set absurd fees
+ */
+static bool fill_from_update(struct gossmap *map,
 			     struct short_channel_id_dir *scidd,
 			     struct half_chan *hc,
-			     u64 cupdate_off,
-			     void (*logcb)(void *cbarg,
-					   enum log_level level,
-					   const char *fmt,
-					   ...),
-			     void *cbarg)
+			     u64 cupdate_off)
 {
 	/* Note that first two bytes are message type */
 	const u64 scid_off = cupdate_off + 2 + (64 + 32);
@@ -543,18 +541,15 @@ static void fill_from_update(struct gossmap *map,
 	hc->proportional_fee = proportional_fee;
 	hc->delay = delay;
 
-	/* Check they fit: we turn off if not, log (at debug, it happens!). */
+	/* Check they fit: we turn off if not. */
 	if (hc->base_fee != base_fee
 	    || hc->proportional_fee != proportional_fee
 	    || hc->delay != delay) {
 		hc->htlc_max = 0;
 		hc->enabled = false;
-		if (logcb)
-			logcb(cbarg, LOG_DBG,
-			      "Bad cupdate for %s, ignoring (delta=%u, fee=%u/%u)",
-			      fmt_short_channel_id_dir(tmpctx, scidd),
-			      delay, base_fee, proportional_fee);
+		return false;
 	}
+	return true;
 }
 
 /* BOLT #7:
@@ -572,23 +567,24 @@ static void fill_from_update(struct gossmap *map,
  *     * [`u32`:`fee_proportional_millionths`]
  *     * [`u64`:`htlc_maximum_msat`]
  */
-static void update_channel(struct gossmap *map, u64 cupdate_off)
+static bool update_channel(struct gossmap *map, u64 cupdate_off)
 {
 	struct short_channel_id_dir scidd;
 	struct gossmap_chan *chan;
 	struct half_chan hc;
+	bool ret;
 
-	fill_from_update(map, &scidd, &hc, cupdate_off,
-			 map->logcb, map->cbarg);
+	ret = fill_from_update(map, &scidd, &hc, cupdate_off);
 	chan = gossmap_find_chan(map, &scidd.scid);
 	/* This can happen if channel gets deleted! */
 	if (!chan)
-		return;
+		return ret;
 
 	/* Preserve this */
 	hc.nodeidx = chan->half[scidd.dir].nodeidx;
 	chan->half[scidd.dir] = hc;
 	chan->cupdate_off[scidd.dir] = cupdate_off;
+	return ret;
 }
 
 static void remove_channel_by_deletemsg(struct gossmap *map, u64 del_off)
@@ -677,7 +673,7 @@ static bool csum_matches(const struct gossmap *map,
 /* Returns false only if must_be_clean is true. */
 static bool map_catchup(struct gossmap *map, bool must_be_clean, bool *changed)
 {
-	size_t reclen;
+	size_t reclen, num_bad_cupdates = 0;
 
 	*changed = false;
 
@@ -749,7 +745,7 @@ static bool map_catchup(struct gossmap *map, bool must_be_clean, bool *changed)
 			if (redundant && must_be_clean)
 				return false;
 		} else if (type == WIRE_CHANNEL_UPDATE)
-			update_channel(map, off);
+			num_bad_cupdates += update_channel(map, off);
 		else if (type == WIRE_GOSSIP_STORE_DELETE_CHAN)
 			remove_channel_by_deletemsg(map, off);
 		else if (type == WIRE_NODE_ANNOUNCEMENT)
@@ -775,6 +771,12 @@ static bool map_catchup(struct gossmap *map, bool must_be_clean, bool *changed)
 		*changed = true;
 	}
 
+	if (num_bad_cupdates != 0)
+		map->logcb(map->cbarg,
+			   LOG_DBG,
+			   "Got %zu bad cupdates, ignoring them (expected on mainnet)",
+			   num_bad_cupdates);
+
 	return true;
 }
 
@@ -1168,7 +1170,7 @@ void gossmap_apply_localmods(struct gossmap *map,
 			off = insert_local_space(&map->local_updates, tal_bytelen(cupdatemsg));
 			memcpy(map->local_updates + off, cupdatemsg, tal_bytelen(cupdatemsg));
 			chan->cupdate_off[h] = map->map_size + tal_bytelen(map->local_announces) + off;
-			fill_from_update(map, &scidd, &chan->half[h], chan->cupdate_off[h], NULL, NULL);
+			fill_from_update(map, &scidd, &chan->half[h], chan->cupdate_off[h]);
 
 			/* We wrote the right update, correct? */
 			assert(short_channel_id_eq(scidd.scid, mod->scid));

common/jsonrpc_errors.h

@@ -116,6 +116,7 @@ enum jsonrpc_errcode {
 	OFFER_BAD_INVREQ_REPLY = 1004,
 	OFFER_TIMEOUT = 1005,
 	OFFER_ALREADY_ENABLED = 1006,
+	OFFER_USED_SINGLE_USE = 1007,
 
 	/* Errors from datastore command */
 	DATASTORE_DEL_DOES_NOT_EXIST = 1200,

common/onion_encode.h

@@ -7,13 +7,7 @@
 struct route_step;
 struct tlv_encrypted_data_tlv_payment_relay;
 
-enum onion_payload_type {
-	ONION_V0_PAYLOAD = 0,
-	ONION_TLV_PAYLOAD = 1,
-};
-
 struct onion_payload {
-	enum onion_payload_type type;
 	/* Is this the final hop? */
 	bool final;
 

common/sphinx.c

@@ -677,54 +677,13 @@ struct route_step *process_onionpacket(
 	/* Any of these could fail, falling thru with cursor == NULL */
 	payload_size = fromwire_bigsize(&cursor, &max);
 
-	/* Legacy!  0 length payload means fixed 32 byte structure */
-	if (payload_size == 0 && max >= 32) {
-		struct tlv_payload *legacy = tlv_payload_new(tmpctx);
-		const u8 *legacy_cursor = cursor;
-		size_t legacy_max = 32;
-		u8 *onwire_tlv;
-
-		legacy->amt_to_forward = tal(legacy, u64);
-		legacy->outgoing_cltv_value = tal(legacy, u32);
-		legacy->short_channel_id = tal(legacy, struct short_channel_id);
-
-		/* BOLT-obsolete #4:
-		 * ## Legacy `hop_data` payload format
-		 *
-		 * The `hop_data` format is identified by a single `0x00`-byte
-		 * length, for backward compatibility.  Its payload is defined
-		 * as:
-		 *
-		 * 1. type: `hop_data` (for `realm` 0)
-		 * 2. data:
-		 *    * [`short_channel_id`:`short_channel_id`]
-		 *    * [`u64`:`amt_to_forward`]
-		 *    * [`u32`:`outgoing_cltv_value`]
-		 *    * [`12*byte`:`padding`]
-		 */
-		*legacy->short_channel_id = fromwire_short_channel_id(&legacy_cursor, &legacy_max);
-		*legacy->amt_to_forward = fromwire_u64(&legacy_cursor, &legacy_max);
-		*legacy->outgoing_cltv_value = fromwire_u32(&legacy_cursor, &legacy_max);
-
-		/* Re-linearize it as a modern TLV! */
-		onwire_tlv = tal_arr(tmpctx, u8, 0);
-		towire_tlv_payload(&onwire_tlv, legacy);
-
-		/* Length, then tlv */
-		step->raw_payload = tal_arr(step, u8, 0);
-		towire_bigsize(&step->raw_payload, tal_bytelen(onwire_tlv));
-		towire_u8_array(&step->raw_payload, onwire_tlv, tal_bytelen(onwire_tlv));
+	/* FIXME: raw_payload *includes* the length, which is redundant and
+	 * means we can't just ust fromwire_tal_arrn. */
+	fromwire_pad(&cursor, &max, payload_size);
+	if (cursor != NULL)
+		step->raw_payload = tal_dup_arr(step, u8, paddedheader,
+						cursor - paddedheader, 0);
 
-		payload_size = 32;
-		fromwire_pad(&cursor, &max, payload_size);
-	} else {
-		/* FIXME: raw_payload *includes* the length, which is redundant and
-		 * means we can't just ust fromwire_tal_arrn. */
-		fromwire_pad(&cursor, &max, payload_size);
-		if (cursor != NULL)
-			step->raw_payload = tal_dup_arr(step, u8, paddedheader,
-							cursor - paddedheader, 0);
-	}
 	fromwire_hmac(&cursor, &max, &step->next->hmac);
 
 	/* BOLT #4:

common/utils.c

@@ -215,6 +215,15 @@ char *str_lowering(const void *ctx, const char *string TAKES)
 	return ret;
 }
 
+char *str_uppering(const void *ctx, const char *string TAKES)
+{
+	char *ret;
+
+	ret = tal_strdup(ctx, string);
+	for (char *p = ret; *p; p++) *p = toupper(*p);
+	return ret;
+}
+
 /* Realloc helper for tal membufs */
 void *membuf_tal_resize(struct membuf *mb, void *rawelems, size_t newsize)
 {

common/utils.h

@@ -188,6 +188,15 @@ void *membuf_tal_resize(struct membuf *mb, void *rawelems, size_t newsize);
  */
 char *str_lowering(const void *ctx, const char *string TAKES);
 
+/**
+ * tal_struppering - return the same string by in upper case.
+ * @ctx: the context to tal from (often NULL)
+ * @string: the string that is going to be UPPERED (can be take())
+ *
+ * FIXME: move this in ccan
+ */
+char *str_uppering(const void *ctx, const char *string TAKES);
+
 /**
  * Assign two different structs which are the same size.
  * We use this for assigning secrets <-> sha256 for example.