Remove watch txids and make watches more bwatch-like: part 1

bitcoin/script.c

@@ -6,13 +6,26 @@
 #include <bitcoin/pubkey.h>
 #include <bitcoin/script.h>
 #include <ccan/mem/mem.h>
+#include <common/pseudorand.h>
 #include <common/randbytes.h>
 #include <common/utils.h>
 #include <wally_script.h>
 
 /* To push 0-75 bytes onto stack. */
 #define OP_PUSHBYTES(val) (val)
 
+size_t script_with_len_hash(const struct script_with_len *swl)
+{
+	return siphash24(siphash_seed(), swl->script, swl->len);
+}
+
+bool script_with_len_eq(const struct script_with_len *a,
+			       const struct script_with_len *b)
+{
+	return memeq(a->script, a->len, b->script, b->len);
+}
+
+
 /* Bitcoin's OP_HASH160 is RIPEMD(SHA256()) */
 static void hash160(struct ripemd160 *redeemhash, const void *mem, size_t len)
 {

bitcoin/script.h

@@ -12,6 +12,16 @@ struct ripemd160;
 struct rel_locktime;
 struct abs_locktime;
 
+/* libwally uses pointer/size pairs */
+struct script_with_len {
+	const u8 *script;
+	size_t len;
+};
+
+size_t script_with_len_hash(const struct script_with_len *swl);
+bool script_with_len_eq(const struct script_with_len *a,
+			const struct script_with_len *b);
+
 /* tal_count() gives the length of the script. */
 u8 *bitcoin_redeem_2of2(const tal_t *ctx,
 			const struct pubkey *key1,

bitcoin/tx.c

@@ -326,13 +326,12 @@ struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
 	return wally_tx_output_get_amount(&tx->wtx->outputs[outnum]);
 }
 
-void bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum,
-				      struct amount_sat *amount)
+struct amount_sat bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum)
 {
 	struct amount_asset asset_amt;
 	asset_amt = bitcoin_tx_output_get_amount(tx, outnum);
 	assert(amount_asset_is_main(&asset_amt));
-	*amount = amount_asset_to_sat(&asset_amt);
+	return amount_asset_to_sat(&asset_amt);
 }
 
 void bitcoin_tx_input_set_witness(struct bitcoin_tx *tx, int innum,

bitcoin/tx.h

@@ -176,8 +176,8 @@ u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *t
  * Internally we use a `wally_tx` to represent the transaction. The
  * satoshi amount isn't a struct amount_sat, so we need a conversion
  */
-void bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum,
-				      struct amount_sat *amount);
+struct amount_sat bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum);
+
 /**
  * Helper to just get an amount_sat for the output amount.
  */

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2608-gb35fabb6
+CCAN version: init-2611-g050dc66d

ccan/ccan/htable/htable.c

@@ -136,9 +136,21 @@ bool htable_copy_(struct htable *dst, const struct htable *src)
 	*dst = *src;
 	dst->table = htable;
 	memcpy(dst->table, src->table, sizeof(size_t) << src->bits);
+	dst->locked = 0;
 	return true;
 }
 
+void htable_lock(struct htable *ht)
+{
+	ht->locked++;
+}
+
+void htable_unlock(struct htable *ht)
+{
+	assert(ht->locked != 0);
+	ht->locked--;
+}
+
 static size_t hash_bucket(const struct htable *ht, size_t h)
 {
 	return h & ((1 << ht->bits)-1);
@@ -380,6 +392,7 @@ bool htable_add_(struct htable *ht, size_t hash, const void *p)
 	/* Cannot insert NULL, or (void *)1. */
 	assert(p);
 	assert(entry_is_valid((uintptr_t)p));
+	assert(ht->locked == 0);
 
 	/* Getting too full? */
 	if (ht->elems+1 + ht->deleted > ht_max(ht)) {

ccan/ccan/htable/htable.h

@@ -25,7 +25,7 @@ struct htable {
 	size_t (*rehash)(const void *elem, void *priv);
 	void *priv;
 	unsigned int bits, perfect_bitnum;
-	size_t elems, deleted;
+	size_t elems, deleted, locked;
 	/* These are the bits which are the same in all pointers. */
 	uintptr_t common_mask, common_bits;
 	uintptr_t *table;
@@ -49,7 +49,7 @@ struct htable {
  *	static struct htable ht = HTABLE_INITIALIZER(ht, rehash, NULL);
  */
 #define HTABLE_INITIALIZER(name, rehash, priv)				\
-	{ rehash, priv, 0, 0, 0, 0, -1, 0, &name.common_bits }
+	{ rehash, priv, 0, 0, 0, 0, 0, -1, 0, &name.common_bits }
 
 /**
  * htable_init - initialize an empty hash table.
@@ -113,7 +113,7 @@ struct htable *htable_check(const struct htable *ht, const char *abortstr);
  * @dst: the hash table to overwrite
  * @src: the hash table to copy
  *
- * Only fails on out-of-memory.
+ * Only fails on out-of-memory.  Note that the copy is not locked (see htable_lock()).
  *
  * Equivalent to (but faster than):
  *    if (!htable_init_sized(dst, src->rehash, src->priv, 1U << src->bits))
@@ -128,6 +128,24 @@ struct htable *htable_check(const struct htable *ht, const char *abortstr);
 #define htable_copy(dst, src) htable_copy_(dst, htable_debug(src, HTABLE_LOC))
 bool htable_copy_(struct htable *dst, const struct htable *src);
 
+/**
+ * htable_lock - prevent additions to the hash table.
+ * @ht: the hash table
+ *
+ * Causes an assertion on htable_add.  This is useful to enforce restrictions
+ * during iteration.  This function nests, so you can htable_lock() multiple
+ * times before calling htable_unlock() multiple times.
+ */
+void htable_lock(struct htable *ht);
+
+/**
+ * htable_unlock - allow additions to the hash table.
+ * @ht: the locked hash table
+ *
+ * See htable_lock().
+ */
+void htable_unlock(struct htable *ht);
+
 /**
  * htable_add - add a pointer into a hash table.
  * @ht: the htable
@@ -136,6 +154,8 @@ bool htable_copy_(struct htable *dst, const struct htable *src);
  *
  * Also note that this can only fail due to allocation failure.  Otherwise, it
  * returns true.
+ *
+ * Note that iteration is NOT safe over htable_add, as the table could be resized.
  */
 #define htable_add(ht, hash, p) \
 	htable_add_(htable_debug(ht, HTABLE_LOC), hash, p)
@@ -148,6 +168,7 @@ bool htable_add_(struct htable *ht, size_t hash, const void *p);
  * @p: the pointer
  *
  * Returns true if the pointer was found (and deleted).
+ * Note that iteration is safe over htable_del.
  */
 #define htable_del(ht, hash, p) \
 	htable_del_(htable_debug(ht, HTABLE_LOC), hash, p)

ccan/ccan/htable/htable_type.h

@@ -32,6 +32,10 @@
  * Count entries:
  *	size_t <name>_count(const struct <name> *ht);
  *
+ * Lock and unlock (to prevent adds):
+ *	void <name>_lock(struct <name> *ht);
+ *	void <name>_unlock(struct <name> *ht);
+ *
  * Add function only fails if we run out of memory:
  *	bool <name>_add(struct <name> *ht, const <type> *e);
  *
@@ -87,6 +91,14 @@
 	{								\
 		return htable_count(&ht->raw);				\
 	}								\
+	static inline UNNEEDED void name##_lock(struct name *ht)	\
+	{								\
+		htable_lock(&ht->raw);					\
+	}								\
+	static inline UNNEEDED void name##_unlock(struct name *ht)	\
+	{								\
+		htable_unlock(&ht->raw);				\
+	}								\
 	static inline UNNEEDED void name##_clear(struct name *ht)	\
 	{								\
 		htable_clear(&ht->raw);					\

common/addr.c

@@ -7,31 +7,31 @@
 
 char *encode_scriptpubkey_to_addr(const tal_t *ctx,
 				  const struct chainparams *chainparams,
-				  const u8 *scriptpubkey)
+				  const u8 *scriptpubkey,
+				  size_t scriptpubkey_len)
 {
 	char *out;
-	const size_t script_len = tal_bytelen(scriptpubkey);
 	struct bitcoin_address pkh;
 	struct ripemd160 sh;
 	int witver;
 
-	if (is_p2pkh(scriptpubkey, script_len, &pkh))
+	if (is_p2pkh(scriptpubkey, scriptpubkey_len, &pkh))
 		return bitcoin_to_base58(ctx, chainparams, &pkh);
 
-	if (is_p2sh(scriptpubkey, script_len, &sh))
+	if (is_p2sh(scriptpubkey, scriptpubkey_len, &sh))
 		return p2sh_to_base58(ctx, chainparams, &sh);
 
-	if (is_p2tr(scriptpubkey, script_len, NULL))
+	if (is_p2tr(scriptpubkey, scriptpubkey_len, NULL))
 		witver = 1;
-	else if (is_p2wpkh(scriptpubkey, script_len, NULL)
-		 || is_p2wsh(scriptpubkey, script_len, NULL))
+	else if (is_p2wpkh(scriptpubkey, scriptpubkey_len, NULL)
+		 || is_p2wsh(scriptpubkey, scriptpubkey_len, NULL))
 		witver = 0;
 	else {
 		return NULL;
 	}
 	out = tal_arr(ctx, char, 73 + strlen(chainparams->onchain_hrp));
 	if (!segwit_addr_encode(out, chainparams->onchain_hrp, witver,
-				scriptpubkey + 2, script_len - 2))
+				scriptpubkey + 2, scriptpubkey_len - 2))
 		return tal_free(out);
 
 	return out;

common/addr.h

@@ -6,7 +6,8 @@
 /* Given a scriptPubkey, return an encoded address for p2pkh/p2w{pkh,sh}/p2tr */
 char *encode_scriptpubkey_to_addr(const tal_t *ctx,
 				  const struct chainparams *chainparams,
-				  const u8 *scriptpubkey);
+				  const u8 *scriptpubkey,
+				  size_t scriptpubkey_len);
 
 bool decode_scriptpubkey_from_addr(const tal_t *ctx,
 				   const struct chainparams *chainparams,

common/bolt11_json.c

@@ -28,7 +28,7 @@ static void json_add_fallback(struct json_stream *response,
 		json_add_string(response, "type", "P2TR");
 	}
 
-	addr = encode_scriptpubkey_to_addr(tmpctx, chain, fallback);
+	addr = encode_scriptpubkey_to_addr(tmpctx, chain, fallback, fallback_len);
 	if (addr)
 		json_add_string(response, "addr", addr);
 	json_add_hex_talarr(response, "hex", fallback);

common/test/run-amount.c

@@ -1,6 +1,6 @@
 #include "config.h"
 #include "../amount.c"
-#include <common/randbytes.h>
+#include <common/pseudorand.h>
 #include <common/setup.h>
 #include <stdio.h>
 
@@ -37,6 +37,9 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 /* Generated stub for fromwire_u8_array */
 void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire */
 void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "towire called!\n"); abort(); }

common/test/run-base64.c

@@ -1,9 +1,10 @@
 #include "config.h"
 #include "../base64.c"
 #include <assert.h>
+#include <bitcoin/script.h>
 #include <ccan/base64/base64.h>
 #include <common/amount.h>
-#include <common/randbytes.h>
+#include <common/pseudorand.h>
 #include <common/setup.h>
 #include <common/utils.h>
 #include <sodium/utils.h>
@@ -77,6 +78,9 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 /* Generated stub for fromwire_u8_array */
 void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire */
 void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "towire called!\n"); abort(); }

common/test/run-bigsize.c

@@ -4,7 +4,6 @@
 #include <common/amount.h>
 #include <common/json_parse.c>
 #include <common/json_parse_simple.c>
-#include <common/randbytes.h>
 #include <common/setup.h>
 
 static const char *reason;
@@ -91,6 +90,9 @@ bool parse_amount_msat(struct amount_msat *msat UNNEEDED, const char *s UNNEEDED
 /* Generated stub for parse_amount_sat */
 bool parse_amount_sat(struct amount_sat *sat UNNEEDED, const char *s UNNEEDED, size_t slen UNNEEDED)
 { fprintf(stderr, "parse_amount_sat called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire */
 void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "towire called!\n"); abort(); }

common/test/run-blindedpath_enctlv.c

@@ -3,7 +3,6 @@
 #include "../blinding.c"
 #include "../hmac.c"
 #include <common/channel_id.h>
-#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 
@@ -63,6 +62,9 @@ void fromwire_node_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct n
 void fromwire_sciddir_or_pubkey(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 				struct sciddir_or_pubkey *sciddpk UNNEEDED)
 { fprintf(stderr, "fromwire_sciddir_or_pubkey called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

common/test/run-blindedpath_onion.c

@@ -81,6 +81,9 @@ struct onionreply *new_onionreply(const tal_t *ctx UNNEEDED, const u8 *contents
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

common/test/run-bolt11.c

@@ -9,11 +9,13 @@
 #include "../wire/fromwire.c"
 #include "../wire/towire.c"
 #include <ccan/err/err.h>
-#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 
 /* AUTOGENERATED MOCKS START */
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 static struct privkey privkey;

common/test/run-bolt12-encode-test.c

@@ -12,7 +12,6 @@
 #include <common/bolt12.c>
 #include <common/bolt12_merkle.h>
 #include <common/features.c>
-#include <common/randbytes.h>
 #include <common/sciddir_or_pubkey.c>
 #include <common/setup.h>
 #include <inttypes.h>
@@ -72,6 +71,9 @@ void sighash_from_merkle(const char *messagename UNNEEDED,
 			 const struct sha256 *merkle UNNEEDED,
 			 struct sha256 *sighash UNNEEDED)
 { fprintf(stderr, "sighash_from_merkle called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

common/test/run-bolt12-format-string-test.c

@@ -8,7 +8,6 @@
 #include <common/bolt12.h>
 #include <common/bolt12_merkle.h>
 #include <common/features.h>
-#include <common/randbytes.h>
 #include <common/setup.h>
 #include <common/utils.h>
 
@@ -77,6 +76,9 @@ void sighash_from_merkle(const char *messagename UNNEEDED,
 			 const struct sha256 *merkle UNNEEDED,
 			 struct sha256 *sighash UNNEEDED)
 { fprintf(stderr, "sighash_from_merkle called!\n"); abort(); }
+/* Generated stub for siphash_seed */
+const struct siphash_seed *siphash_seed(void)
+{ fprintf(stderr, "siphash_seed called!\n"); abort(); }
 /* Generated stub for towire_blinded_path */
 void towire_blinded_path(u8 **p UNNEEDED, const struct blinded_path *blinded_path UNNEEDED)
 { fprintf(stderr, "towire_blinded_path called!\n"); abort(); }