Merge pull request #328 from tecnovert/non_segwit_utxos

prepare: Set changetype=bech32 in BTC and LTC .conf files.

basicswap/bin/prepare.py

@@ -1315,7 +1315,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
         )
         wallet_conf_path = os.path.join(data_dir, wallet_conf_filename)
         if os.path.exists(wallet_conf_path):
-            exitWithError("{} exists".format(wallet_conf_path))
+            exitWithError(f"{wallet_conf_path} exists")
         with open(wallet_conf_path, "w") as fp:
             if chain != "mainnet":
                 fp.write(chainname + "=1\n")
@@ -1342,7 +1342,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
     core_conf_name: str = core_settings.get("config_filename", coin + ".conf")
     core_conf_path = os.path.join(data_dir, core_conf_name)
     if os.path.exists(core_conf_path):
-        exitWithError("{} exists".format(core_conf_path))
+        exitWithError(f"{core_conf_path} exists")
     with open(core_conf_path, "w") as fp:
         if chain != "mainnet":
             if coin in ("navcoin",):
@@ -1393,6 +1393,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
                 )
         elif coin == "litecoin":
             fp.write("prune=4000\n")
+            fp.write("changetype=bech32\n")
             if LTC_RPC_USER != "":
                 fp.write(
                     "rpcauth={}:{}${}\n".format(
@@ -1410,6 +1411,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
         elif coin == "bitcoin":
             fp.write("deprecatedrpc=create_bdb\n")
             fp.write("prune=2000\n")
+            fp.write("changetype=bech32\n")
             fp.write("fallbackfee=0.0002\n")
             if BTC_RPC_USER != "":
                 fp.write(
@@ -1784,7 +1786,7 @@ def test_particl_encryption(data_dir, settings, chain, use_tor_proxy):
         coin_name = "particl"
         coin_settings = settings["chainclients"][coin_name]
         daemon_args += getCoreBinArgs(c, coin_settings, prepare=True)
-        extra_config = {"stdout_to_file": True}
+        extra_config = {"stdout_to_file": True, "coin_name": coin_name}
         if coin_settings["manage_daemon"]:
             filename: str = getCoreBinName(c, coin_settings, coin_name + "d")
             daemons.append(
@@ -1906,7 +1908,7 @@ def initialise_wallets(
                             )
                         ]
 
-                    extra_config = {"stdout_to_file": True}
+                    extra_config = {"stdout_to_file": True, "coin_name": coin_name}
                     daemons.append(
                         startDaemon(
                             coin_settings["datadir"],

basicswap/bin/run.py

@@ -59,15 +59,20 @@ def signal_handler(sig, frame):
 def startDaemon(node_dir, bin_dir, daemon_bin, opts=[], extra_config={}):
     daemon_bin = os.path.expanduser(os.path.join(bin_dir, daemon_bin))
     datadir_path = os.path.expanduser(node_dir)
+    coin_name = extra_config.get("coin_name", "")
 
     # Rewrite litecoin.conf
     # TODO: Remove
-    needs_rewrite: bool = False
     ltc_conf_path = os.path.join(datadir_path, "litecoin.conf")
     if os.path.exists(ltc_conf_path):
+        needs_rewrite: bool = False
+        add_changetype: bool = True
         with open(ltc_conf_path) as fp:
             for line in fp:
                 line = line.strip()
+                if line.startswith("changetype="):
+                    add_changetype = False
+                    break
                 if line.endswith("=onion"):
                     needs_rewrite = True
                     break
@@ -83,6 +88,29 @@ def startDaemon(node_dir, bin_dir, daemon_bin, opts=[], extra_config={}):
                         fp_to.write(line.strip()[:-6] + "\n")
                     else:
                         fp_to.write(line)
+                if add_changetype:
+                    fp_to.write("changetype=bech32\n")
+                    add_changetype = False
+        if add_changetype:
+            logger.info("Adding changetype to litecoin.conf")
+            with open(ltc_conf_path, "a") as fp:
+                fp.write("changetype=bech32\n")
+
+    # Rewrite bitcoin.conf
+    # TODO: Remove
+    btc_conf_path = os.path.join(datadir_path, "bitcoin.conf")
+    if coin_name == "bitcoin" and os.path.exists(btc_conf_path):
+        add_changetype: bool = True
+        with open(btc_conf_path) as fp:
+            for line in fp:
+                line = line.strip()
+                if line.startswith("changetype="):
+                    add_changetype = False
+                    break
+        if add_changetype:
+            logger.info("Adding changetype to bitcoin.conf")
+            with open(btc_conf_path, "a") as fp:
+                fp.write("changetype=bech32\n")
 
     args = [
         daemon_bin,
@@ -474,6 +502,7 @@ def runClient(
                         "stdout_to_file": True,
                         "stdout_filename": "dcrd_stdout.log",
                         "use_shell": use_shell,
+                        "coin_name": "decred",
                     }
                     daemons.append(
                         startDaemon(
@@ -502,6 +531,7 @@ def runClient(
                         "stdout_to_file": True,
                         "stdout_filename": "dcrwallet_stdout.log",
                         "use_shell": use_shell,
+                        "coin_name": "decred",
                     }
                     daemons.append(
                         startDaemon(
@@ -524,8 +554,15 @@ def runClient(
                 extra_opts = getCoreBinArgs(
                     coin_id, v, use_tor_proxy=swap_client.use_tor_proxy
                 )
+                extra_config = {"coin_name": c}
                 daemons.append(
-                    startDaemon(v["datadir"], v["bindir"], filename, opts=extra_opts)
+                    startDaemon(
+                        v["datadir"],
+                        v["bindir"],
+                        filename,
+                        opts=extra_opts,
+                        extra_config=extra_config,
+                    )
                 )
                 pid = daemons[-1].handle.pid
                 pids.append((c, pid))

basicswap/interface/btc.py

@@ -561,7 +561,7 @@ class BTCInterface(Secp256k1Interface):
         override_feerate = chain_client_settings.get("override_feerate", None)
         if override_feerate:
             self._log.debug(
-                "Fee rate override used for %s: %f", self.coin_name(), override_feerate
+                f"Fee rate override used for {self.coin_name()}: {override_feerate}"
             )
             return override_feerate, "override_feerate"
 
@@ -1318,22 +1318,37 @@ class BTCInterface(Secp256k1Interface):
         rv = self.rpc_wallet("fundrawtransaction", [tx.hex(), options])
         return bytes.fromhex(rv["hex"])
 
-    def lockNonSegwitPrevouts(self) -> None:
+    def getNonSegwitOutputs(self):
-        # For tests
+        unspents = self.rpc_wallet("listunspent", [0, 99999999])
-        unspent = self.rpc_wallet("listunspent")
+        nonsegwit_unspents = []
-
+        for u in unspents:
-        to_lock = []
-        for u in unspent:
             if u.get("spendable", False) is False:
                 continue
             if "desc" in u:
                 desc = u["desc"]
                 if self.use_p2shp2wsh():
                     if not desc.startswith("sh(wpkh"):
-                        to_lock.append({"txid": u["txid"], "vout": u["vout"]})
+                        nonsegwit_unspents.append(
+                            {
+                                "txid": u["txid"],
+                                "vout": u["vout"],
+                                "amount": u["amount"],
+                            }
+                        )
                 else:
                     if not desc.startswith("wpkh"):
-                        to_lock.append({"txid": u["txid"], "vout": u["vout"]})
+                        nonsegwit_unspents.append(
+                            {
+                                "txid": u["txid"],
+                                "vout": u["vout"],
+                                "amount": u["amount"],
+                            }
+                        )
+        return nonsegwit_unspents
+
+    def lockNonSegwitPrevouts(self) -> None:
+        # For tests
+        to_lock = self.getNonSegwitOutputs()
 
         if len(to_lock) > 0:
             self._log.debug(f"Locking {len(to_lock)} non segwit prevouts")
@@ -1661,7 +1676,7 @@ class BTCInterface(Secp256k1Interface):
                 "listunspent",
                 [
                     0,
-                    9999999,
+                    99999999,
                     [
                         dest_address,
                     ],
@@ -2392,6 +2407,59 @@ class BTCInterface(Secp256k1Interface):
     def isTxNonFinalError(self, err_str: str) -> bool:
         return "non-BIP68-final" in err_str or "non-final" in err_str
 
+    def combine_non_segwit_prevouts(self):
+        self._log.info("Combining non-segwit prevouts")
+        if self._use_segwit is False:
+            raise RuntimeError("Not configured to use segwit outputs.")
+        prevouts_to_spend = self.getNonSegwitOutputs()
+        if len(prevouts_to_spend) < 1:
+            raise RuntimeError("No non-segwit outputs found.")
+
+        total_amount: int = 0
+        for n, prevout in enumerate(prevouts_to_spend):
+            total_amount += self.make_int(prevout["amount"])
+        addr_to: str = self.getNewAddress(
+            self._use_segwit, "combine_non_segwit_prevouts"
+        )
+
+        txn = self.rpc(
+            "createrawtransaction",
+            [prevouts_to_spend, {addr_to: self.format_amount(total_amount)}],
+        )
+        fee_rate, rate_src = self.get_fee_rate(self._conf_target)
+        fee_rate_str: str = self.format_amount(fee_rate, True, 1)
+        self._log.debug(
+            f"Using fee rate: {fee_rate_str}, src: {rate_src}, confirms target: {self._conf_target}"
+        )
+        options = {
+            "add_inputs": False,
+            "subtractFeeFromOutputs": [
+                0,
+            ],
+            "feeRate": fee_rate_str,
+        }
+        tx_fee_set = self.rpc_wallet("fundrawtransaction", [txn, options])["hex"]
+        tx_signed = self.rpc_wallet("signrawtransactionwithwallet", [tx_fee_set])["hex"]
+        tx = self.rpc(
+            "decoderawtransaction",
+            [
+                tx_signed,
+            ],
+        )
+        self._log.info(
+            "Submitting tx to combine non-segwit prevouts: {}".format(
+                self._log.id(bytes.fromhex(tx["txid"]))
+            )
+        )
+        self.rpc(
+            "sendrawtransaction",
+            [
+                tx_signed,
+            ],
+        )
+
+        return tx["txid"]
+
 
 def testBTCInterface():
     print("TODO: testBTCInterface")

basicswap/interface/part.py

@@ -187,6 +187,9 @@ class PARTInterface(BTCInterface):
             ) + self.make_int(u["amount"], r=1)
         return unspent_addr
 
+    def combine_non_segwit_prevouts(self):
+        raise RuntimeError("No non-segwit outputs found.")
+
 
 class PARTInterfaceBlind(PARTInterface):
 

basicswap/templates/debug.html

@@ -1,5 +1,5 @@
 {% include 'header.html' %}
-{% from 'style.html' import breadcrumb_line_svg, start_process_svg %} 
+{% from 'style.html' import breadcrumb_line_svg, start_process_svg %}
 <div class="container mx-auto">
  <section class="p-5 mt-5">
   <div class="flex flex-wrap items-center -m-2">
@@ -74,6 +74,27 @@
               {{ start_process_svg| safe }} Start Process</button>
              </td>
             </tr>
+            <tr class="opacity-100 text-gray-500 dark:text-gray-100">
+             <td class="py-3 px-6 bold">List non-segwit UTXOs</td>
+             <td td class="py-3 px-6 ">
+              <button name="list_non_segwit_prevouts" type="submit" value="Yes" class="w-60 flex flex-wrap justify-center py-2 px-4 bg-blue-500 hover:bg-blue-600 font-medium text-sm text-white border border-blue-500 rounded-md shadow-button focus:ring-0 focus:outline-none">
+              List</button>
+             </td>
+            </tr>
+            <tr class="opacity-100 text-gray-500 dark:text-gray-100">
+             <td class="py-3 px-6 bold">Combine non-segwit BTC UTXOs</td>
+             <td td class="py-3 px-6 ">
+              <button name="combine_non_segwit_prevouts_btc" type="submit" value="Yes" class="w-60 flex flex-wrap justify-center py-2 px-4 bg-blue-500 hover:bg-blue-600 font-medium text-sm text-white border border-blue-500 rounded-md shadow-button focus:ring-0 focus:outline-none">
+              Combine</button>
+             </td>
+            </tr>
+            <tr class="opacity-100 text-gray-500 dark:text-gray-100">
+             <td class="py-3 px-6 bold">Combine non-segwit LTC UTXOs</td>
+             <td td class="py-3 px-6 ">
+              <button name="combine_non_segwit_prevouts_ltc" type="submit" value="Yes" class="w-60 flex flex-wrap justify-center py-2 px-4 bg-blue-500 hover:bg-blue-600 font-medium text-sm text-white border border-blue-500 rounded-md shadow-button focus:ring-0 focus:outline-none">
+              Combine</button>
+             </td>
+            </tr>
             <input type="hidden" name="formid" value="{{ form_id }}">
            </form>
           </table>

basicswap/ui/page_debug.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2023-2024 The Basicswap developers
+# Copyright (c) 2023-2025 The Basicswap developers
 # Distributed under the MIT software license, see the accompanying
 # file LICENSE or http://www.opensource.org/licenses/mit-license.php.
 
+import json
 import traceback
 from .util import (
     have_data_entry,
@@ -32,6 +33,9 @@ def page_debug(self, url_split, post_string):
                 swap_client.initialiseWallet(Coins.XMR)
                 messages.append("Done.")
             except Exception as e:
+                swap_client.log.error(
+                    traceback.format_exc() if swap_client.debug else f"reinit_xmr: {e}"
+                )
                 err_messages.append(f"Failed: {e}.")
 
         if have_data_entry(form_data, "remove_expired"):
@@ -40,12 +44,59 @@ def page_debug(self, url_split, post_string):
                 remove_expired_data(swap_client)
                 messages.append("Done.")
             except Exception as e:
-                if swap_client.debug is True:
+                swap_client.log.error(
-                    swap_client.log.error(traceback.format_exc())
+                    traceback.format_exc()
-                else:
+                    if swap_client.debug
-                    swap_client.log.error(f"remove_expired_data: {e}")
+                    else f"remove_expired_data: {e}"
+                )
                 err_messages.append("Failed.")
 
+        if have_data_entry(form_data, "list_non_segwit_prevouts"):
+            try:
+                rvj = {}
+                rvj["BTC"] = swap_client.ci(Coins.BTC).getNonSegwitOutputs()
+                rvj["LTC"] = swap_client.ci(Coins.LTC).getNonSegwitOutputs()
+
+                # json.dumps indent=4 ends up in one line in html side
+                message_output = "BTC:<br/>"
+                for utxo in rvj["BTC"]:
+                    message_output += json.dumps(utxo) + "<br/>"
+                message_output += "LTC:<br/>"
+                for utxo in rvj["LTC"]:
+                    message_output += json.dumps(utxo) + "<br/>"
+                messages.append(message_output)
+            except Exception as e:
+                swap_client.log.error(
+                    traceback.format_exc()
+                    if swap_client.debug
+                    else f"list_non_segwit_prevouts: {e}"
+                )
+                err_messages.append(f"Failed: {e}.")
+        if have_data_entry(form_data, "combine_non_segwit_prevouts_btc"):
+            try:
+                ci = swap_client.ci(Coins.BTC)
+                txid = ci.combine_non_segwit_prevouts()
+                messages.append(f"Combined non-segwit BTC UTXOs, txid: {txid}.")
+            except Exception as e:
+                swap_client.log.error(
+                    traceback.format_exc()
+                    if swap_client.debug
+                    else f"combine_non_segwit_prevouts_btc: {e}"
+                )
+                err_messages.append(f"Failed: {e}.")
+        if have_data_entry(form_data, "combine_non_segwit_prevouts_ltc"):
+            try:
+                ci = swap_client.ci(Coins.LTC)
+                txid = ci.combine_non_segwit_prevouts()
+                messages.append(f"Combined non-segwit LTC UTXOs, txid: {txid}.")
+            except Exception as e:
+                swap_client.log.error(
+                    traceback.format_exc()
+                    if swap_client.debug
+                    else f"combine_non_segwit_prevouts_ltc: {e}"
+                )
+                err_messages.append(f"Failed: {e}.")
+
     template = server.env.get_template("debug.html")
     return self.render_template(
         template,

tests/basicswap/common.py

@@ -102,6 +102,9 @@ def prepareDataDir(
 
         if base_p2p_port == BTC_BASE_PORT:
             fp.write("deprecatedrpc=create_bdb\n")
+            fp.write("changetype=bech32\n")
+        elif base_p2p_port == LTC_BASE_PORT:
+            fp.write("changetype=bech32\n")
         elif base_p2p_port == BASE_PORT:  # Particl
             fp.write("zmqpubsmsg=tcp://127.0.0.1:{}\n".format(BASE_ZMQ_PORT + node_id))
             # minstakeinterval=5  # Using walletsettings stakelimit instead

tests/basicswap/test_btc_xmr.py

@@ -1808,6 +1808,41 @@ class BasicSwapTest(TestFunctions):
             ci.setActiveWallet("wallet.dat")
             chain_client_settings["manage_daemon"] = False
 
+    def test_015_changetype(self):
+        logging.info(f"---------- Test {self.test_coin_from.name} changetype")
+        ci = self.swap_clients[0].ci(self.test_coin_from)
+
+        addr_p2sh = ci.rpc_wallet(
+            "getnewaddress", ["test_015_changetype", "p2sh-segwit"]
+        )
+        txid = ci.rpc_wallet("sendtoaddress", [addr_p2sh, 1])
+
+        tx_hex = ci.rpc_wallet("gettransaction", [txid])["hex"]
+        tx = ci.rpc_wallet("decoderawtransaction", [tx_hex])
+        change_vout: int = -1
+        for vout in tx["vout"]:
+            if "address" in vout["scriptPubKey"]:
+                if vout["scriptPubKey"]["address"] == addr_p2sh:
+                    continue
+            else:
+                if addr_p2sh in vout["scriptPubKey"]["addresses"]:
+                    continue
+            change_vout = vout["n"]
+            assert vout["scriptPubKey"]["type"] == "witness_v0_keyhash"
+        assert change_vout > -1
+        ci.rpc_wallet("sendtoaddress", [addr_p2sh, 2])
+        ci.rpc_wallet("sendtoaddress", [addr_p2sh, 3])
+
+        txid = ci.combine_non_segwit_prevouts()
+        tx_hex = ci.rpc_wallet("gettransaction", [txid])["hex"]
+        tx = ci.rpc_wallet("decoderawtransaction", [tx_hex])
+
+        assert len(tx["vin"]) == 3
+        for vin in tx["vin"]:
+            assert len(vin["scriptSig"]["hex"]) > 0
+        for vout in tx["vout"]:
+            assert vout["scriptPubKey"]["type"] == "witness_v0_keyhash"
+
     def test_01_0_lock_bad_prevouts(self):
         logging.info(
             "---------- Test {} lock_bad_prevouts".format(self.test_coin_from.name)