diff --git a/.cirrus.yml b/.cirrus.yml index 6b93595..0578f0b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,7 +8,7 @@ lint_task: script: - flake8 --version - PYTHONWARNINGS="ignore" flake8 --ignore=E501,F841,W503 --exclude=basicswap/contrib,messages_pb2.py,.eggs,.tox - - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,gitianpubkeys,*.pyc,*basicswap/contrib,*mnemonics.py + - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,pgp,*.pyc,*basicswap/contrib,*mnemonics.py test_task: environment: diff --git a/.travis.yml b/.travis.yml index 02ee0f5..d09a2f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ jobs: before_script: script: - PYTHONWARNINGS="ignore" flake8 --ignore=E501,F841,W503 --exclude=basicswap/contrib,messages_pb2.py,.eggs,.tox - - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,gitianpubkeys,*.pyc,*basicswap/contrib,*mnemonics.py + - codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=tests/lint/spelling.ignore-words.txt -S .git,.eggs,.tox,pgp,*.pyc,*basicswap/contrib,*mnemonics.py after_success: - echo "End lint" - stage: test diff --git a/basicswap/basicswap.py b/basicswap/basicswap.py index e556e1b..e6eaa66 100644 --- a/basicswap/basicswap.py +++ b/basicswap/basicswap.py @@ -608,12 +608,12 @@ class BasicSwap(BaseApp): if self.coin_clients[c]['connection_type'] == 'rpc' and chain_client_settings['manage_daemon'] is True: self.stopDaemon(c) - def waitForDaemonRPC(self, coin_type): + def waitForDaemonRPC(self, coin_type, with_wallet=True): for i in range(self.startup_tries): if not self.is_running: return try: - self.coin_clients[coin_type]['interface'].testDaemonRPC() + self.coin_clients[coin_type]['interface'].testDaemonRPC(with_wallet) return except Exception as ex: self.log.warning('Can\'t connect to %s RPC: %s. Trying again in %d second/s.', coin_type, str(ex), (1 + i)) @@ -3778,12 +3778,11 @@ class BasicSwap(BaseApp): self.mxDB.acquire() use_session = scoped_session(self.session_factory) - link = session.query(AutomationLink).filter_by(active_ind=1, linked_type=TableTypes.OFFER, linked_id=offer.offer_id).first() - + link = use_session.query(AutomationLink).filter_by(active_ind=1, linked_type=TableTypes.OFFER, linked_id=offer.offer_id).first() if not link: return False - strategy = session.query(AutomationStrategy).filter_by(active_ind=1, record_id=link.strategy_id).first() + strategy = use_session.query(AutomationStrategy).filter_by(active_ind=1, record_id=link.strategy_id).first() opts = json.loads(strategy.data.decode('utf-8')) self.log.debug('Evaluating against strategy {}'.format(strategy.record_id)) @@ -3800,7 +3799,7 @@ class BasicSwap(BaseApp): return False if strategy.only_known_identities: - identity_stats = session.query(KnownIdentity).filter_by(address=bid.bid_addr).first() + identity_stats = use_session.query(KnownIdentity).filter_by(address=bid.bid_addr).first() if not identity_stats: return False @@ -3811,6 +3810,9 @@ class BasicSwap(BaseApp): return False return True + except Exception as e: + self.log.error('shouldAutoAcceptBid: %s', str(e)) + return False finally: if session is None: use_session.close() diff --git a/basicswap/interface_btc.py b/basicswap/interface_btc.py index 4f26f54..438afff 100644 --- a/basicswap/interface_btc.py +++ b/basicswap/interface_btc.py @@ -211,8 +211,11 @@ class BTCInterface(CoinInterface): ensure(new_conf_target >= 1 and new_conf_target < 33, 'Invalid conf_target value') self._conf_target = new_conf_target - def testDaemonRPC(self): - self.rpc_callback('getwalletinfo', []) + def testDaemonRPC(self, with_wallet=True): + if with_wallet: + self.rpc_callback('getwalletinfo', []) + else: + self.rpc_callback('getblockchaininfo', []) def getDaemonVersion(self): return self.rpc_callback('getnetworkinfo')['version'] diff --git a/basicswap/interface_xmr.py b/basicswap/interface_xmr.py index 45cdea9..11b5aa9 100644 --- a/basicswap/interface_xmr.py +++ b/basicswap/interface_xmr.py @@ -110,7 +110,7 @@ class XMRInterface(CoinInterface): with self._mx_wallet: self.rpc_wallet_cb('open_wallet', {'filename': self._wallet_filename}) - def testDaemonRPC(self): + def testDaemonRPC(self, with_wallet=True): self.rpc_wallet_cb('get_languages') def getDaemonVersion(self): diff --git a/bin/basicswap_prepare.py b/bin/basicswap_prepare.py index 6256085..da8622c 100755 --- a/bin/basicswap_prepare.py +++ b/bin/basicswap_prepare.py @@ -117,7 +117,9 @@ TOR_DNS_PORT = int(os.getenv('TOR_DNS_PORT', 5353)) TEST_TOR_PROXY = toBool(os.getenv('TEST_TOR_PROXY', 'true')) # Expects a known exit node TEST_ONION_LINK = toBool(os.getenv('TEST_ONION_LINK', 'false')) -extract_core_overwrite = True +BITCOIN_FASTSYNC_URL = os.getenv('BITCOIN_FASTSYNC_URL', 'http://utxosets.blob.core.windows.net/public/') +BITCOIN_FASTSYNC_FILE = os.getenv('BITCOIN_FASTSYNC_FILE', 'utxo-snapshot-bitcoin-mainnet-720179.tar') + use_tor_proxy = False default_socket = socket.socket @@ -199,9 +201,10 @@ def testOnionLink(): logger.info('Onion links work.') -def extractCore(coin, version_data, settings, bin_dir, release_path): +def extractCore(coin, version_data, settings, bin_dir, release_path, extra_opts={}): version, version_tag, signers = version_data logger.info('extractCore %s v%s%s', coin, version, version_tag) + extract_core_overwrite = extra_opts.get('extract_core_overwrite', True) if coin == 'monero': bins = ['monerod', 'monero-wallet-rpc'] @@ -260,7 +263,7 @@ def extractCore(coin, version_data, settings, bin_dir, release_path): logging.warning('Unable to set file permissions: %s, for %s', str(e), out_path) -def prepareCore(coin, version_data, settings, data_dir): +def prepareCore(coin, version_data, settings, data_dir, extra_opts={}): version, version_tag, signers = version_data logger.info('prepareCore %s v%s%s', coin, version, version_tag) @@ -384,8 +387,8 @@ def prepareCore(coin, version_data, settings, data_dir): filename = '{}_{}.pgp'.format(coin, signing_key_name) pubkeyurls = ( - 'https://raw.githubusercontent.com/tecnovert/basicswap/master/gitianpubkeys/' + filename, - 'https://gitlab.com/particl/basicswap/-/raw/master/gitianpubkeys/' + filename, + 'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/keys/' + filename, + 'https://gitlab.com/particl/basicswap/-/raw/master/pgp/keys/' + filename, ) for url in pubkeyurls: try: @@ -405,7 +408,7 @@ def prepareCore(coin, version_data, settings, data_dir): and not (verified.status == 'signature valid' and verified.key_status == 'signing key has expired'): raise ValueError('Signature verification failed.') - extractCore(coin, version_data, settings, bin_dir, release_path) + extractCore(coin, version_data, settings, bin_dir, release_path, extra_opts) def writeTorSettings(fp, coin, coin_settings, tor_control_password): @@ -424,10 +427,11 @@ def writeTorSettings(fp, coin, coin_settings, tor_control_password): fp.write(f'bind=0.0.0.0:{onionport}=onion\n') -def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False, tor_control_password=None): +def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}): core_settings = settings['chainclients'][coin] bin_dir = core_settings['bindir'] data_dir = core_settings['datadir'] + tor_control_password = extra_opts.get('tor_control_password', None) if not os.path.exists(data_dir): os.makedirs(data_dir) @@ -466,7 +470,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False if os.path.exists(wallet_conf_path): exitWithError('{} exists'.format(wallet_conf_path)) with open(wallet_conf_path, 'w') as fp: - if use_containers: + if extra_opts.get('use_containers', False) is True: fp.write('daemon-address={}:{}\n'.format(core_settings['rpchost'], core_settings['rpcport'])) fp.write('untrusted-daemon=1\n') fp.write('no-dns=1\n') @@ -481,6 +485,7 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False if not core_settings['manage_daemon']: fp.write(f'proxy={TOR_PROXY_HOST}:{TOR_PROXY_PORT}\n') return + core_conf_path = os.path.join(data_dir, coin + '.conf') if os.path.exists(core_conf_path): exitWithError('{} exists'.format(core_conf_path)) @@ -532,10 +537,43 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, use_containers=False else: logger.warning('Unknown coin %s', coin) - wallet_util = coin + '-wallet' - if os.path.exists(os.path.join(bin_dir, wallet_util)): - logger.info('Creating wallet.dat for {}.'.format(wallet_util.capitalize())) - callrpc_cli(bin_dir, data_dir, chain, '-wallet=wallet.dat create', wallet_util) + if coin == 'bitcoin' and extra_opts.get('use_btc_fastsync', False) is True: + logger.info('Initialising BTC chain with fastsync %s', BITCOIN_FASTSYNC_FILE) + base_dir = extra_opts['data_dir'] + sync_file_path = os.path.join(base_dir, BITCOIN_FASTSYNC_FILE) + if not os.path.exists(sync_file_path): + sync_file_url = os.path.join(BITCOIN_FASTSYNC_URL, BITCOIN_FASTSYNC_FILE) + downloadFile(sync_file_url, sync_file_path) + + asc_filename = BITCOIN_FASTSYNC_FILE + '.asc' + asc_file_path = os.path.join(base_dir, asc_filename) + if not os.path.exists(asc_file_path): + asc_file_urls = ( + 'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/sigs/' + asc_filename, + 'https://gitlab.com/particl/basicswap/-/raw/master/pgp/sigs/' + asc_filename, + ) + for url in asc_file_urls: + try: + downloadFile(url, asc_file_path) + break + except Exception as e: + print('Download failed', e) + gpg = gnupg.GPG() + with open(asc_file_path, 'rb') as fp: + verified = gpg.verify_file(fp, sync_file_path) + + if verified.valid is False \ + and not (verified.status == 'signature valid' and verified.key_status == 'signing key has expired'): + raise ValueError('Signature verification failed.') + + with tarfile.open(sync_file_path) as ft: + ft.extractall(path=data_dir) + # Create the wallet later, no option to set bestblock through wallet_util + else: + wallet_util = coin + '-wallet' + if os.path.exists(os.path.join(bin_dir, wallet_util)): + logger.info('Creating wallet.dat for {}.'.format(wallet_util.capitalize())) + callrpc_cli(bin_dir, data_dir, chain, '-wallet=wallet.dat create', wallet_util) def write_torrc(data_dir, tor_control_password): @@ -696,13 +734,16 @@ def printHelp(): logger.info('--htmlhost= Interface to host on, default:127.0.0.1.') logger.info('--xmrrestoreheight=n Block height to restore Monero wallet from, default:{}.'.format(DEFAULT_XMR_RESTORE_HEIGHT)) logger.info('--noextractover Prevent extracting cores if files exist. Speeds up tests') - logger.info('--usetorproxy Use TOR proxy. Note that some download links may be inaccessible over TOR.') + logger.info('--usetorproxy Use TOR proxy during setup. Note that some download links may be inaccessible over TOR.') + logger.info('--enabletor Setup Basicswap instance to use TOR.') + logger.info('--disabletor Setup Basicswap instance to not use TOR.') + logger.info('--usebtcfastsync Initialise the BTC chain with a snapshot from btcpayserver FastSync.\n' + + ' See https://github.com/btcpayserver/btcpayserver-docker/blob/master/contrib/FastSync/README.md') logger.info('\n' + 'Known coins: %s', ', '.join(known_coins.keys())) def main(): - global extract_core_overwrite global use_tor_proxy data_dir = None bin_dir = None @@ -720,6 +761,8 @@ def main(): enable_tor = False disable_tor = False tor_control_password = None + use_btc_fastsync = False + extract_core_overwrite = True for v in sys.argv[1:]: if len(v) < 2 or v[0] != '-': @@ -767,6 +810,9 @@ def main(): if name == 'disabletor': disable_tor = True continue + if name == 'usebtcfastsync': + use_btc_fastsync = True + continue if len(s) == 2: if name == 'datadir': data_dir = os.path.expanduser(s[1].strip('"')) @@ -987,6 +1033,14 @@ def main(): logger.info('Done.') return 0 + extra_opts = { + 'use_btc_fastsync': use_btc_fastsync, + 'extract_core_overwrite': extract_core_overwrite, + 'data_dir': data_dir, + 'use_containers': use_containers, + 'tor_control_password': tor_control_password, + } + if add_coin != '': logger.info('Adding coin: %s', add_coin) if not os.path.exists(config_path): @@ -1010,10 +1064,10 @@ def main(): settings['use_tor_proxy'] = use_tor_proxy if not no_cores: - prepareCore(add_coin, known_coins[add_coin], settings, data_dir) + prepareCore(add_coin, known_coins[add_coin], settings, data_dir, extra_opts) if not prepare_bin_only: - prepareDataDir(add_coin, settings, chain, particl_wallet_mnemonic, use_containers=use_containers) + prepareDataDir(add_coin, settings, chain, particl_wallet_mnemonic, extra_opts) with open(config_path, 'w') as fp: json.dump(settings, fp, indent=4) @@ -1053,14 +1107,14 @@ def main(): if not no_cores: for c in with_coins: - prepareCore(c, known_coins[c], settings, data_dir) + prepareCore(c, known_coins[c], settings, data_dir, extra_opts) if prepare_bin_only: logger.info('Done.') return 0 for c in with_coins: - prepareDataDir(c, settings, chain, particl_wallet_mnemonic, use_containers=use_containers, tor_control_password=tor_control_password) + prepareDataDir(c, settings, chain, particl_wallet_mnemonic, extra_opts) with open(config_path, 'w') as fp: json.dump(settings, fp, indent=4) @@ -1116,6 +1170,14 @@ def main(): swap_client.setDaemonPID(c, daemons[-1].pid) swap_client.setCoinRunParams(c) swap_client.createCoinInterface(c) + + # Create wallet if it doesn't exist yet + if c == Coins.BTC: + swap_client.waitForDaemonRPC(c, with_wallet=False) + wallets = swap_client.callcoinrpc(c, 'listwallets') + if 'wallet.dat' not in wallets: + swap_client.callcoinrpc(c, 'createwallet', ['wallet.dat']) + swap_client.waitForDaemonRPC(c) swap_client.initialiseWallet(c) swap_client.finalise() diff --git a/doc/install.md b/doc/install.md index 9cd05f1..9029c61 100644 --- a/doc/install.md +++ b/doc/install.md @@ -37,6 +37,12 @@ Adjust `--withcoins` and `--withoutcoins` as desired, eg: `--withcoins=monero,bi **Record the mnemonic from the output of the above command.** +##### FastSync + +Use `--usebtcfastsync` to optionally initialise the Bitcoin datadir with a chain snapshot from btcpayserver FastSync. +[FastSync README.md](https://github.com/btcpayserver/btcpayserver-docker/blob/master/contrib/FastSync/README.md) + + #### Set the timezone (optional): Edit the `.env` file in the docker directory, set TZ to your local timezone. diff --git a/gitianpubkeys/bitcoin_laanwj.pgp b/pgp/keys/bitcoin_laanwj.pgp similarity index 100% rename from gitianpubkeys/bitcoin_laanwj.pgp rename to pgp/keys/bitcoin_laanwj.pgp diff --git a/gitianpubkeys/litecoin_thrasher.pgp b/pgp/keys/litecoin_thrasher.pgp similarity index 100% rename from gitianpubkeys/litecoin_thrasher.pgp rename to pgp/keys/litecoin_thrasher.pgp diff --git a/gitianpubkeys/namecoin_JeremyRand.pgp b/pgp/keys/namecoin_JeremyRand.pgp similarity index 100% rename from gitianpubkeys/namecoin_JeremyRand.pgp rename to pgp/keys/namecoin_JeremyRand.pgp diff --git a/gitianpubkeys/particl_tecnovert.pgp b/pgp/keys/particl_tecnovert.pgp similarity index 100% rename from gitianpubkeys/particl_tecnovert.pgp rename to pgp/keys/particl_tecnovert.pgp diff --git a/pgp/sigs/utxo-snapshot-bitcoin-mainnet-720179.tar.asc b/pgp/sigs/utxo-snapshot-bitcoin-mainnet-720179.tar.asc new file mode 100644 index 0000000..81fed43 --- /dev/null +++ b/pgp/sigs/utxo-snapshot-bitcoin-mainnet-720179.tar.asc @@ -0,0 +1,16 @@ +-----BEGIN PGP SIGNATURE----- + +iQIzBAABCAAdFiEEjlF9wS7BzDf2QjqKE/E2UcnPDWsFAmKbqx4ACgkQE/E2UcnP +DWv5ng//WFFmaiVo+nz8uwtktwEsMsPidGErFY7mkyJxjtviYNZIgxZzuBKvguFO +wDaeoT90CiWIY0aSkLRopVr7E8oEX3ev6pT7iTsuEeZrJ8HITj5j/B6IwwGvclO7 +Qq1dIZUeNurTFwBXi6TmIPQMtncR9TUWi/20k/NyupsPxlVRBZNrslgKqxKyoXwz +lcGIV7IqAiUh87/1bdd2qk2K1PF3Z5ah7M2CdjBGRxnJnh0LfZuenFXqdF1EfYq0 +rmXQcYKeHWOcrT/WGpILBgyw7ER6+tqZ8cMD7VdPxX6fsiLiGLA23CCyS54vlnAw +HN+qOR9KFHaXRkzAn/VEpXoyVEDlDI3qFtqHubBPX4hMuSweQWnx4NcGFe/zXifN +SyQe5bXwxy0kiggDIczTcrvOCJUwKrYheNp7XHw/920Q1T91wGkk6b/3W2RJWIKx +jpnMYg+oASe7CFZ2C4pHvlBLOvV1bsPDOuOulti62OtdYAIYjFY2MIR63YyAx+2m +1SscL1GZfuU01K0XmW9PC1QoEF1tQFjIOI5Am3nivlfuqnfAWXmaKIg/dpvoWr1O +MQBiQpOrUjHv1fcC1nu3k/20BuzI2o5u99vdUFZzqhg7+VEMXAHnoGwY7+XVPG0A +VL1vltlOOQzJb0p3Zmt9U7bHE0/sWd/BdhXt4K58GzkgjtZ4aaM= +=VmAa +-----END PGP SIGNATURE-----