From dc0bd147b81c98c5b34a0010a94d169d54c631e2 Mon Sep 17 00:00:00 2001 From: tecnovert Date: Tue, 14 Feb 2023 23:34:01 +0200 Subject: [PATCH] tests: Add script test --- basicswap/basicswap.py | 92 ++- basicswap/http_server.py | 7 +- basicswap/interface/btc.py | 2 +- basicswap/interface/part.py | 25 +- basicswap/js_server.py | 140 ++++- doc/protocols/sequence_diagrams/notes.txt | 8 + scripts/.gitignore | 3 + scripts/createoffers.py | 536 +++++++++++++++--- tests/basicswap/extended/test_scripts.py | 654 ++++++++++++++++++++++ tests/basicswap/test_btc_xmr.py | 6 + tests/basicswap/test_run.py | 12 + 11 files changed, 1379 insertions(+), 106 deletions(-) create mode 100644 tests/basicswap/extended/test_scripts.py diff --git a/basicswap/basicswap.py b/basicswap/basicswap.py index edebcfb..b17d7ea 100644 --- a/basicswap/basicswap.py +++ b/basicswap/basicswap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2019-2022 tecnovert +# Copyright (c) 2019-2023 tecnovert # Distributed under the MIT software license, see the accompanying # file LICENSE or http://www.opensource.org/licenses/mit-license.php. @@ -867,7 +867,7 @@ class BasicSwap(BaseApp): def updateIdentityBidState(self, session, address: str, bid) -> None: identity_stats = session.query(KnownIdentity).filter_by(address=address).first() if not identity_stats: - identity_stats = KnownIdentity(address=address, created_at=int(time.time())) + identity_stats = KnownIdentity(active_ind=1, address=address, created_at=int(time.time())) if bid.state == BidStates.SWAP_COMPLETED: if bid.was_sent: @@ -1171,7 +1171,7 @@ class BasicSwap(BaseApp): def buildNotificationsCache(self, session): self._notifications_cache.clear() - q = session.execute(f'SELECT created_at, event_type, event_data FROM notifications WHERE active_ind=1 ORDER BY created_at ASC LIMIT {self._show_notifications}') + q = session.execute(f'SELECT created_at, event_type, event_data FROM notifications WHERE active_ind = 1 ORDER BY created_at ASC LIMIT {self._show_notifications}') for entry in q: self._notifications_cache[entry[0]] = (entry[1], json.loads(entry[2].decode('UTF-8'))) @@ -1181,6 +1181,71 @@ class BasicSwap(BaseApp): rv.append((time.strftime('%d-%m-%y %H:%M:%S', time.localtime(k)), int(v[0]), v[1])) return rv + def setIdentityData(self, filters, data): + address = filters['address'] + ci = self.ci(Coins.PART) + ensure(ci.isValidAddress(address), 'Invalid identity address') + + try: + now = int(time.time()) + session = self.openSession() + q = session.execute(f'SELECT COUNT(*) FROM knownidentities WHERE address = "{address}"').first() + if q[0] < 1: + q = session.execute(f'INSERT INTO knownidentities (active_ind, address, created_at) VALUES (1, "{address}", {now})') + + values = [] + pattern = '' + if 'label' in data: + pattern += (', ' if pattern != '' else '') + pattern += 'label = "{}"'.format(data['label']) + values.append(address) + q = session.execute(f'UPDATE knownidentities SET {pattern} WHERE address = "{address}"') + + finally: + self.closeSession(session) + + def listIdentities(self, filters): + try: + session = self.openSession() + + query_str = 'SELECT address, label, num_sent_bids_successful, num_recv_bids_successful, ' + \ + ' num_sent_bids_rejected, num_recv_bids_rejected, num_sent_bids_failed, num_recv_bids_failed ' + \ + ' FROM knownidentities ' + \ + ' WHERE active_ind = 1 ' + + address = filters.get('address', None) + if address is not None: + query_str += f' AND address = "{address}" ' + + sort_dir = filters.get('sort_dir', 'DESC').upper() + sort_by = filters.get('sort_by', 'created_at') + query_str += f' ORDER BY {sort_by} {sort_dir}' + + limit = filters.get('limit', None) + if limit is not None: + query_str += f' LIMIT {limit}' + offset = filters.get('offset', None) + if offset is not None: + query_str += f' OFFSET {offset}' + + q = session.execute(query_str) + rv = [] + for row in q: + identity = { + 'address': row[0], + 'label': row[1], + 'num_sent_bids_successful': zeroIfNone(row[2]), + 'num_recv_bids_successful': zeroIfNone(row[3]), + 'num_sent_bids_rejected': zeroIfNone(row[4]), + 'num_recv_bids_rejected': zeroIfNone(row[5]), + 'num_sent_bids_failed': zeroIfNone(row[6]), + 'num_recv_bids_failed': zeroIfNone(row[7]), + } + rv.append(identity) + return rv + finally: + self.closeSession(session) + def vacuumDB(self): try: session = self.openSession() @@ -2127,8 +2192,9 @@ class BasicSwap(BaseApp): session = scoped_session(self.session_factory) identity = session.query(KnownIdentity).filter_by(address=address).first() if identity is None: - identity = KnownIdentity(address=address) + identity = KnownIdentity(active_ind=1, address=address) identity.label = label + identity.updated_at = int(time.time()) session.add(identity) session.commit() finally: @@ -5434,7 +5500,6 @@ class BasicSwap(BaseApp): settings_changed = True if settings_changed: - settings_path = os.path.join(self.data_dir, cfg.CONFIG_FILENAME) settings_path_new = settings_path + '.new' shutil.copyfile(settings_path, settings_path + '.last') @@ -5838,8 +5903,9 @@ class BasicSwap(BaseApp): filter_coin_to = filters.get('coin_to', None) if filter_coin_to and filter_coin_to > -1: q = q.filter(Offer.coin_to == int(filter_coin_to)) + filter_include_sent = filters.get('include_sent', None) - if filter_include_sent and filter_include_sent is not True: + if filter_include_sent is not None and filter_include_sent is not True: q = q.filter(Offer.was_sent == False) # noqa: E712 order_dir = filters.get('sort_dir', 'desc') @@ -5874,15 +5940,14 @@ class BasicSwap(BaseApp): session.remove() self.mxDB.release() - def listBids(self, sent=False, offer_id=None, for_html=False, filters={}, with_identity_info=False): + def listBids(self, sent=False, offer_id=None, for_html=False, filters={}): self.mxDB.acquire() try: rv = [] now = int(time.time()) session = scoped_session(self.session_factory) - identity_fields = '' - query_str = 'SELECT bids.created_at, bids.expire_at, bids.bid_id, bids.offer_id, bids.amount, bids.state, bids.was_received, tx1.state, tx2.state, offers.coin_from, bids.rate, bids.bid_addr {} FROM bids '.format(identity_fields) + \ + query_str = 'SELECT bids.created_at, bids.expire_at, bids.bid_id, bids.offer_id, bids.amount, bids.state, bids.was_received, tx1.state, tx2.state, offers.coin_from, bids.rate, bids.bid_addr FROM bids ' + \ 'LEFT JOIN offers ON offers.offer_id = bids.offer_id ' + \ 'LEFT JOIN transactions AS tx1 ON tx1.bid_id = bids.bid_id AND tx1.tx_type = {} '.format(TxTypes.ITX) + \ 'LEFT JOIN transactions AS tx2 ON tx2.bid_id = bids.bid_id AND tx2.tx_type = {} '.format(TxTypes.PTX) @@ -5901,9 +5966,14 @@ class BasicSwap(BaseApp): bid_state_ind = filters.get('bid_state_ind', -1) if bid_state_ind != -1: query_str += 'AND bids.state = {} '.format(bid_state_ind) + + with_available_or_active = filters.get('with_available_or_active', False) with_expired = filters.get('with_expired', True) - if with_expired is not True: - query_str += 'AND bids.expire_at > {} '.format(now) + if with_available_or_active: + query_str += 'AND (bids.state NOT IN ({}, {}, {}, {}, {}) AND (bids.state > {} OR bids.expire_at > {})) '.format(BidStates.SWAP_COMPLETED, BidStates.BID_ERROR, BidStates.BID_REJECTED, BidStates.SWAP_TIMEDOUT, BidStates.BID_ABANDONED, BidStates.BID_RECEIVED, now) + else: + if with_expired is not True: + query_str += 'AND bids.expire_at > {} '.format(now) sort_dir = filters.get('sort_dir', 'DESC').upper() sort_by = filters.get('sort_by', 'created_at') diff --git a/basicswap/http_server.py b/basicswap/http_server.py index 0616e1f..a06f3b4 100644 --- a/basicswap/http_server.py +++ b/basicswap/http_server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2019-2022 tecnovert +# Copyright (c) 2019-2023 tecnovert # Distributed under the MIT software license, see the accompanying # file LICENSE or http://www.opensource.org/licenses/mit-license.php. @@ -697,11 +697,8 @@ class HttpThread(threading.Thread, HTTPServer): data = response.read() conn.close() - def stopped(self): - return self.stop_event.is_set() - def serve_forever(self): - while not self.stopped(): + while not self.stop_event.is_set(): self.handle_request() self.socket.close() diff --git a/basicswap/interface/btc.py b/basicswap/interface/btc.py index dd3c50f..391a799 100644 --- a/basicswap/interface/btc.py +++ b/basicswap/interface/btc.py @@ -1237,7 +1237,7 @@ class BTCInterface(CoinInterface): def describeTx(self, tx_hex: str): return self.rpc_callback('decoderawtransaction', [tx_hex]) - def getSpendableBalance(self): + def getSpendableBalance(self) -> int: return self.make_int(self.rpc_callback('getbalances')['mine']['trusted']) def createUTXO(self, value_sats: int): diff --git a/basicswap/interface/part.py b/basicswap/interface/part.py index c2108f6..d9b8481 100644 --- a/basicswap/interface/part.py +++ b/basicswap/interface/part.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2020-2022 tecnovert +# Copyright (c) 2020-2023 tecnovert # Distributed under the MIT software license, see the accompanying # file LICENSE or http://www.opensource.org/licenses/mit-license.php. @@ -77,10 +77,10 @@ class PARTInterface(BTCInterface): # TODO: Double check return True - def getNewAddress(self, use_segwit, label='swap_receive'): + def getNewAddress(self, use_segwit, label='swap_receive') -> str: return self.rpc_callback('getnewaddress', [label]) - def getNewStealthAddress(self, label='swap_stealth'): + def getNewStealthAddress(self, label='swap_stealth') -> str: return self.rpc_callback('getnewstealthaddress', [label]) def haveSpentIndex(self): @@ -105,7 +105,7 @@ class PARTInterface(BTCInterface): def getScriptForPubkeyHash(self, pkh): return CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]) - def formatStealthAddress(self, scan_pubkey, spend_pubkey): + def formatStealthAddress(self, scan_pubkey, spend_pubkey) -> str: prefix_byte = chainparams[self.coin_type()][self._network]['stealth_key_prefix'] return encodeStealthAddress(prefix_byte, scan_pubkey, spend_pubkey) @@ -116,7 +116,7 @@ class PARTInterface(BTCInterface): length += getWitnessElementLen(len(e) // 2) # hex -> bytes return length - def getWalletRestoreHeight(self): + def getWalletRestoreHeight(self) -> int: start_time = self.rpc_callback('getwalletinfo')['keypoololdest'] blockchaininfo = self.rpc_callback('getblockchaininfo') @@ -131,6 +131,15 @@ class PARTInterface(BTCInterface): block_header = self.rpc_callback('getblockheader', [block_hash]) return block_header['height'] + def isValidAddress(self, address: str) -> bool: + try: + rv = self.rpc_callback('validateaddress', [address]) + if rv['isvalid'] is True: + return True + except Exception as ex: + self._log.debug('validateaddress failed: {}'.format(address)) + return False + class PARTInterfaceBlind(PARTInterface): @staticmethod @@ -622,7 +631,7 @@ class PARTInterfaceBlind(PARTInterface): return bytes.fromhex(lock_refund_swipe_tx_hex) - def getSpendableBalance(self): + def getSpendableBalance(self) -> int: return self.make_int(self.rpc_callback('getbalances')['mine']['blind_trusted']) def publishBLockTx(self, vkbv, Kbs, output_amount, feerate, delay_for: int = 10, unlock_time: int = 0) -> bytes: @@ -840,7 +849,7 @@ class PARTInterfaceAnon(PARTInterface): rv = self.rpc_callback('sendtypeto', params) return bytes.fromhex(rv['txid']) - def findTxnByHash(self, txid_hex): + def findTxnByHash(self, txid_hex: str): # txindex is enabled for Particl try: @@ -854,5 +863,5 @@ class PARTInterfaceAnon(PARTInterface): return None - def getSpendableBalance(self): + def getSpendableBalance(self) -> int: return self.make_int(self.rpc_callback('getbalances')['mine']['anon_trusted']) diff --git a/basicswap/js_server.py b/basicswap/js_server.py index f48cfcc..95cebda 100644 --- a/basicswap/js_server.py +++ b/basicswap/js_server.py @@ -9,6 +9,7 @@ import random import urllib.parse from .util import ( + ensure, toBool, ) from .basicswap_util import ( @@ -38,7 +39,7 @@ from .ui.page_offers import postNewOffer from .protocols.xmr_swap_1 import recoverNoScriptTxnWithKey, getChainBSplitKey -def getFormData(post_string, is_json): +def getFormData(post_string: str, is_json: bool): if post_string == '': raise ValueError('No post data') if is_json: @@ -138,6 +139,7 @@ def js_offers(self, url_split, post_string, is_json, sent=False) -> bytes: return bytes(json.dumps(rv), 'UTF-8') offer_id = bytes.fromhex(url_split[3]) + with_extra_info = False filters = { 'coin_from': -1, 'coin_to': -1, @@ -174,12 +176,15 @@ def js_offers(self, url_split, post_string, is_json, sent=False) -> bytes: if have_data_entry(post_data, 'include_sent'): filters['include_sent'] = toBool(get_data_entry(post_data, 'include_sent')) + if have_data_entry(post_data, 'with_extra_info'): + with_extra_info = toBool(get_data_entry(post_data, 'with_extra_info')) + offers = swap_client.listOffers(sent, filters) rv = [] for o in offers: ci_from = swap_client.ci(o.coin_from) ci_to = swap_client.ci(o.coin_to) - rv.append({ + offer_data = { 'swap_type': o.swap_type, 'addr_from': o.addr_from, 'addr_to': o.addr_to, @@ -191,8 +196,11 @@ def js_offers(self, url_split, post_string, is_json, sent=False) -> bytes: 'amount_from': ci_from.format_amount(o.amount_from), 'amount_to': ci_to.format_amount((o.amount_from * o.rate) // ci_from.COIN()), 'rate': ci_to.format_amount(o.rate), - }) - + } + if with_extra_info: + offer_data['amount_negotiable'] = o.amount_negotiable + offer_data['rate_negotiable'] = o.rate_negotiable + rv.append(offer_data) return bytes(json.dumps(rv), 'UTF-8') @@ -200,7 +208,60 @@ def js_sentoffers(self, url_split, post_string, is_json) -> bytes: return js_offers(self, url_split, post_string, is_json, True) -def js_bids(self, url_split, post_string, is_json) -> bytes: +def parseBidFilters(post_data): + offer_id = None + filters = {} + + if have_data_entry(post_data, 'offer_id'): + offer_id = bytes.fromhex(get_data_entry(post_data, 'offer_id')) + assert (len(offer_id) == 28) + + if have_data_entry(post_data, 'sort_by'): + sort_by = get_data_entry(post_data, 'sort_by') + assert (sort_by in ['created_at', ]), 'Invalid sort by' + filters['sort_by'] = sort_by + if have_data_entry(post_data, 'sort_dir'): + sort_dir = get_data_entry(post_data, 'sort_dir') + assert (sort_dir in ['asc', 'desc']), 'Invalid sort dir' + filters['sort_dir'] = sort_dir + + if have_data_entry(post_data, 'offset'): + filters['offset'] = int(get_data_entry(post_data, 'offset')) + if have_data_entry(post_data, 'limit'): + filters['limit'] = int(get_data_entry(post_data, 'limit')) + assert (filters['limit'] > 0 and filters['limit'] <= PAGE_LIMIT), 'Invalid limit' + + if have_data_entry(post_data, 'with_available_or_active'): + filters['with_available_or_active'] = toBool(get_data_entry(post_data, 'with_available_or_active')) + elif have_data_entry(post_data, 'with_expired'): + filters['with_expired'] = toBool(get_data_entry(post_data, 'with_expired')) + + if have_data_entry(post_data, 'with_extra_info'): + filters['with_extra_info'] = toBool(get_data_entry(post_data, 'with_extra_info')) + + return offer_id, filters + + +def formatBids(swap_client, bids, filters) -> bytes: + with_extra_info = filters.get('with_extra_info', False) + rv = [] + for b in bids: + bid_data = { + 'bid_id': b[2].hex(), + 'offer_id': b[3].hex(), + 'created_at': b[0], + 'expire_at': b[1], + 'coin_from': b[9], + 'amount_from': swap_client.ci(b[9]).format_amount(b[4]), + 'bid_state': strBidState(b[5]) + } + if with_extra_info: + bid_data['addr_from'] = b[11] + rv.append(bid_data) + return bytes(json.dumps(rv), 'UTF-8') + + +def js_bids(self, url_split, post_string: str, is_json: bool) -> bytes: swap_client = self.server.swap_client swap_client.checkSystemStatus() if len(url_split) > 3: @@ -281,22 +342,21 @@ def js_bids(self, url_split, post_string, is_json) -> bytes: data = describeBid(swap_client, bid, xmr_swap, offer, xmr_offer, events, edit_bid, show_txns, for_api=True) return bytes(json.dumps(data), 'UTF-8') - bids = swap_client.listBids() - return bytes(json.dumps([{ - 'bid_id': b[2].hex(), - 'offer_id': b[3].hex(), - 'created_at': b[0], - 'expire_at': b[1], - 'coin_from': b[9], - 'amount_from': swap_client.ci(b[9]).format_amount(b[4]), - 'bid_state': strBidState(b[5]) - } for b in bids]), 'UTF-8') + post_data = getFormData(post_string, is_json) + offer_id, filters = parseBidFilters(post_data) + + bids = swap_client.listBids(offer_id=offer_id, filters=filters) + return formatBids(swap_client, bids, filters) def js_sentbids(self, url_split, post_string, is_json) -> bytes: swap_client = self.server.swap_client swap_client.checkSystemStatus() - return bytes(json.dumps(swap_client.listBids(sent=True)), 'UTF-8') + post_data = getFormData(post_string, is_json) + offer_id, filters = parseBidFilters(post_data) + + bids = swap_client.listBids(sent=True, offer_id=offer_id, filters=filters) + return formatBids(swap_client, bids, filters) def js_network(self, url_split, post_string, is_json) -> bytes: @@ -318,7 +378,7 @@ def js_smsgaddresses(self, url_split, post_string, is_json) -> bytes: swap_client = self.server.swap_client swap_client.checkSystemStatus() if len(url_split) > 3: - post_data = getFormData(post_string, is_json) + post_data = {} if post_string == '' else getFormData(post_string, is_json) if url_split[3] == 'new': addressnote = get_data_entry_or(post_data, 'addressnote', '') new_addr, pubkey = swap_client.newSMSGAddress(addressnote=addressnote) @@ -417,11 +477,54 @@ def js_generatenotification(self, url_split, post_string, is_json) -> bytes: def js_notifications(self, url_split, post_string, is_json) -> bytes: swap_client = self.server.swap_client swap_client.checkSystemStatus() - swap_client.getNotifications() return bytes(json.dumps(swap_client.getNotifications()), 'UTF-8') +def js_identities(self, url_split, post_string: str, is_json: bool) -> bytes: + swap_client = self.server.swap_client + swap_client.checkSystemStatus() + + filters = { + 'page_no': 1, + 'limit': PAGE_LIMIT, + 'sort_by': 'created_at', + 'sort_dir': 'desc', + } + + if len(url_split) > 3: + address = url_split[3] + filters['address'] = address + + if post_string != '': + post_data = getFormData(post_string, is_json) + + if have_data_entry(post_data, 'sort_by'): + sort_by = get_data_entry(post_data, 'sort_by') + assert (sort_by in ['created_at', 'rate']), 'Invalid sort by' + filters['sort_by'] = sort_by + if have_data_entry(post_data, 'sort_dir'): + sort_dir = get_data_entry(post_data, 'sort_dir') + assert (sort_dir in ['asc', 'desc']), 'Invalid sort dir' + filters['sort_dir'] = sort_dir + + if have_data_entry(post_data, 'offset'): + filters['offset'] = int(get_data_entry(post_data, 'offset')) + if have_data_entry(post_data, 'limit'): + filters['limit'] = int(get_data_entry(post_data, 'limit')) + assert (filters['limit'] > 0 and filters['limit'] <= PAGE_LIMIT), 'Invalid limit' + + set_data = {} + if have_data_entry(post_data, 'set_label'): + set_data['label'] = get_data_entry(post_data, 'set_label') + + if set_data: + ensure('address' in filters, 'Must provide an address to modify data') + swap_client.setIdentityData(filters, set_data) + + return bytes(json.dumps(swap_client.listIdentities(filters)), 'UTF-8') + + def js_vacuumdb(self, url_split, post_string, is_json) -> bytes: swap_client = self.server.swap_client swap_client.checkSystemStatus() @@ -528,6 +631,7 @@ pages = { 'rateslist': js_rates_list, 'generatenotification': js_generatenotification, 'notifications': js_notifications, + 'identities': js_identities, 'vacuumdb': js_vacuumdb, 'getcoinseed': js_getcoinseed, 'setpassword': js_setpassword, diff --git a/doc/protocols/sequence_diagrams/notes.txt b/doc/protocols/sequence_diagrams/notes.txt index 5df4d60..e8e0c97 100644 --- a/doc/protocols/sequence_diagrams/notes.txt +++ b/doc/protocols/sequence_diagrams/notes.txt @@ -1,3 +1,11 @@ + +Rendered files can be found in: + +basicswap/static/sequence_diagrams + + +To render: + nvm use 14 npm install -g mscgenjs-cli diff --git a/scripts/.gitignore b/scripts/.gitignore index afed073..04ba415 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1 +1,4 @@ *.csv +*.json +*.last +*.sqlite diff --git a/scripts/createoffers.py b/scripts/createoffers.py index 73e3710..5ed0109 100755 --- a/scripts/createoffers.py +++ b/scripts/createoffers.py @@ -9,10 +9,13 @@ Create offers """ -__version__ = '0.1' +__version__ = '0.2' import os import json +import time +import random +import shutil import signal import urllib import logging @@ -22,23 +25,35 @@ from urllib.request import urlopen delay_event = threading.Event() - -def post_json_req(url, json_data): - req = urllib.request.Request(url) - req.add_header('Content-Type', 'application/json; charset=utf-8') - post_bytes = json.dumps(json_data).encode('utf-8') - req.add_header('Content-Length', len(post_bytes)) - return urlopen(req, post_bytes, timeout=300).read() +DEFAULT_CONFIG_FILE: str = 'createoffers.json' +DEFAULT_STATE_FILE: str = 'createoffers_state.json' -def read_json_api(port, path=None, json_data=None): - url = f'http://127.0.0.1:{port}/json' - if path is not None: - url += '/' + path +def post_req(url: str, json_data=None): + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + if json_data: + req.add_header('Content-Type', 'application/json; charset=utf-8') + post_bytes = json.dumps(json_data).encode('utf-8') + req.add_header('Content-Length', len(post_bytes)) + else: + post_bytes = None + return urlopen(req, data=post_bytes, timeout=300).read() - if json_data is not None: - return json.loads(post_json_req(url, json_data)) - return json.loads(urlopen(url, timeout=300).read()) + +def make_json_api_func(host: str, port: int): + host = host + port = port + + def api_func(path=None, json_data=None, timeout=300): + nonlocal host, port + url = f'http://{host}:{port}/json' + if path is not None: + url += '/' + path + if json_data is not None: + return json.loads(post_req(url, json_data)) + response = urlopen(url, timeout=300).read() + return json.loads(response) + return api_func def signal_handler(sig, frame) -> None: @@ -55,74 +70,208 @@ def findCoin(coin: str, known_coins) -> str: raise ValueError(f'Unknown coin {coin}') -def readTemplates(known_coins): - offer_templates = [] - with open('offer_rules.csv', 'r') as fp: - for i, line in enumerate(fp): - if i < 1: - continue - line = line.strip() - if line[0] == '#': - continue - row_data = line.split(',') - try: - if len(row_data) < 6: - raise ValueError('missing data') - offer_template = {} - offer_template['coin_from'] = findCoin(row_data[0], known_coins) - offer_template['coin_to'] = findCoin(row_data[1], known_coins) - offer_template['amount'] = row_data[2] - offer_template['minrate'] = float(row_data[3]) - offer_template['ratetweakpercent'] = float(row_data[4]) - offer_template['amount_variable'] = row_data[5].lower() in ('true', 1) - offer_template['address'] = row_data[6] - offer_templates.append(offer_template) - except Exception as e: - print(f'Warning: Skipping row {i}, {e}') - continue - return offer_templates +def readConfig(args, known_coins): + config_path: str = args.configfile + num_changes: int = 0 + with open(config_path) as fs: + config = json.load(fs) + + if not 'offers' in config: + config['offers'] = [] + if not 'bids' in config: + config['bids'] = [] + if not 'stealthex' in config: + config['stealthex'] = [] + + if not 'min_seconds_between_offers' in config: + config['min_seconds_between_offers'] = 60 + print('Set min_seconds_between_offers', config['min_seconds_between_offers']) + num_changes += 1 + if not 'max_seconds_between_offers' in config: + config['max_seconds_between_offers'] = config['min_seconds_between_offers'] * 4 + print('Set max_seconds_between_offers', config['max_seconds_between_offers']) + num_changes += 1 + + if not 'min_seconds_between_bids' in config: + config['min_seconds_between_bids'] = 60 + print('Set min_seconds_between_bids', config['min_seconds_between_bids']) + num_changes += 1 + if not 'max_seconds_between_bids' in config: + config['max_seconds_between_bids'] = config['min_seconds_between_bids'] * 4 + print('Set max_seconds_between_bids', config['max_seconds_between_bids']) + num_changes += 1 + + offer_templates = config['offers'] + offer_templates_map = {} + num_enabled = 0 + for i, offer_template in enumerate(offer_templates): + num_enabled += 1 if offer_template.get('enabled', True) else 0 + if 'name' not in offer_template: + print('naming offer template', i) + offer_template['name'] = f'Offer {i}' + num_changes += 1 + if offer_template.get('min_coin_from_amt', 0) < offer_template['amount']: + print('Setting min_coin_from_amt for', offer_template['name']) + offer_template['min_coin_from_amt'] = offer_template['amount'] + num_changes += 1 + + if offer_template.get('enabled', True) is False: + continue + offer_template['coin_from'] = findCoin(offer_template['coin_from'], known_coins) + offer_template['coin_to'] = findCoin(offer_template['coin_to'], known_coins) + + if offer_template['name'] in offer_templates_map: + print('renaming offer template', offer_template['name']) + original_name = offer_template['name'] + offset = 2 + while f'{original_name}_{offset}' in offer_templates_map: + offset += 1 + offer_template['name'] = f'{original_name}_{offset}' + num_changes += 1 + offer_templates_map[offer_template['name']] = offer_template + config['num_enabled_offers'] = num_enabled + + bid_templates = config['bids'] + bid_templates_map = {} + num_enabled = 0 + for i, bid_template in enumerate(bid_templates): + num_enabled += 1 if bid_template.get('enabled', True) else 0 + if 'name' not in bid_template: + print('naming bid template', i) + bid_template['name'] = f'Bid {i}' + num_changes += 1 + + if bid_template.get('enabled', True) is False: + continue + + if bid_template.get('min_swap_amount', 0.0) < 0.00001: + print('Setting min_swap_amount for bid template', bid_template['name']) + bid_template['min_swap_amount'] = 0.00001 + + bid_template['coin_from'] = findCoin(bid_template['coin_from'], known_coins) + bid_template['coin_to'] = findCoin(bid_template['coin_to'], known_coins) + + if bid_template['name'] in bid_templates_map: + print('renaming bid template', offer_templates_map_template['name']) + original_name = bid_template['name'] + offset = 2 + while f'{original_name}_{offset}' in bid_templates_map: + offset += 1 + bid_template['name'] = f'{original_name}_{offset}' + num_changes += 1 + bid_templates_map[bid_template['name']] = bid_template + config['num_enabled_bids'] = num_enabled + + num_enabled = 0 + stealthex_swaps = config['stealthex'] + for i, swap in enumerate(stealthex_swaps): + num_enabled += 1 if swap.get('enabled', True) else 0 + swap['coin_from'] = findCoin(swap['coin_from'], known_coins) + #bid_template['coin_to'] = findCoin(bid_template['coin_to'], known_coins) + config['num_enabled_swaps'] = num_enabled + + if num_changes > 0: + shutil.copyfile(config_path, config_path + '.last') + with open(config_path, 'w') as fp: + json.dump(config, fp, indent=4) + + return config def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--version', action='version', version='%(prog)s {version}'.format(version=__version__)) + parser.add_argument('--host', dest='host', help='RPC host (default=127.0.0.1)', type=str, default='127.0.0.1', required=False) parser.add_argument('--port', dest='port', help='RPC port (default=12700)', type=int, default=12700, required=False) + parser.add_argument('--oneshot', dest='oneshot', help='Exit after one iteration (default=false)', required=False, action='store_true') + parser.add_argument('--debug', dest='debug', help='Print extra debug messages (default=false)', required=False, action='store_true') + parser.add_argument('--configfile', dest='configfile', help=f'config file path (default={DEFAULT_CONFIG_FILE})', type=str, default=DEFAULT_CONFIG_FILE, required=False) + parser.add_argument('--statefile', dest='statefile', help=f'state file path (default={DEFAULT_STATE_FILE})', type=str, default=DEFAULT_STATE_FILE, required=False) args = parser.parse_args() - if not os.path.exists('offer_rules.csv'): - with open('offer_rules.csv', 'w') as fp: - # Set address to -1 to use new addresses - fp.write('coin from,coin to,offer value,min rate,rate tweak percent,amount variable,address') + read_json_api = make_json_api_func(args.host, args.port) - known_coins = read_json_api(args.port, 'coins') + if not os.path.exists(args.configfile): + raise ValueError(f'Config file "{args.configfile}" not found.') + + known_coins = read_json_api('coins') coins_map = {} for known_coin in known_coins: coins_map[known_coin['name']] = known_coin + script_state = {} + if os.path.exists(args.statefile): + with open(args.statefile) as fs: + script_state = json.load(fs) + signal.signal(signal.SIGINT, signal_handler) while not delay_event.is_set(): - # Read templates each iteration so they can be modified without restarting - offer_templates = readTemplates(known_coins) + # Read config each iteration so they can be modified without restarting + config = readConfig(args, known_coins) + offer_templates = config['offers'] + random.shuffle(offer_templates) + bid_templates = config['bids'] + random.shuffle(bid_templates) + + stealthex_swaps = config['stealthex'] + random.shuffle(bid_templates) + + # override wallet api calls for testing + if 'wallet_port_override' in config: + wallet_api_port = int(config['wallet_port_override']) + print(f'Overriding wallet api port: {wallet_api_port}') + read_json_api_wallet = make_json_api_func(args.host, wallet_api_port) + else: + read_json_api_wallet = read_json_api + + num_state_changes: int = 0 try: - recieved_offers = read_json_api(args.port, 'offers', {'active': 'active', 'include_sent': False}) - print('recieved_offers', recieved_offers) - - sent_offers = read_json_api(args.port, 'sentoffers', {'active': 'active'}) + sent_offers = read_json_api('sentoffers', {'active': 'active'}) + if args.debug and len(offer_templates) > 0: + print('Processing {} offer templates'.format(config['num_enabled_offers'])) for offer_template in offer_templates: offers_found = 0 - for offer in sent_offers: - if offer['coin_from'] == offer_template['coin_from'] and offer['coin_to'] == offer_template['coin_to']: - offers_found += 1 - if offers_found > 0: - continue coin_from_data = coins_map[offer_template['coin_from']] coin_to_data = coins_map[offer_template['coin_to']] - rates = read_json_api(args.port, 'rates', {'coin_from': coin_from_data['id'], 'coin_to': coin_to_data['id']}) + wallet_from = read_json_api_wallet('wallets/{}'.format(coin_from_data['ticker'])) + + for offer in sent_offers: + created_offers = script_state.get('offers', {}) + prev_template_offers = created_offers.get(offer_template['name'], {}) + + if next((x for x in prev_template_offers if x['offer_id'] == offer['offer_id']), None): + offers_found += 1 + if float(wallet_from['balance']) <= float(offer_template['min_coin_from_amt']): + offer_id = offer['offer_id'] + print('Revoking offer {}, wallet from balance below minimum'.format(offer_id)) + result = read_json_api(f'revokeoffer/{offer_id}') + print('revokeoffer', result) + + if offers_found > 0: + continue + + if float(wallet_from['balance']) <= float(offer_template['min_coin_from_amt']): + print('Skipping template {}, wallet from balance below minimum'.format(offer_template['name'])) + continue + + delay_next_offer_before = script_state.get('delay_next_offer_before', 0) + if delay_next_offer_before > int(time.time()): + print('Delaying offers until {}'.format(delay_next_offer_before)) + break + + """ + recieved_offers = read_json_api(args.port, 'offers', {'active': 'active', 'include_sent': False, 'coin_from': coin_from_data['id'], 'coin_to': coin_to_data['id']}) + print('recieved_offers', recieved_offers) + + TODO - adjust rates based on extisting offers + """ + + rates = read_json_api('rates', {'coin_from': coin_from_data['id'], 'coin_to': coin_to_data['id']}) print('Rates', rates) coingecko_rate = float(rates['coingecko']['rate_inferred']) use_rate = coingecko_rate @@ -137,21 +286,282 @@ def main(): use_rate = offer_template['minrate'] print('Creating offer for: {} at rate: {}'.format(offer_template, use_rate)) + template_from_addr = offer_template['address'] offer_data = { - 'addr_from': offer_template['address'], + 'addr_from': -1 if template_from_addr == 'auto' else template_from_addr, 'coin_from': coin_from_data['ticker'], 'coin_to': coin_to_data['ticker'], 'amt_from': offer_template['amount'], 'amt_var': offer_template['amount_variable'], + 'valid_for_seconds': offer_template.get('offer_valid_seconds', config.get('offer_valid_seconds', 3600)), 'rate': use_rate, 'swap_type': 'adaptor_sig', 'lockhrs': '24', 'automation_strat_id': 1} - new_offer = read_json_api(args.port, 'offers/new', offer_data) - print('New offer: {}'.format(new_offer)) - except Exception as e: - print('Error: Clamping rate to minimum.') + if args.debug: + print('offer data {}'.format(offer_data)) + new_offer = read_json_api('offers/new', offer_data) + print('New offer: {}'.format(new_offer['offer_id'])) + num_state_changes += 1 + if not 'offers' in script_state: + script_state['offers'] = {} + template_name = offer_template['name'] + if not template_name in script_state['offers']: + script_state['offers'][template_name] = [] + script_state['offers'][template_name].append({'offer_id': new_offer['offer_id'], 'time': int(time.time())}) + max_seconds_between_offers = config['max_seconds_between_offers'] + min_seconds_between_offers = config['min_seconds_between_offers'] + if max_seconds_between_offers > min_seconds_between_offers: + time_between_offers = random.randint(min_seconds_between_offers, max_seconds_between_offers) + else: + time_between_offers = min_seconds_between_offers + script_state['delay_next_offer_before'] = int(time.time()) + time_between_offers + if args.debug and len(bid_templates) > 0: + print('Processing {} bid templates'.format(config['num_enabled_bids'])) + for bid_template in bid_templates: + + delay_next_bid_before = script_state.get('delay_next_bid_before', 0) + if delay_next_bid_before > int(time.time()): + print('Delaying bids until {}'.format(delay_next_bid_before)) + break + + # Check bids in progress + max_concurrent = bid_template.get('max_concurrent', 1) + if not 'bids' in script_state: + script_state['bids'] = {} + template_name = bid_template['name'] + if not template_name in script_state['bids']: + script_state['bids'][template_name] = [] + previous_bids = script_state['bids'][template_name] + + bids_in_progress: int = 0 + for previous_bid in previous_bids: + if not previous_bid['active']: + continue + previous_bid_id = previous_bid['bid_id'] + previous_bid_info = read_json_api(f'bids/{previous_bid_id}') + bid_state = previous_bid_info['bid_state'] + if bid_state in ('Completed', 'Timed-out', 'Abandoned', 'Error', 'Rejected'): + print(f'Marking bid inactive {previous_bid_id}, state {bid_state}') + previous_bid['active'] = False + num_state_changes += 1 + continue + if bid_state in ('Sent', 'Received') and previous_bid_info['expired_at'] < int(time.time()): + print(f'Marking bid inactive {previous_bid_id}, expired') + previous_bid['active'] = False + num_state_changes += 1 + continue + bids_in_progress += 1 + + if bids_in_progress >= max_concurrent: + print('Max concurrent bids reached for template') + continue + + # Bidder sends coin_to and receives coin_from + coin_from_data = coins_map[bid_template['coin_from']] + coin_to_data = coins_map[bid_template['coin_to']] + + offers_options = { + 'active': 'active', + 'include_sent': False, + 'coin_from': coin_from_data['id'], + 'coin_to': coin_to_data['id'], + 'with_extra_info': True, + 'sort_by': 'rate', + 'sort_dir': 'asc', + } + + recieved_offers = read_json_api('offers', offers_options) + print('recieved_offers', recieved_offers) + + for offer in recieved_offers: + offer_id = offer['offer_id'] + offer_amount = float(offer['amount_from']) + offer_rate = float(offer['rate']) + bid_amount = offer_amount + + min_swap_amount = bid_template.get('min_swap_amount', 0.01) # TODO: Make default vary per coin + can_adjust_amount: bool = offer['amount_negotiable'] and bid_template.get('amount_variable', True) + if can_adjust_amount is False and offer_amount > bid_template['amount']: + if args.debug: + print(f'Bid amount too low for offer {offer_id}') + continue + if (can_adjust_amount is False and offer_amount < bid_template['amount']) or offer_amount < min_swap_amount: + if args.debug: + print(f'Offer amount too low for bid {offer_id}') + continue + + if offer_rate > bid_template['maxrate']: + if args.debug: + print(f'Bid rate too low for offer {offer_id}') + continue + + sent_bids = read_json_api('sentbids', {'offer_id': offer['offer_id'], 'with_available_or_active': True}) + if len(sent_bids) > 0: + if args.debug: + print(f'Already bidding on offer {offer_id}') + continue + + offer_identity = read_json_api('identities/{}'.format(offer['addr_from'])) + if len(offer_identity) > 0: + successful_sent_bids = offer_identity[0]['num_sent_bids_successful'] + failed_sent_bids = offer_identity[0]['num_sent_bids_failed'] + if failed_sent_bids > 3 and failed_sent_bids > successful_sent_bids: + if args.debug: + print(f'Not bidding on offer {offer_id}, too many failed bids ({failed_sent_bids}).') + continue + + max_coin_from_balance = bid_template.get('max_coin_from_balance', -1) + if max_coin_from_balance > 0: + wallet_from = read_json_api_wallet('wallets/{}'.format(coin_from_data['ticker'])) + total_balance_from = float(wallet_from['balance']) + float(wallet_from['unconfirmed']) + if args.debug: + print(f'Total coin from balance {total_balance_from}') + if total_balance_from + bid_amount > max_coin_from_balance: + if can_adjust_amount and max_coin_from_balance - total_balance_from > min_swap_amount: + bid_amount = max_coin_from_balance - total_balance_from + print(f'Reduced bid amount to {bid_amount}') + else: + if args.debug: + print(f'Bid amount would exceed maximum wallet total for offer {offer_id}') + continue + + min_coin_to_balance = bid_template['min_coin_to_balance'] + if min_coin_to_balance > 0: + wallet_to = read_json_api_wallet('wallets/{}'.format(coin_to_data['ticker'])) + + total_balance_to = float(wallet_to['balance']) + float(wallet_to['unconfirmed']) + if args.debug: + print(f'Total coin to balance {total_balance_to}') + + swap_amount_to = bid_amount * offer_rate + if total_balance_to - swap_amount_to < min_coin_to_balance: + if can_adjust_amount: + adjusted_swap_amount_to = total_balance_to - min_coin_to_balance + adjusted_bid_amount = adjusted_swap_amount_to / offer_rate + + if adjusted_bid_amount > min_swap_amount: + print(f'Reduced bid amount to {bid_amount}') + bid_amount = adjusted_bid_amount + swap_amount_to = adjusted_bid_amount * offer_rate + + if total_balance_to - swap_amount_to < min_coin_to_balance: + if args.debug: + print(f'Bid amount would exceed minimum coin to wallet total for offer {offer_id}') + continue + + bid_data = { + 'offer_id': offer['offer_id'], + 'amount_from': bid_amount} + + if 'address' in bid_template: + addr_from = bid_template['address'] + if addr_from != -1 and addr_from != 'auto': + bid_data['addr_from'] = addr_from + + if config.get('test_mode', False): + print('Would create bid: {}'.format(bid_data)) + bid_id = 'simulated' + else: + if args.debug: + print('Creating bid: {}'.format(bid_data)) + new_bid = read_json_api('bids/new', bid_data) + print('New bid: {} on offer {}'.format(new_bid['bid_id'], offer['offer_id'])) + bid_id = new_bid['bid_id'] + + num_state_changes += 1 + script_state['bids'][template_name].append({'bid_id': bid_id, 'time': int(time.time()), 'active': True}) + + max_seconds_between_bids = config['max_seconds_between_bids'] + min_seconds_between_bids = config['min_seconds_between_bids'] + if max_seconds_between_bids > min_seconds_between_bids: + time_between_bids = random.randint(min_seconds_between_bids, max_seconds_between_bids) + else: + time_between_bids = min_seconds_between_bids + script_state['delay_next_bid_before'] = int(time.time()) + time_between_bids + break # Create max one bid per iteration + + if args.debug and len(stealthex_swaps) > 0: + print('Processing {} stealthex templates'.format(config['num_enabled_swaps'])) + for stealthex_swap in stealthex_swaps: + if stealthex_swap.get('enabled', True) is False: + continue + coin_from_data = coins_map[stealthex_swap['coin_from']] + + wallet_from = read_json_api_wallet('wallets/{}'.format(coin_from_data['ticker'])) + + current_balance = float(wallet_from['balance']) + + min_balance_from = float(stealthex_swap['min_balance_from']) + min_swap_amount = float(stealthex_swap['min_amount_tx']) + max_swap_amount = float(stealthex_swap['max_amount_tx']) + + # TODO: Check range limits + + if current_balance >= min_balance_from + min_swap_amount: + swap_amount = max_swap_amount + if current_balance - swap_amount < min_balance_from: + swap_amount = max(min_swap_amount, current_balance - min_balance_from) + + estimate_url = 'https://api.stealthex.io/api/v2/estimate/{}/{}?amount={}&api_key={}&fixed=true'.format(coin_from_data['ticker'].lower(), stealthex_swap['coin_to'].lower(), swap_amount, stealthex_swap['api_key']) + if args.debug: + print(f'Estimate URL: {estimate_url}') + estimate_response = json.loads(post_req(estimate_url)) + + amount_to = float(estimate_response['estimated_amount']) + rate = swap_amount / amount_to + min_rate = float(stealthex_swap['min_rate']) + if rate < min_rate: + if args.debug: + print('Stealthex rate {} below minimum {} for {} to {}'.format(rate, min_rate, coin_from_data['ticker'], stealthex_swap['coin_to'])) + continue + + exchange_url = 'https://api.stealthex.io/api/v2/exchange?api_key={}'.format(stealthex_swap['api_key']) + + address_to = stealthex_swap.get('receive_address', 'auto') + if address_to == 'auto': + address_to = read_json_api('wallets/{}/nextdepositaddr'.format(stealthex_swap['coin_to'])) + + address_refund = stealthex_swap.get('refund_address', 'auto') + if address_refund == 'auto': + address_refund = read_json_api('wallets/{}/nextdepositaddr'.format(coin_from_data['ticker'])) + + exchange_data = { + 'currency_from': coin_from_data['ticker'].lower(), + 'currency_to': stealthex_swap['coin_to'].lower(), + 'address_to': address_to, + 'amount_from': swap_amount, + 'fixed': True, + #'extra_id_to': + #'referral': + 'refund_address': address_refund, + #'refund_extra_id': + 'rate_id': estimate_response['rate_id'], + } + + if args.debug: + print(f'Exchange URL: {estimate_url}') + print(f'Exchange data: {exchange_data}') + + exchange_response = json.loads(post_req(exchange_url, exchange_data)) + + if 'Error' in estimate_response: + raise ValueError('Exchange error ' + estimate_response) + + raise ValueError('TODO') + + if num_state_changes > 0: + if os.path.exists(args.statefile): + shutil.copyfile(args.statefile, args.statefile + '.last') + with open(args.statefile, 'w') as fp: + json.dump(script_state, fp, indent=4) + + except Exception as e: + print(f'Error: {e}.') + + if args.oneshot: + break print('Looping indefinitely, ctrl+c to exit.') delay_event.wait(60) diff --git a/tests/basicswap/extended/test_scripts.py b/tests/basicswap/extended/test_scripts.py new file mode 100644 index 0000000..12ca7b3 --- /dev/null +++ b/tests/basicswap/extended/test_scripts.py @@ -0,0 +1,654 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 tecnovert +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +""" +Start test_xmr_persistent.py + +python tests/basicswap/extended/test_scripts.py + +pytest -v -s tests/basicswap/extended/test_scripts.py::Test::test_bid_tracking + +""" + +import os +import sys +import json +import time +import math +import logging +import sqlite3 +import unittest +import threading +import subprocess +import http.client +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib import parse + +from tests.basicswap.util import ( + read_json_api, + waitForServer, +) + + +logger = logging.getLogger() +logger.level = logging.DEBUG +if not len(logger.handlers): + logger.addHandler(logging.StreamHandler(sys.stdout)) + + +PORT_OFS = int(os.getenv('PORT_OFS', 1)) +UI_PORT = 12700 + PORT_OFS + + +class HttpHandler(BaseHTTPRequestHandler): + + def js_response(self, url_split, post_string, is_json): + return bytes(json.dumps(self.server.return_data[url_split[3]]), 'UTF-8') + + def putHeaders(self, status_code, content_type): + self.send_response(status_code) + self.send_header('Content-Type', content_type) + self.end_headers() + + def handle_http(self, status_code, path, post_string='', is_json=False): + parsed = parse.urlparse(self.path) + url_split = parsed.path.split('/') + if post_string == '' and len(parsed.query) > 0: + post_string = parsed.query + if len(url_split) > 1 and url_split[1] == 'json': + self.putHeaders(status_code, 'text/plain') + return self.js_response(url_split, post_string, is_json) + + self.putHeaders(status_code, 'text/plain') + return bytes('No response', 'UTF-8') + + def do_GET(self): + response = self.handle_http(200, self.path) + self.wfile.write(response) + + def do_POST(self): + post_string = self.rfile.read(int(self.headers.get('Content-Length'))) + + is_json = True if 'json' in self.headers.get('Content-Type', '') else False + response = self.handle_http(200, self.path, post_string, is_json) + self.wfile.write(response) + + def do_HEAD(self): + self.putHeaders(200, 'text/html') + + +class HttpThread(threading.Thread, HTTPServer): + host = '127.0.0.1' + port_no = 12699 + stop_event = threading.Event() + return_data = {'test': 1} + + def __init__(self): + threading.Thread.__init__(self) + + HTTPServer.__init__(self, (self.host, self.port_no), HttpHandler) + + def stop(self): + self.stop_event.set() + + # Send fake request + conn = http.client.HTTPConnection(self.host, self.port_no) + conn.connect() + conn.request('GET', '/none') + response = conn.getresponse() + data = response.read() + conn.close() + + def serve_forever(self): + while not self.stop_event.is_set(): + self.handle_request() + self.socket.close() + + def run(self): + self.serve_forever() + + +def clear_offers(delay_event, node_id) -> None: + logging.info(f'clear_offers node {node_id}') + offers = read_json_api(UI_PORT + node_id, 'offers') + + for offer in offers: + read_json_api(UI_PORT + node_id, 'revokeoffer/{}'.format(offer['offer_id'])) + + for i in range(20): + delay_event.wait(1) + offers = read_json_api(UI_PORT + node_id, 'offers') + if len(offers) == 0: + return + raise ValueError('clear_offers failed') + + +def wait_for_offers(delay_event, node_id, num_offers) -> None: + logging.info(f'Waiting for {num_offers} offers on node {node_id}') + for i in range(20): + delay_event.wait(1) + offers = read_json_api(UI_PORT + node_id, 'offers') + if len(offers) >= num_offers: + return + raise ValueError('wait_for_offers failed') + + +def delete_file(filepath: str) -> None: + if os.path.exists(filepath): + os.remove(filepath) + + +def get_created_offers(rv_stdout): + offers = [] + for line in rv_stdout: + if line.startswith('New offer'): + offers.append(line.split(':')[1].strip()) + return offers + + +def count_lines_with(rv_stdout, str_needle): + lines_found = 0 + for line in rv_stdout: + if str_needle in line: + lines_found += 1 + return lines_found + + +def get_created_bids(rv_stdout): + bids = [] + for line in rv_stdout: + if line.startswith('New bid'): + bids.append(line.split(':')[1].strip()) + return bids + + +def get_possible_bids(rv_stdout): + bids = [] + tag = 'Would create bid: ' + for line in rv_stdout: + if line.startswith(tag): + bids.append(json.loads(line[len(tag):].replace("'", '"'))) + return bids + + +class Test(unittest.TestCase): + delay_event = threading.Event() + thread_http = HttpThread() + + @classmethod + def setUpClass(cls): + super(Test, cls).setUpClass() + cls.thread_http.start() + + script_path = 'scripts/createoffers.py' + datadir = '/tmp/bsx_scripts' + if not os.path.isdir(datadir): + os.makedirs(datadir) + + cls.node0_configfile = os.path.join(datadir, 'node0.json') + cls.node0_statefile = os.path.join(datadir, 'node0_state.json') + cls.node0_args = [script_path, '--port', str(UI_PORT), '--configfile', cls.node0_configfile, '--statefile', cls.node0_statefile, '--oneshot', '--debug'] + + cls.node1_configfile = os.path.join(datadir, 'node1.json') + cls.node1_statefile = os.path.join(datadir, 'node1_state.json') + cls.node1_args = [script_path, '--port', str(UI_PORT + 1), '--configfile', cls.node1_configfile, '--statefile', cls.node1_statefile, '--oneshot', '--debug'] + + + @classmethod + def tearDownClass(cls): + logging.info('Stopping test') + cls.thread_http.stop() + + def test_offers(self): + + waitForServer(self.delay_event, UI_PORT + 0) + waitForServer(self.delay_event, UI_PORT + 1) + + # Reset test + clear_offers(self.delay_event, 0) + delete_file(self.node0_statefile) + delete_file(self.node1_statefile) + wait_for_offers(self.delay_event, 1, 0) + + node0_test1_config = { + 'offers': [ + { + 'name': 'offer example 1', + 'coin_from': 'Particl', + 'coin_to': 'Monero', + 'amount': 20, + 'minrate': 0.05, + 'ratetweakpercent': 5, + 'amount_variable': True, + 'address': -1, + 'min_coin_from_amt': 20, + 'max_coin_to_amt': -1 + }, + { + 'name': 'offer example 1_2', + 'coin_from': 'Particl', + 'coin_to': 'Monero', + 'amount': 21, + 'minrate': 0.07, + 'ratetweakpercent': 5, + 'amount_variable': True, + 'address': -1, + 'min_coin_from_amt': 21, + 'max_coin_to_amt': -1 + } + ], + } + with open(self.node0_configfile, 'w') as fp: + json.dump(node0_test1_config, fp, indent=4) + + logging.info('Test that an offer is created') + result = subprocess.run(self.node0_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_offers(rv_stdout)) == 1) + + offers = read_json_api(UI_PORT, 'offers') + assert (len(offers) == 1) + + logging.info('Test that an offer is not created while delaying') + result = subprocess.run(self.node0_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_offers(rv_stdout)) == 0) + + with open(self.node0_statefile) as fs: + node0_state = json.load(fs) + node0_state['delay_next_offer_before'] = 0 + with open(self.node0_statefile, 'w') as fp: + json.dump(node0_state, fp, indent=4) + + logging.info('Test that the second offer is created when not delaying') + result = subprocess.run(self.node0_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_offers(rv_stdout)) == 1) + + with open(self.node0_statefile) as fs: + node0_state = json.load(fs) + assert (len(node0_state['offers']['offer example 1']) == 1) + assert (len(node0_state['offers']['offer example 1_2']) == 1) + + offers = read_json_api(UI_PORT, 'offers') + assert (len(offers) == 2) + + addr_bid_from = read_json_api(UI_PORT + 1, 'smsgaddresses/new')['new_address'] + node1_test1_config = { + 'bids': [ + { + 'name': 'bid example 1', + 'coin_from': 'PART', + 'coin_to': 'XMR', + 'amount': 10, + 'maxrate': 0.06, + 'amount_variable': True, + 'address': addr_bid_from, + 'min_swap_amount': 0.1, + 'max_coin_from_balance': -1, + 'min_coin_to_balance': -1, + 'max_concurrent': 4, + }, + { + 'coin_from': 'PART', + 'coin_to': 'XMR', + 'amount': 10, + 'maxrate': 0.04, + 'amount_variable': True, + 'address': -1, + 'min_swap_amount': 0.1, + 'max_coin_from_balance': -1, + 'min_coin_to_balance': -1, + } + ], + } + with open(self.node1_configfile, 'w') as fp: + json.dump(node1_test1_config, fp, indent=4) + + wait_for_offers(self.delay_event, 1, 2) + + logging.info('Test that a bid is created') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_bids(rv_stdout)) == 1) + + logging.info('Test no bids are created while delaying') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (count_lines_with(rv_stdout, 'Delaying bids until') == 1) + + with open(self.node1_statefile) as fs: + node1_state = json.load(fs) + node1_state['delay_next_bid_before'] = 0 + with open(self.node1_statefile, 'w') as fp: + json.dump(node1_state, fp, indent=4) + + logging.info('Test that a bid is not created if one already exists on that offer') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (count_lines_with(rv_stdout, 'Bid rate too low for offer') == 3) + assert (count_lines_with(rv_stdout, 'Already bidding on offer') == 1) + + logging.info(f'Modifying node1 config') + node1_test1_config['bids'][0]['maxrate'] = 0.07 + node1_test1_config['bids'][0]['max_coin_from_balance'] = 100 + node1_test1_config['bids'][0]['min_coin_to_balance'] = 100 + node1_test1_config['bids'][0]['min_swap_amount'] = 9 + node1_test1_config['wallet_port_override'] = 12699 + node1_test1_config['test_mode'] = True + with open(self.node1_configfile, 'w') as fp: + json.dump(node1_test1_config, fp, indent=4) + + self.thread_http.return_data = { + 'PART': { + 'balance': '0.0', + 'unconfirmed': '0.0', + 'expected_seed': True, + 'encrypted': False, + 'locked': False, + 'anon_balance': 0.0, + 'anon_pending': 0.0, + 'blind_balance': 0.0, + 'blind_unconfirmed': 0.0, + 'version': 23000300, + 'name': 'Particl', + 'blocks': 3556, + 'synced': '100.00' + }, + 'XMR': { + 'balance': '362299.12', + 'unconfirmed': '0.0', + 'expected_seed': True, + 'encrypted': False, + 'locked': False, + 'main_address': '', + 'version': 65562, + 'name': 'Monero', + 'blocks': 10470, + 'synced': '100.00', + 'known_block_count': 10470 + } + } + + # Check max_coin_from_balance (bids increase coin_from) + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + possible_bids = get_possible_bids(rv_stdout) + assert (len(possible_bids) == 1) + assert (float(possible_bids[0]['amount_from']) == 21.0) + + # Test multiple bids are delayed + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (count_lines_with(rv_stdout, 'Delaying bids until') == 1) + + delete_file(self.node1_statefile) + self.thread_http.return_data['PART']['balance'] = 100.0 + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (count_lines_with(rv_stdout, 'Bid amount would exceed maximum wallet total') == 1) + + self.thread_http.return_data['PART']['balance'] = 90.0 + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + possible_bids = get_possible_bids(rv_stdout) + assert (len(possible_bids) == 1) + assert (math.isclose(float(possible_bids[0]['amount_from']), 10.0)) + + # Check min_swap_amount + delete_file(self.node1_statefile) + self.thread_http.return_data['PART']['balance'] = 95.0 + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + possible_bids = get_possible_bids(rv_stdout) + assert (count_lines_with(rv_stdout, 'Bid amount would exceed maximum wallet total') == 1) + + # Check min_coin_to_balance (bids decrease coin_to) + self.thread_http.return_data['PART']['balance'] = 0.0 + self.thread_http.return_data['XMR']['balance'] = 101.0 + + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + possible_bids = get_possible_bids(rv_stdout) + possible_bids = get_possible_bids(rv_stdout) + assert (len(possible_bids) == 1) + assert (float(possible_bids[0]['amount_from'] < 20.0)) + + logging.info(f'Adding mock data to node1 db for tests') + rows = [] + offers = read_json_api(UI_PORT, 'offers') + + now = int(time.time()) + for offer in offers: + rows.append((1, offer['addr_from'], 5, 5, now, now)) + db_path = '/tmp/test_persistent/client1/db_regtest.sqlite' + with sqlite3.connect(db_path) as dbc: + c = dbc.cursor() + c.executemany('INSERT INTO knownidentities (active_ind, address, num_sent_bids_failed, num_recv_bids_failed, updated_at, created_at) VALUES (?,?,?,?,?,?)', rows) + dbc.commit() + + delete_file(self.node1_statefile) + self.thread_http.return_data['XMR']['balance'] = 10000.0 + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_possible_bids(get_possible_bids(rv_stdout))) == 0) + assert (count_lines_with(rv_stdout, 'too many failed bids') == 1) + + ''' + TODO + node0_test1_config['stealthex'] = [ + { + 'coin_from': 'XMR', + 'coin_to': 'BTC', + 'min_balance_from': 1, + 'min_amount_tx': 1, + 'max_amount_tx': 5, + 'min_rate': 0.01, + 'refund_address': 'auto', + 'receive_address': 'auto', + 'api_key': 'API_KEY_HERE' + } + ] + node0_test1_config['wallet_port_override'] = 12699 + node0_test1_config['test_mode'] = True + with open(self.node0_configfile, 'w') as fp: + json.dump(node0_test1_config, fp, indent=4) + + result = subprocess.run(self.node0_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + ''' + + def test_bid_tracking(self): + + waitForServer(self.delay_event, UI_PORT + 0) + waitForServer(self.delay_event, UI_PORT + 1) + + # Reset test + clear_offers(self.delay_event, 0) + delete_file(self.node0_statefile) + delete_file(self.node1_statefile) + wait_for_offers(self.delay_event, 1, 0) + + addrs = [] + for i in range(2): + addrs.append(read_json_api(UI_PORT, 'smsgaddresses/new')['new_address']) + + node0_test2_config = { + 'offers': [ + { + 'name': 'offer example 1', + 'coin_from': 'Particl', + 'coin_to': 'Monero', + 'amount': 20, + 'minrate': 0.04, + 'ratetweakpercent': 5, + 'amount_variable': True, + 'address': addrs[0], + 'min_coin_from_amt': 20, + 'max_coin_to_amt': -1 + }, + { + 'name': 'offer example 1_2', + 'coin_from': 'Particl', + 'coin_to': 'Monero', + 'amount': 21, + 'minrate': 0.05, + 'ratetweakpercent': 5, + 'amount_variable': True, + 'address': addrs[1], + 'min_coin_from_amt': 21, + 'max_coin_to_amt': -1 + }, + { + 'name': 'offer example 1_3', + 'coin_from': 'Particl', + 'coin_to': 'Monero', + 'amount': 22, + 'minrate': 0.06, + 'ratetweakpercent': 5, + 'amount_variable': True, + 'address': 'auto', + 'min_coin_from_amt': 22, + 'max_coin_to_amt': -1 + } + ], + } + with open(self.node0_configfile, 'w') as fp: + json.dump(node0_test2_config, fp, indent=4) + + offer_ids = [] + logging.info('Create three offers') + + + for i in range(3): + if i > 0: + with open(self.node0_statefile) as fs: + node0_state = json.load(fs) + node0_state['delay_next_offer_before'] = 0 + with open(self.node0_statefile, 'w') as fp: + json.dump(node0_state, fp, indent=4) + + result = subprocess.run(self.node0_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + created_offers = get_created_offers(rv_stdout) + assert (len(get_created_offers(rv_stdout)) == 1) + offer_ids.append(created_offers[0]) + + found_addrs = {} + for offer_id in offer_ids: + offer = read_json_api(UI_PORT, f'offers/{offer_id}')[0] + found_addrs[offer['addr_from']] = found_addrs.get(offer['addr_from'], 0) + 1 + + for addr in addrs: + assert (found_addrs[addr] == 1) + + addr_bid_from = read_json_api(UI_PORT + 1, 'smsgaddresses/new')['new_address'] + node1_test1_config = { + 'bids': [ + { + 'name': 'bid example 1', + 'coin_from': 'PART', + 'coin_to': 'XMR', + 'amount': 50, + 'maxrate': 0.08, + 'amount_variable': False, + 'address': addr_bid_from, + 'min_swap_amount': 1, + 'max_coin_from_balance': -1, + 'min_coin_to_balance': -1 + } + ], + } + with open(self.node1_configfile, 'w') as fp: + json.dump(node1_test1_config, fp, indent=4) + + wait_for_offers(self.delay_event, 1, 3) + + logging.info('Check that no bids are created (offer values too low)') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_bids(rv_stdout)) == 0) + assert (count_lines_with(rv_stdout, 'Offer amount too low for bid') == 3) + + node1_test1_config['bids'][0]['amount_variable'] = True + with open(self.node1_configfile, 'w') as fp: + json.dump(node1_test1_config, fp, indent=4) + + logging.info('Check that one bid is created at the best rate') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + created_bids = get_created_bids(rv_stdout) + assert (len(created_bids) == 1) + + bid_id = created_bids[0].split(' ')[0] + bid = read_json_api(UI_PORT + 1, f'bids/{bid_id}') + assert (math.isclose(float(bid['bid_rate']), 0.04)) + assert (math.isclose(float(bid['amt_from']), 20.0)) + assert (bid['addr_from'] == addr_bid_from) + + logging.info('Check that bids are delayed') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (count_lines_with(rv_stdout, 'Delaying bids until') == 1) + assert (len(get_created_bids(rv_stdout)) == 0) + + with open(self.node1_statefile) as fs: + node1_state = json.load(fs) + node1_state['delay_next_bid_before'] = 0 + with open(self.node1_statefile, 'w') as fp: + json.dump(node1_state, fp, indent=4) + + logging.info('Test that a bid is not created while one is active') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + assert (len(get_created_bids(rv_stdout)) == 0) + assert (count_lines_with(rv_stdout, 'Max concurrent bids') == 1) + + logging.info('Waiting for bid to complete') + bid_complete: bool = False + for i in range(60): + self.delay_event.wait(5) + bid = read_json_api(UI_PORT + 1, f'bids/{bid_id}') + print('bid_state', bid['bid_state']) + if bid['bid_state'] == 'Completed': + bid_complete = True + break + + assert bid_complete + + logging.info('Test that a bid is created after one expires') + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + created_bids = get_created_bids(rv_stdout) + assert (len(created_bids) == 1) + assert (count_lines_with(rv_stdout, 'Marking bid inactive') == 1) + + logging.info('Test that two bids are created if max concurrent is raised') + node1_test1_config['bids'][0]['max_concurrent'] = 2 + with open(self.node1_configfile, 'w') as fp: + json.dump(node1_test1_config, fp, indent=4) + + with open(self.node1_statefile) as fs: + node1_state = json.load(fs) + node1_state['delay_next_bid_before'] = 0 + with open(self.node1_statefile, 'w') as fp: + json.dump(node1_state, fp, indent=4) + + result = subprocess.run(self.node1_args, stdout=subprocess.PIPE) + rv_stdout = result.stdout.decode().split('\n') + created_bids = get_created_bids(rv_stdout) + assert (len(created_bids) == 1) + + bid_id = created_bids[0].split(' ')[0] + bid = read_json_api(UI_PORT + 1, f'bids/{bid_id}') + assert (math.isclose(float(bid['bid_rate']), 0.05)) + assert (math.isclose(float(bid['amt_from']), 21.0)) + assert (bid['addr_from'] == addr_bid_from) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/basicswap/test_btc_xmr.py b/tests/basicswap/test_btc_xmr.py index 55ee1ad..369d3da 100644 --- a/tests/basicswap/test_btc_xmr.py +++ b/tests/basicswap/test_btc_xmr.py @@ -254,6 +254,12 @@ class BasicSwapTest(BaseTest): self.callnoderpc('unloadwallet', [new_wallet_name]) assert (addr == 'bcrt1qps7hnjd866e9ynxadgseprkc2l56m00dvwargr') + self.swap_clients[0].initialiseWallet(Coins.BTC, raise_errors=True) + assert self.swap_clients[0].checkWalletSeed(Coins.BTC) is True + for i in range(1500): + self.callnoderpc('getnewaddress') + assert self.swap_clients[0].checkWalletSeed(Coins.BTC) is True + def do_test_01_full_swap(self, coin_from, coin_to): logging.info('---------- Test {} to {}'.format(coin_from.name, coin_to.name)) diff --git a/tests/basicswap/test_run.py b/tests/basicswap/test_run.py index d04e0b0..f2bf8e4 100644 --- a/tests/basicswap/test_run.py +++ b/tests/basicswap/test_run.py @@ -133,6 +133,18 @@ class Test(BaseTest): rv = read_json_api(1800, 'getcoinseed', {'coin': 'BTC'}) assert (rv['seed'] == '8e54a313e6df8918df6d758fafdbf127a115175fdd2238d0e908dd8093c9ac3b') + rv = read_json_api(1800, 'identities/ppCsRro5po7Yu6kyu5XjSyr3A1PPdk9j1F', {'set_label': 'test 1'}) + assert (len(rv) == 1) + assert (rv[0]['address'] == 'ppCsRro5po7Yu6kyu5XjSyr3A1PPdk9j1F') + assert (rv[0]['label'] == 'test 1') + rv = read_json_api(1800, 'identities/ppCsRro5po7Yu6kyu5XjSyr3A1PPdk9j1F', {'set_label': 'test 2'}) + assert (len(rv) == 1) + assert (rv[0]['address'] == 'ppCsRro5po7Yu6kyu5XjSyr3A1PPdk9j1F') + assert (rv[0]['label'] == 'test 2') + + rv = read_json_api(1800, 'identities/pPCsRro5po7Yu6kyu5XjSyr3A1PPdk9j1F', {'set_label': 'test 3'}) + assert (rv['error'] == 'Invalid identity address') + def test_01_verifyrawtransaction(self): txn = '0200000001eb6e5c4ebba4efa32f40c7314cad456a64008e91ee30b2dd0235ab9bb67fbdbb01000000ee47304402200956933242dde94f6cf8f195a470f8d02aef21ec5c9b66c5d3871594bdb74c9d02201d7e1b440de8f4da672d689f9e37e98815fb63dbc1706353290887eb6e8f7235012103dc1b24feb32841bc2f4375da91fa97834e5983668c2a39a6b7eadb60e7033f9d205a803b28fe2f86c17db91fa99d7ed2598f79b5677ffe869de2e478c0d1c02cc7514c606382012088a8201fe90717abb84b481c2a59112414ae56ec8acc72273642ca26cc7a5812fdc8f68876a914225fbfa4cb725b75e511810ac4d6f74069bdded26703520140b27576a914207eb66b2fd6ed9924d6217efc7fa7b38dfabe666888acffffffff01e0167118020000001976a9140044e188928710cecba8311f1cf412135b98145c88ac00000000' prevout = {