From 73ca40ea8d7eeea8f38b46067ed31fa9bb3e4294 Mon Sep 17 00:00:00 2001 From: philemon Date: Fri, 11 Nov 2016 16:05:27 +0300 Subject: [PATCH] Assorted fixes/improvements: - Importing addresses with --rescan working again - Tracking and spending non-MMGen addresses now fully functional - mmgen-txcreate: improvements in unspent outputs display - mmgen-txsign: use bitcoind wallet dump as keylist fixed - Testnet support: - Practice sending transactions without risking funds (free testnet coins: https://tpfaucet.appspot.com/) - Test suite fully supported - To enable, set MMGEN_TESTNET environment variable --- ...Install-MMGen-on-Debian-or-Ubuntu-Linux.md | 4 +- mmgen/addr.py | 6 +- mmgen/bitcoin.py | 27 +-- mmgen/globalvars.py | 5 +- mmgen/main_addrimport.py | 26 +-- mmgen/main_txcreate.py | 5 +- mmgen/main_txsign.py | 3 +- mmgen/obj.py | 18 +- mmgen/rpc.py | 15 +- mmgen/term.py | 3 - mmgen/test.py | 1 + mmgen/tool.py | 19 +- mmgen/tw.py | 61 ++++--- mmgen/util.py | 3 +- test/gentest.py | 164 ++++++++++++------ test/ref/98831F3A-E2687906[256,1].mmdat | 6 + ...A[1,31-33,500-501,1010-1011].testnet.addrs | 19 ++ ...-33,500-501,1010-1011].testnet.akeys.mmenc | Bin 0 -> 1447 bytes test/ref/FFB367[1.234].testnet.rawtx | 6 + test/test.py | 55 +++--- test/tooltest.py | 2 +- 21 files changed, 282 insertions(+), 166 deletions(-) create mode 100644 test/ref/98831F3A-E2687906[256,1].mmdat create mode 100644 test/ref/98831F3A[1,31-33,500-501,1010-1011].testnet.addrs create mode 100644 test/ref/98831F3A[1,31-33,500-501,1010-1011].testnet.akeys.mmenc create mode 100644 test/ref/FFB367[1.234].testnet.rawtx diff --git a/doc/wiki/install-linux/Install-MMGen-on-Debian-or-Ubuntu-Linux.md b/doc/wiki/install-linux/Install-MMGen-on-Debian-or-Ubuntu-Linux.md index 8e63e510..b2ecbe5a 100644 --- a/doc/wiki/install-linux/Install-MMGen-on-Debian-or-Ubuntu-Linux.md +++ b/doc/wiki/install-linux/Install-MMGen-on-Debian-or-Ubuntu-Linux.md @@ -2,13 +2,13 @@ Install required Debian/Ubuntu packages: - $ sudo apt-get install python-pip python-dev python-pexpect python-ecdsa python-scrypt libssl-dev git + $ sudo apt-get install python-pip python-dev python-pexpect python-ecdsa python-scrypt libssl-dev git autoconf libtool Install the Python Cryptography Toolkit: $ sudo pip install pycrypto -Install the secp256k1 library +Install the secp256k1 library: $ git clone https://github.com/bitcoin-core/secp256k1.git $ cd secp256k1 diff --git a/mmgen/addr.py b/mmgen/addr.py index bac86412..5f1aa11b 100755 --- a/mmgen/addr.py +++ b/mmgen/addr.py @@ -58,17 +58,17 @@ internal ECDSA library for address generation. def _wif2addr_python(wif): privhex = wif2hex(wif) if not privhex: return False - return privnum2addr(int(privhex,16),wif[0] != '5') + return privnum2addr(int(privhex,16),wif[0] != ('5','9')[g.testnet]) def _wif2addr_keyconv(wif): - if wif[0] == '5': + if wif[0] == ('5','9')[g.testnet]: from subprocess import check_output return check_output(['keyconv', wif]).split()[1] else: return _wif2addr_python(wif) def _wif2addr_secp256k1(wif): - return _privhex2addr_secp256k1(wif2hex(wif),wif[0] != '5') + return _privhex2addr_secp256k1(wif2hex(wif),wif[0] != ('5','9')[g.testnet]) def _privhex2addr_python(privhex,compressed=False): return privnum2addr(int(privhex,16),compressed) diff --git a/mmgen/bitcoin.py b/mmgen/bitcoin.py index abb4f1f0..00db1351 100755 --- a/mmgen/bitcoin.py +++ b/mmgen/bitcoin.py @@ -51,13 +51,15 @@ b58a='123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' # The 'zero address': # 1111111111111111111114oLvT2 (use step2 = ('0' * 40) to generate) +import mmgen.globalvars as g + def pubhex2hexaddr(pubhex): step1 = sha256(unhexlify(pubhex)).digest() return hashlib_new('ripemd160',step1).hexdigest() -def hexaddr2addr(hexaddr, vers_num='00'): - # See above: - hexaddr2 = vers_num + hexaddr +def hexaddr2addr(hexaddr,p2sh=False): + # devdoc/ref_transactions.md: + hexaddr2 = ('00','6f','05','c4')[g.testnet+(2*p2sh)] + hexaddr step1 = sha256(unhexlify(hexaddr2)).digest() step2 = sha256(step1).hexdigest() pubkey = hexaddr2 + step2[:8] @@ -65,9 +67,8 @@ def hexaddr2addr(hexaddr, vers_num='00'): return ('1' * lzeroes) + _numtob58(int(pubkey,16)) def verify_addr(addr,verbose=False,return_hex=False): - - for vers_num,ldigit in ('00','1'),('05','3'): - if addr[0] != ldigit: continue + for vers_num,ldigit in ('00','1'),('05','3'),('6f','mn'),('c4','2'): + if addr[0] not in ldigit: continue num = _b58tonum(addr) if num == False: break addr_hex = '{:050x}'.format(num) @@ -145,7 +146,7 @@ def b58decode_pad(s): # Compressed address support: def wif2hex(wif): - compressed = wif[0] != '5' + compressed = wif[0] != ('5','9')[g.testnet] idx = (66,68)[bool(compressed)] num = _b58tonum(wif) if num == False: return False @@ -153,19 +154,25 @@ def wif2hex(wif): if compressed and key[66:68] != '01': return False round1 = sha256(unhexlify(key[:idx])).digest() round2 = sha256(round1).hexdigest() - return key[2:66] if (key[:2] == '80' and key[idx:] == round2[:8]) else False + return key[2:66] if (key[:2] == ('80','ef')[g.testnet] and key[idx:] == round2[:8]) else False def hex2wif(hexpriv,compressed=False): - step1 = '80' + hexpriv + ('','01')[bool(compressed)] + step1 = ('80','ef')[g.testnet] + hexpriv + ('','01')[bool(compressed)] step2 = sha256(unhexlify(step1)).digest() step3 = sha256(step2).hexdigest() key = step1 + step3[:8] return _numtob58(int(key,16)) +# devdoc/guide_wallets.md: +# Uncompressed public keys start with 0x04; compressed public keys begin with +# 0x03 or 0x02 depending on whether they're greater or less than the midpoint +# of the curve. def privnum2pubhex(numpriv,compressed=False): pko = ecdsa.SigningKey.from_secret_exponent(numpriv,_secp256k1) + # pubkey = 32-byte X coord + 32-byte Y coord (unsigned big-endian) pubkey = hexlify(pko.get_verifying_key().to_string()) - if compressed: + if compressed: # discard Y coord, replace with appropriate version byte + # even Y: <0, odd Y: >0 -- https://bitcointalk.org/index.php?topic=129652.0 p = ('03','02')[pubkey[-1] in '02468ace'] return p+pubkey[:64] else: diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index f729bb57..a1a17868 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -45,6 +45,7 @@ no_license = os.getenv('MMGEN_NOLICENSE') bogus_wallet_data = os.getenv('MMGEN_BOGUS_WALLET_DATA') disable_hold_protect = os.getenv('MMGEN_DISABLE_HOLD_PROTECT') color = (False,True)[sys.stdout.isatty() and not os.getenv('MMGEN_DISABLE_COLOR')] +testnet = (False,True)[bool(os.getenv('MMGEN_TESTNET'))] proj_name = 'MMGen' prog_name = os.path.basename(sys.argv[0]) @@ -63,12 +64,14 @@ incompatible_opts = ( ('label','keep_label'), ('tx_id', 'info'), ('tx_id', 'terse_info'), + ('batch', 'rescan'), ) min_screen_width = 80 +minconf = 1 # Global value sets user opt -dfl_vars = 'seed_len','hash_preset','usr_randchars','debug','tx_confs','tx_fee_adj','tx_fee','key_generator' +dfl_vars = 'minconf','seed_len','hash_preset','usr_randchars','debug','tx_confs','tx_fee_adj','tx_fee','key_generator' keyconv_exec = 'keyconv' diff --git a/mmgen/main_addrimport.py b/mmgen/main_addrimport.py index 21d0a5be..d12ad86f 100755 --- a/mmgen/main_addrimport.py +++ b/mmgen/main_addrimport.py @@ -25,14 +25,17 @@ import time from mmgen.common import * from mmgen.addr import AddrList,KeyAddrList +# In batch mode, bitcoind just rescans each address separately anyway, so make +# --batch and --rescan incompatible. + opts_data = { - 'desc': """Import addresses (both {pnm} and non-{pnm}) into a bitcoind + 'desc': """Import addresses (both {pnm} and non-{pnm}) into an {pnm} tracking wallet""".format(pnm=g.proj_name), 'usage':'[opts] [mmgen address file]', 'options': """ -h, --help Print this help message --b, --batch Batch mode. Import all addresses in one RPC call --l, --addrlist Address source is a flat list of addresses +-b, --batch Import all addresses in one RPC call. +-l, --addrlist Address source is a flat list of (non-MMGen) Bitcoin addresses -k, --keyaddr-file Address source is a key-address file -q, --quiet Suppress warnings -r, --rescan Rescan the blockchain. Required if address to import is @@ -42,6 +45,8 @@ opts_data = { 'notes': """\n This command can also be used to update the comment fields of addresses already in the tracking wallet. + +The --batch option cannot be used with the --rescan option. """ } @@ -116,12 +121,15 @@ for n,e in enumerate(ai.data): if e.idx: label = '%s:%s' % (ai.seed_id,e.idx) if e.label: label += ' ' + e.label - else: label = 'non-{pnm}'.format(pnm=g.proj_name) + m = label + else: + label = 'btc:{}'.format(e.addr) + m = 'non-'+g.proj_name if opt.batch: arg_list.append((e.addr,label,False)) elif opt.rescan: - t = threading.Thread(target=import_address, args=(e.addr,label,True)) + t = threading.Thread(target=import_address,args=[e.addr,label,True]) t.daemon = True t.start() @@ -131,7 +139,7 @@ for n,e in enumerate(ai.data): if t.is_alive(): elapsed = int(time.time() - start) count = '%s/%s:' % (n+1, ai.num_addrs) - msg_r(msg_fmt % (secs_to_hms(elapsed),count,e.addr,'(%s)'%label)) + msg_r(msg_fmt % (secs_to_hms(elapsed),count,e.addr,'(%s)' % m)) time.sleep(1) else: if err_flag: die(2,'\nImport failed') @@ -140,12 +148,10 @@ for n,e in enumerate(ai.data): else: import_address(e.addr,label,False) count = '%s/%s:' % (n+1, ai.num_addrs) - msg_r(msg_fmt % (count, e.addr, '(%s)'%label)) + msg_r(msg_fmt % (count, e.addr, '(%s)' % m)) if err_flag: die(2,'\nImport failed') msg(' - OK') if opt.batch: - if opt.rescan: - msg('Warning: this command may take a long time to complete!') - ret = c.importaddress(arg_list,batch=True,timeout=(False,3600)[bool(opt.rescan)]) + ret = c.importaddress(arg_list,batch=True) msg('OK: %s addresses imported' % len(ret)) diff --git a/mmgen/main_txcreate.py b/mmgen/main_txcreate.py index 376a3865..f4f09899 100755 --- a/mmgen/main_txcreate.py +++ b/mmgen/main_txcreate.py @@ -36,8 +36,9 @@ opts_data = { -c, --comment-file= f Source the transaction's comment from file 'f' -C, --tx-confs= c Desired number of confirmations (default: {g.tx_confs}) -d, --outdir= d Specify an alternate directory 'd' for output --e, --echo-passphrase Print passphrase to screen when typing it +-e, --clear-screen Clear screen before displaying unspent outputs -f, --tx-fee= f Transaction fee (default: {g.tx_fee} BTC (but see below)) +-m, --minconf= n Minimum number of confirmations required to spend outputs (default: 1) -i, --info Display unspent outputs and exit -q, --quiet Suppress warnings; overwrite files without prompting -v, --verbose Produce more verbose output @@ -199,7 +200,7 @@ if not opt.info: fee_estimate = get_fee_estimate() -tw = MMGenTrackingWallet() +tw = MMGenTrackingWallet(minconf=opt.minconf) tw.view_and_sort() tw.display_total() diff --git a/mmgen/main_txsign.py b/mmgen/main_txsign.py index d933ced4..8911a80e 100755 --- a/mmgen/main_txsign.py +++ b/mmgen/main_txsign.py @@ -227,10 +227,9 @@ if opt.mmgen_keys_from_file: if opt.keys_from_file: l = get_lines_from_file(opt.keys_from_file,'key-address data',trim_comments=True) - kl = KeyAddrList(keylist=l) + kl = KeyAddrList(keylist=[m.split()[0] for m in l]) # accept bitcoind wallet dumps if kal: kl.remove_dups(kal,key='wif') kl.generate_addrs() -# pp_die(kl) tx_num_str = '' for tx_num,tx_file in enumerate(tx_files,1): diff --git a/mmgen/obj.py b/mmgen/obj.py index 1e2e43b0..9ec5565f 100755 --- a/mmgen/obj.py +++ b/mmgen/obj.py @@ -195,6 +195,7 @@ class AddrIdx(int,InitErrors): m = "'%s': addr idx cannot be less than one" % num else: return me + return cls.init_fail(m,on_fail) class AddrIdxList(list,InitErrors): @@ -349,7 +350,7 @@ class BTCAddr(str,Hilite,InitErrors): cls.arg_chk(cls,on_fail) me = str.__new__(cls,s) from mmgen.bitcoin import verify_addr - if verify_addr(s): + if type(s) in (str,unicode,BTCAddr) and verify_addr(s): return me else: m = "'%s': value is not a Bitcoin address" % s @@ -375,13 +376,14 @@ class SeedID(str,Hilite,InitErrors): if seed: from mmgen.seed import Seed from mmgen.util import make_chksum_8 - assert type(seed) == Seed - return str.__new__(cls,make_chksum_8(seed.get_data())) + if type(seed) == Seed: + return str.__new__(cls,make_chksum_8(seed.get_data())) elif sid: from string import hexdigits - assert len(sid) == cls.width and set(sid) <= set(hexdigits.upper()) - return str.__new__(cls,sid) - m = "'%s': value cannot be converted to SeedID" % s + if len(sid) == cls.width and set(sid) <= set(hexdigits.upper()): + return str.__new__(cls,sid) + + m = "'%s': value cannot be converted to SeedID" % str(seed or sid) return cls.init_fail(m,on_fail) class MMGenID(str,Hilite,InitErrors): @@ -395,9 +397,9 @@ class MMGenID(str,Hilite,InitErrors): s = str(s) if ':' in s: a,b = s.split(':',1) - sid = SeedID(sid=a,on_fail='return') + sid = SeedID(sid=a,on_fail='silent') if sid: - idx = AddrIdx(b,on_fail='return') + idx = AddrIdx(b,on_fail='silent') if idx: return str.__new__(cls,'%s:%s' % (sid,idx)) diff --git a/mmgen/rpc.py b/mmgen/rpc.py index af100d7d..9bec6c4a 100755 --- a/mmgen/rpc.py +++ b/mmgen/rpc.py @@ -20,9 +20,11 @@ rpc.py: Bitcoin RPC library for the MMGen suite """ -import httplib,base64,json,decimal +import httplib,base64,json from mmgen.common import * +from decimal import Decimal +from mmgen.obj import BTCAmt class BitcoinRPCConnection(object): @@ -30,7 +32,7 @@ class BitcoinRPCConnection(object): def __init__( self, - host='localhost',port=8332, + host='localhost',port=(8332,18332)[g.testnet], user=None,passwd=None,auth_cookie=None, ): @@ -64,7 +66,7 @@ class BitcoinRPCConnection(object): for k in cf: if k in kwargs and kwargs[k]: cf[k] = kwargs[k] - c = httplib.HTTPConnection(self.host, self.port, False, cf['timeout']) + hc = httplib.HTTPConnection(self.host, self.port, False, cf['timeout']) if cf['batch']: p = [{'method':cmd,'params':r,'id':n} for n,r in enumerate(args[0],1)] @@ -80,7 +82,6 @@ class BitcoinRPCConnection(object): dmsg('=== rpc.py debug ===') dmsg(' RPC POST data ==> %s\n' % p) - from mmgen.obj import BTCAmt caller = self class MyJSONEncoder(json.JSONEncoder): def default(self, obj): @@ -94,14 +95,14 @@ class BitcoinRPCConnection(object): # print(dump) try: - c.request('POST', '/', json.dumps(p,cls=MyJSONEncoder), { + hc.request('POST', '/', json.dumps(p,cls=MyJSONEncoder), { 'Host': self.host, 'Authorization': 'Basic ' + base64.b64encode(self.auth_str) }) except Exception as e: return die_maybe(None,2,'%s\nUnable to connect to bitcoind' % e) - r = c.getresponse() # returns HTTPResponse instance + r = hc.getresponse() # returns HTTPResponse instance if r.status != 200: e1 = r.read() @@ -118,7 +119,7 @@ class BitcoinRPCConnection(object): if not r2: return die_maybe(r,2,'Error: empty reply') - from decimal import Decimal +# from decimal import Decimal r3 = json.loads(r2.decode('utf8'), parse_float=Decimal) ret = [] diff --git a/mmgen/term.py b/mmgen/term.py index 9ca6ab50..4bc75f3c 100755 --- a/mmgen/term.py +++ b/mmgen/term.py @@ -23,9 +23,6 @@ term.py: Terminal-handling routines for the MMGen suite import os,struct from mmgen.common import * -CUR_SHOW = '\033[?25h' -CUR_HIDE = '\033[?25l' - def _kb_hold_protect_unix(): fd = sys.stdin.fileno() diff --git a/mmgen/test.py b/mmgen/test.py index 3787c0b3..bd71bf84 100755 --- a/mmgen/test.py +++ b/mmgen/test.py @@ -80,6 +80,7 @@ def read_from_tmpfile(cfg,fn,binary=False): return read_from_file(os.path.join(cfg['tmpdir'],fn),binary=binary) def ok(): + if opt.profile: return if opt.verbose or opt.exact_output: sys.stderr.write(green('OK\n')) else: msg(' OK') diff --git a/mmgen/tool.py b/mmgen/tool.py index 154ff8dc..ad1e32cd 100755 --- a/mmgen/tool.py +++ b/mmgen/tool.py @@ -74,7 +74,7 @@ cmd_data = OrderedDict([ ('listaddresses',["addrs [str='']",'minconf [int=1]','showempty [bool=False]','pager [bool=False]','showbtcaddrs [bool=False]']), ('getbalance', ['minconf [int=1]']), ('txview', ['<{} TX file> [str]'.format(pnm),'pager [bool=False]','terse [bool=False]']), - ('twview', ["sort [str='age']",'reverse [bool=False]','wide [bool=False]','pager [bool=False]']), + ('twview', ["sort [str='age']",'reverse [bool=False]','minconf [int=1]','wide [bool=False]','pager [bool=False]']), ('add_label', ['<{} address> [str]'.format(pnm),'