mmgen-regtest: --empty option, improved debugging output

mmgen-tool: move help data to main_tool.py

mmgen/addr.py

@@ -61,7 +61,7 @@ class AddrGeneratorSegwit(MMGenObject):
 class KeyGenerator(MMGenObject):
 	def __new__(cls,generator=None,silent=False):
 		if cls.test_for_secp256k1(silent=silent) and generator != 1:
-			if (not hasattr(opt,'key_generator')) or opt.key_generator == 2 or generator == 2:
+			if not opt.key_generator or opt.key_generator == 2 or generator == 2:
 				return super(cls,cls).__new__(KeyGeneratorSecp256k1)
 		else:
 			msg('Using (slow) native Python ECDSA library for address generation')
@@ -92,7 +92,7 @@ class KeyGeneratorSecp256k1(KeyGenerator):
 
 class AddrListEntry(MMGenListItem):
 	addr  = MMGenListItemAttr('addr','BTCAddr')
-	idx   = MMGenImmutableAttr('idx','AddrIdx')
+	idx   = MMGenListItemAttr('idx','AddrIdx') # not present in flat addrlists
 	label = MMGenListItemAttr('label','TwComment',reassign_ok=True)
 	sec   = MMGenListItemAttr('sec',PrivKey,typeconv=False)
 
@@ -223,7 +223,7 @@ Removed %s duplicate WIF key%s from keylist (also in {pnm} key-address file
 
 	def update_msgs(self):
 		self.msgs = AddrList.msgs
- 		self.msgs.update(type(self).msgs)
+		self.msgs.update(type(self).msgs)
 
 	def generate(self,seed,addrnums):
 		assert type(addrnums) is AddrIdxList
@@ -684,7 +684,7 @@ re-import your addresses.
 
 	def add_tw_data(self):
 		vmsg('Getting address data from tracking wallet')
-		c = bitcoin_connection()
+		c = rpc_connection()
 		accts = c.listaccounts(0,True)
 		data,i = {},0
 		alists = c.getaddressesbyaccount([[k] for k in accts],batch=True)

mmgen/globalvars.py

@@ -39,7 +39,7 @@ class g(object):
 	# Variables - these might be altered at runtime:
 
 	version      = '0.9.299'
-	release_date = 'July 2017'
+	release_date = 'August 2017'
 
 	proj_name = 'MMGen'
 	proj_url  = 'https://github.com/mmgen/mmgen'
@@ -77,9 +77,9 @@ class g(object):
 	force_256_color      = False
 	testnet              = False
 	regtest              = False
-	chain                = None # set by first call to bitcoin_connection()
+	chain                = None # set by first call to rpc_connection()
 	chains               = 'mainnet','testnet','regtest'
-	bitcoind_version     = None # set by first call to bitcoin_connection()
+	bitcoind_version     = None # set by first call to rpc_connection()
 	rpc_host             = ''
 	rpc_port             = 0
 	rpc_user             = ''
@@ -118,7 +118,7 @@ class g(object):
 	required_opts = (
 		'quiet','verbose','debug','outdir','echo_passphrase','passwd_file','stdout',
 		'show_hash_presets','label','keep_passphrase','keep_hash_preset','yes',
-		'brain_params','b16','usr_randchars','coin','bob','alice'
+		'brain_params','b16','usr_randchars','coin','bob','alice','key_generator'
 	)
 	incompatible_opts = (
 		('bob','alice'),

mmgen/main_addrimport.py

@@ -63,20 +63,17 @@ def import_mmgen_list(infile):
 			rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses')
 	return al
 
-def import_flat_list(lines):
-	return AddrList(addrlist=lines)
-
 if len(cmd_args) == 1:
 	infile = cmd_args[0]
 	check_infile(infile)
 	if opt.addrlist:
 		lines = get_lines_from_file(
 			infile,'non-{pnm} addresses'.format(pnm=g.proj_name),trim_comments=True)
-		al = import_flat_list(lines)
+		al = AddrList(addrlist=lines)
 	else:
 		al = import_mmgen_list(infile)
 elif len(cmd_args) == 0 and opt.address:
-	al = import_flat_list([opt.address])
+	al = AddrList(addrlist=[opt.address])
 	infile = 'command line'
 else:
 	die(1,"""
@@ -88,7 +85,7 @@ m = ' from Seed ID {}'.format(al.al_id.sid) if hasattr(al.al_id,'sid') else ''
 qmsg('OK. {} addresses{}'.format(al.num_addrs,m))
 
 if not opt.test:
-	c = bitcoin_connection()
+	c = rpc_connection()
 
 m = """
 WARNING: You've chosen the '--rescan' option.  Rescanning the blockchain is
@@ -115,7 +112,7 @@ def import_address(addr,label,rescan):
 		err_flag = True
 
 w_n_of_m = len(str(al.num_addrs)) * 2 + 2
-w_mmid   = '' if opt.addrlist else len(str(max(al.idxs()))) + 12
+w_mmid   = '' if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 12
 
 if opt.rescan:
 	import threading

mmgen/main_regtest.py

@@ -29,6 +29,7 @@ opts_data = lambda: {
 	'options': """
 -h, --help          Print this help message
 -m, --mixed         Create Bob and Alice's wallets with mixed address types
+-e, --empty         Don't fund Bob and Alice's wallets on setup
 --, --longhelp      Print help message for long options (common options)
 -q, --quiet         Produce quieter output
 -v, --verbose       Produce more verbose output

mmgen/main_tool.py

@@ -22,7 +22,89 @@ mmgen-tool:  Perform various MMGen- and Bitcoin-related operations.
 """
 
 from mmgen.common import *
-import mmgen.tool as tool
+
+stdin_msg = """
+To force a command to read from STDIN in place of its first argument (for
+supported commands), use '-' as the first argument.
+""".strip()
+
+cmd_help = """
+Bitcoin address/key operations (compressed public keys supported):
+  addr2hexaddr   - convert Bitcoin address from base58 to hex format
+  hex2wif        - convert a private key from hex to WIF format
+  hexaddr2addr   - convert Bitcoin address from hex to base58 format
+  privhex2addr   - generate Bitcoin address from private key in hex format
+  privhex2pubhex - generate a hex public key from a hex private key
+  pubhex2addr    - convert a hex pubkey to an address
+  pubhex2redeem_script - convert a hex pubkey to a witness redeem script
+  wif2redeem_script - convert a WIF private key to a witness redeem script
+  wif2segwit_pair - generate both a Segwit redeem script and address from WIF
+  pubkey2addr    - convert Bitcoin public key to address
+  randpair       - generate a random private key/address pair
+  randwif        - generate a random private key in WIF format
+  wif2addr       - generate a Bitcoin address from a key in WIF format
+  wif2hex        - convert a private key from WIF to hex format
+
+Wallet/TX operations (bitcoind must be running):
+  getbalance    - like 'bitcoin-cli getbalance' but shows confirmed/unconfirmed,
+                  spendable/unspendable balances for individual {pnm} wallets
+  listaddress   - list the specified {pnm} address and its balance
+  listaddresses - list {pnm} addresses and their balances
+  txview        - show raw/signed {pnm} transaction in human-readable form
+  twview        - view tracking wallet
+
+General utilities:
+  hexdump      - encode data into formatted hexadecimal form (file or stdin)
+  unhexdump    - decode formatted hexadecimal data (file or stdin)
+  bytespec     - convert a byte specifier such as '1GB' into an integer
+  hexlify      - display string in hexadecimal format
+  hexreverse   - reverse bytes of a hexadecimal string
+  rand2file    - write 'n' bytes of random data to specified file
+  randhex      - print 'n' bytes (default 32) of random data in hex format
+  hash256      - compute sha256(sha256(data)) (double sha256)
+  hash160      - compute ripemd160(sha256(data)) (converts hexpubkey to hexaddr)
+  b58randenc   - generate a random 32-byte number and convert it to base 58
+  b58tostr     - convert a base 58 number to a string
+  strtob58     - convert a string to base 58
+  b58tohex     - convert a base 58 number to hexadecimal
+  hextob58     - convert a hexadecimal number to base 58
+  b32tohex     - convert a base 32 number to hexadecimal
+  hextob32     - convert a hexadecimal number to base 32
+
+File encryption:
+  encrypt      - encrypt a file
+  decrypt      - decrypt a file
+    {pnm} encryption suite:
+      * Key: Scrypt (user-configurable hash parameters, 32-byte salt)
+      * Enc: AES256_CTR, 16-byte rand IV, sha256 hash + 32-byte nonce + data
+      * The encrypted file is indistinguishable from random data
+
+{pnm}-specific operations:
+  add_label    - add descriptive label for {pnm} address in tracking wallet
+  remove_label - remove descriptive label for {pnm} address in tracking wallet
+  addrfile_chksum    - compute checksum for {pnm} address file
+  keyaddrfile_chksum - compute checksum for {pnm} key-address file
+  passwdfile_chksum  - compute checksum for {pnm} password file
+  find_incog_data    - Use an Incog ID to find hidden incognito wallet data
+  id6          - generate 6-character {pnm} ID for a file (or stdin)
+  id8          - generate 8-character {pnm} ID for a file (or stdin)
+  str2id6      - generate 6-character {pnm} ID for a string, ignoring spaces
+
+Mnemonic operations (choose 'electrum' (default), 'tirosh' or 'all'
+  wordlists):
+  mn_rand128   - generate random 128-bit mnemonic
+  mn_rand192   - generate random 192-bit mnemonic
+  mn_rand256   - generate random 256-bit mnemonic
+  mn_stats     - show stats for mnemonic wordlist
+  mn_printlist - print mnemonic wordlist
+  hex2mn       - convert a 16, 24 or 32-byte number in hex format to a mnemonic
+  mn2hex       - convert a 12, 18 or 24-word mnemonic to a number in hex format
+
+  IMPORTANT NOTE: Though {pnm} mnemonics use the Electrum wordlist, they're
+  computed using a different algorithm and are NOT Electrum-compatible!
+
+  {sm}
+""".format(pnm=g.proj_name,sm='\n  '.join(stdin_msg.split('\n')))
 
 opts_data = lambda: {
 	'desc':    'Perform various {pnm}- and Bitcoin-related operations'.format(pnm=g.proj_name),
@@ -42,7 +124,7 @@ opts_data = lambda: {
                                COMMANDS
 {}
 Type '{} help <command> for help on a particular command
-""".format(tool.cmd_help,g.prog_name)
+""".format(cmd_help,g.prog_name)
 }
 
 cmd_args = opts.init(opts_data,add_opts=['hidden_incog_input_params','in_fmt'])
@@ -51,6 +133,8 @@ if len(cmd_args) < 1: opts.usage()
 
 Command = cmd_args.pop(0).capitalize()
 
+import mmgen.tool as tool
+
 if Command == 'Help' and not cmd_args: tool.usage(None)
 
 if Command not in tool.cmd_data:

mmgen/main_txbump.py

@@ -75,7 +75,7 @@ opts_data = lambda: {
 
 cmd_args = opts.init(opts_data)
 
-c = bitcoin_connection()
+c = rpc_connection()
 
 tx_file = cmd_args.pop(0)
 check_infile(tx_file)

mmgen/main_txdo.py

@@ -84,7 +84,7 @@ if opt.aug1hf: # TODO: remove in 0.9.4
 	g.coin = 'BCH'
 
 seed_files = get_seed_files(opt,cmd_args)
-c = bitcoin_connection()
+c = rpc_connection()
 do_license_msg()
 
 kal = get_keyaddrlist(opt)

mmgen/main_txsend.py

@@ -46,7 +46,7 @@ else: opts.usage()
 
 if not opt.status: do_license_msg()
 
-c = bitcoin_connection()
+c = rpc_connection()
 tx = MMGenTX(infile) # sig check performed here
 vmsg("Signed transaction file '%s' is valid" % infile)
 

mmgen/main_txsign.py

@@ -78,7 +78,7 @@ if opt.aug1hf: # TODO: remove in 0.9.4
 if not infiles: opts.usage()
 for i in infiles: check_infile(i)
 
-c = bitcoin_connection()
+c = rpc_connection()
 
 if not opt.info and not opt.terse_info:
 	do_license_msg(immed=True)

mmgen/obj.py

@@ -388,8 +388,8 @@ class BTCAddr(str,Hilite,InitErrors,MMGenObject):
 		return self[0] in btc_addr_pfxs['mainnet']
 
 	def is_in_tracking_wallet(self):
-		from mmgen.rpc import bitcoin_connection
-		d = bitcoin_connection().validateaddress(self)
+		from mmgen.rpc import rpc_connection
+		d = rpc_connection().validateaddress(self)
 		return d['iswatchonly'] and 'account' in d
 
 class SeedID(str,Hilite,InitErrors):

mmgen/opts.py

@@ -111,7 +111,7 @@ def opt_postproc_initializations():
 
 	g.coin = g.coin.upper() # allow user to use lowercase
 
-def	set_data_dir_root():
+def set_data_dir_root():
 	g.data_dir_root = os.path.normpath(os.path.expanduser(opt.data_dir)) if opt.data_dir else \
 			os.path.join(g.home_dir,'.'+g.proj_name.lower())
 

mmgen/regtest.py

@@ -39,8 +39,8 @@ mmwords = {
 	'alice': os.path.join(data_dir,'9304C211[128].mmwords')
 }
 mmaddrs = {
-	'bob':   os.path.join('/tmp','1163DDF1{}[1-10].addrs'),
-	'alice': os.path.join('/tmp','9304C211{}[1-10].addrs')
+	'bob':   os.path.join(data_dir,'1163DDF1{}[1-10].addrs'),
+	'alice': os.path.join(data_dir,'9304C211{}[1-10].addrs')
 }
 mnemonic = {
 	'bob':   'ignore bubble ignore crash stay long stay patient await glorious destination moon',
@@ -51,25 +51,26 @@ send_addr = {
 	'alice': '2N3HhxasbRvrJyHg72JNVCCPi9EUGrEbFnu',
 }
 
-def run_cmd(*args,**kwargs):
+def start_cmd(*args,**kwargs):
 	common_args = ('-rpcuser={}'.format(rpc_user),'-rpcpassword={}'.format(rpc_password),
 					'-regtest','-datadir={}'.format(data_dir))
 	cmds = {'cli': ('bitcoin-cli','-rpcconnect=localhost','-rpcport={}'.format(rpc_port)),
-			'daemon': ('bitcoind','-rpcbind=localhost:{}'.format(rpc_port),'-rpcallowip=::1')}
+			'daemon': ('bitcoind','-keypool=1','-rpcbind=localhost:{}'.format(rpc_port),'-rpcallowip=::1')}
 	wallet_arg = ()
 	if args[0] == 'daemon':
 		assert 'user' in kwargs
 		wallet_arg = ('-wallet={}'.format(os.path.basename(tr_wallet[kwargs['user']])),)
 	cmd = cmds[args[0]] + common_args + wallet_arg + args[1:] if args[0] in cmds else args
-	if not 'quiet' in kwargs:
-		vmsg(' '.join(cmd))
-	return subprocess.Popen(cmd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+	p = subprocess.Popen(cmd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+	if not 'quiet' in kwargs or g.debug:
+		vmsg('{}'.format(' '.join(cmd)))
+	return p
 
 def test_daemon():
-	p = run_cmd('cli','getblockcount',quiet=True)
-	o = p.stderr.read()
+	p = start_cmd('cli','getblockcount',quiet=True)
+	err = process_output(p,silent=True)[1]
 	ret,state = p.wait(),None
-	if "error: couldn't connect" in o: state = 'stopped'
+	if "error: couldn't connect" in err: state = 'stopped'
 	if not state: state = ('busy','ready')[ret==0]
 	return state
 
@@ -91,16 +92,17 @@ def get_balances():
 	tbal = 0
 	from mmgen.obj import BTCAmt
 	for user in (user1,user2):
-		p = run_cmd('./mmgen-tool','--{}'.format(user),'getbalance','quiet=1')
+		p = start_cmd('./mmgen-tool','--{}'.format(user),'getbalance','quiet=1')
 		bal = BTCAmt(p.stdout.read())
 		ustr = "{}'s balance:".format(user.capitalize())
-		msg('{:<16} {}'.format(ustr,bal))
+		msg('{:<16} {:12}'.format(ustr,bal))
 		tbal += bal
-	msg('{:<16} {}'.format('Total balance:',tbal))
+	msg('{:<16} {:12}'.format('Total balance:',tbal))
+	msg('{:<16} {:12}'.format('Miner fees:',1000-tbal))
 
 def create_data_dir():
 #def keypress_confirm(prompt,default_yes=False,verbose=False,no_nl=False):
-	try: os.stat(os.path.join(regtest_dir,'debug.log'))
+	try: os.stat(regtest_dir)
 	except: pass
 	else:
 		if keypress_confirm('Delete your existing MMGen regtest setup and create a new one?'):
@@ -111,59 +113,75 @@ def create_data_dir():
 	try: os.mkdir(data_dir)
 	except: pass
 
-def print_output(p):
-	qmsg('stdout: [{}]'.format(p.stdout.read().strip()))
-	qmsg('stderr: [{}]'.format(p.stderr.read().strip()))
+def process_output(p,silent=False):
+	out = p.stdout.read()
+	err = p.stderr.read()
+	if g.debug or not silent:
+		vmsg('stdout: [{}]'.format(out.strip()))
+		vmsg('stderr: [{}]'.format(err.strip()))
+	return out,err
 
-def	create_mmgen_wallet(user):
+def create_mmgen_wallet(user):
 	gmsg("Creating {}'s MMGen wallet".format(user.capitalize()))
-	p = run_cmd('mmgen-walletconv','-d',data_dir,'-i','words','-o','words')
+	p = start_cmd('mmgen-walletconv','-d',data_dir,'-i','words','-o','words')
 	p.stdin.write(mnemonic[user]+'\n')
 	p.stdin.close()
-	if opt.verbose: print_output(p)
+	err = process_output(p)[1]
+	if not 'written to file' in err:
+		rdie(1,'Error creating MMGen wallet')
 	p.wait()
 
-def	create_mmgen_addrs(user,addr_type):
+def create_mmgen_addrs(user,addr_type):
 	gmsg('Creating MMGen addresses for user {} (type: {})'.format(user.capitalize(),addr_type))
-	p = run_cmd('mmgen-addrgen','--{}'.format(user),'-d','/tmp','--type',addr_type,mmwords[user],'1-10')
+	suf = ('-'+addr_type,'')[addr_type=='L']
+	try: os.unlink(mmaddrs[user].format(suf))
+	except: pass
+	p = start_cmd('mmgen-addrgen','--{}'.format(user),'-d',data_dir,'--type',addr_type,mmwords[user],'1-10')
 	p.stdin.write(mnemonic[user]+'\n')
 	p.stdin.close()
-	if opt.verbose: print_output(p)
+	err = process_output(p)[1]
+	if not 'written to file' in err:
+		rdie(1,'Error creating MMGen addresses')
 	p.wait()
 
-def	import_mmgen_addrs(user,addr_mmtype):
-	gmsg_r('Importing MMGen addresses for user {} (type: {})'.format(user.capitalize(),addr_mmtype))
-	suf = '' if addr_mmtype=='L' else '-'+addr_mmtype
-	p = run_cmd('mmgen-addrimport','--{}'.format(user),'-q',mmaddrs[user].format(suf))
+def import_mmgen_addrs(user,addr_type):
+	gmsg_r('Importing MMGen addresses for user {} (type: {})'.format(user.capitalize(),addr_type))
+	suf = ('-'+addr_type,'')[addr_type=='L']
+	p = start_cmd('mmgen-addrimport','--{}'.format(user),'-q','--batch',mmaddrs[user].format(suf))
 	p.stdin.write(mnemonic[user]+'\n')
 	p.stdin.close()
-	if opt.verbose: print_output(p)
+	err = process_output(p)[1]
+	if not 'addresses imported' in err:
+		rdie(1,'Error importing MMGen addresses')
 	p.wait()
 
 def start_and_wait(user,silent=False,nonl=False):
 	if opt.verbose: msg('Starting bitcoin regtest daemon')
-	run_cmd('daemon','-daemon',user=user)
+	p = start_cmd('daemon','-daemon',user=user)
+	err = process_output(p)[1]
+	if err:
+		rdie(1,'Error starting the Bitcoin daemon:\n{}'.format(err))
 	wait_for_daemon('ready',silent=silent,nonl=nonl)
 
-def stop_and_wait(silent=False,nonl=False,stop_silent=False):
-	stop(silent=stop_silent)
+def stop_and_wait(silent=False,nonl=False,stop_silent=False,ignore_noconnect_error=False):
+	stop(silent=stop_silent,ignore_noconnect_error=ignore_noconnect_error)
 	wait_for_daemon('stopped',silent=silent,nonl=nonl)
 
-def	setup_wallet(user,addr_type,addr_code):
+def setup_wallet(user,addr_type):
 	gmsg_r("Setting up {}'s tracking wallet".format(user.capitalize()))
 	start_and_wait(user)
 	create_mmgen_wallet(user)
 	create_mmgen_addrs(user,addr_type)
-	import_mmgen_addrs(user,addr_code)
+	import_mmgen_addrs(user,addr_type)
 	stop_and_wait(stop_silent=True)
 
-def	setup_mixed_wallet(user):
+def setup_mixed_wallet(user):
 	gmsg_r("Setting up {}'s wallet (mixed address types)".format(user.capitalize()))
 	start_and_wait(user)
 	create_mmgen_wallet(user)
-	create_mmgen_addrs(user,'legacy')
-	create_mmgen_addrs(user,'compressed')
-	create_mmgen_addrs(user,'segwit')
+	create_mmgen_addrs(user,'L')
+	create_mmgen_addrs(user,'C')
+	create_mmgen_addrs(user,'S')
 	import_mmgen_addrs(user,'L'); msg('')
 	import_mmgen_addrs(user,'C'); msg('')
 	import_mmgen_addrs(user,'S'); msg('')
@@ -171,18 +189,19 @@ def	setup_mixed_wallet(user):
 
 def fund_wallet(user,amt):
 	gmsg('Sending {} BTC to {}'.format(amt,user.capitalize()))
-	p = run_cmd('cli','sendtoaddress',send_addr[user],str(amt))
-	if opt.verbose: print_output(p)
+	p = start_cmd('cli','sendtoaddress',send_addr[user],str(amt))
+	process_output(p)
 	p.wait()
 
 def setup():
-	if test_daemon(): stop_and_wait(silent=True,stop_silent=True)
+	if test_daemon() != 'stopped':
+		stop_and_wait(silent=True,stop_silent=True)
 	create_data_dir()
 	gmsg_r('Starting setup')
 
 	start_and_wait('orig')
 
-	generate(432)
+	generate(432,silent=True)
 
 	stop_and_wait(silent=True,stop_silent=True)
 
@@ -190,21 +209,22 @@ def setup():
 		setup_mixed_wallet('bob')
 		setup_mixed_wallet('alice')
 	else:
-		setup_wallet('bob','compressed','C')
-		setup_wallet('alice','segwit','S')
+		setup_wallet('bob','C')
+		setup_wallet('alice','S')
 
-	start_and_wait('orig',silent=True)
+	if opt.empty:
+		msg("'--empty' selected: skipping funding of wallets")
+	else:
+		start_and_wait('orig',silent=True)
+		fund_wallet('bob',init_amt)
+		fund_wallet('alice',init_amt)
+		generate(1)
+		stop_and_wait(silent=True,stop_silent=True)
 
-	fund_wallet('bob',init_amt)
-	fund_wallet('alice',init_amt)
-
-	generate(1)
-
-	stop_and_wait(silent=True,stop_silent=True)
 	gmsg('Setup complete')
 
 def get_current_user(quiet=False):
-	p = run_cmd('pgrep','-af', 'bitcoind.*-rpcuser={}.*'.format(rpc_user))
+	p = start_cmd('pgrep','-af', 'bitcoind.*-rpcuser={}.*'.format(rpc_user))
 	cmdline = p.stdout.read()
 	if not cmdline: return None
 	user = None
@@ -234,18 +254,24 @@ def user(user=None,quiet=False):
 		start_and_wait(user,silent=False,nonl=True)
 	gmsg('done')
 
-def stop(silent=False):
+def stop(silent=False,ignore_noconnect_error=True):
 	if test_daemon() != 'stopped' and not silent:
 		gmsg('Stopping bitcoin regtest daemon')
-	p = run_cmd('cli','stop')
-	ret = p.wait()
-	return ret
+	p = start_cmd('cli','stop')
+	err = process_output(p)[1]
+	if err:
+		if "couldn't connect to server" in err and not ignore_noconnect_error:
+			rdie(1,'Error stopping the Bitcoin daemon:\n{}'.format(err))
+		msg(err)
+	return p.wait()
 
-def generate(blocks=1):
+def generate(blocks=1,silent=False):
 	if test_daemon() == 'stopped':
 		die(1,'Regtest daemon is not running')
 	wait_for_daemon('ready',silent=True)
-	p = run_cmd('cli','generate',str(blocks))
-	if opt.verbose: print_output(p)
+	p = start_cmd('cli','generate',str(blocks))
+	out = process_output(p,silent=silent)[0]
+	if len(eval(out)) != blocks:
+		rdie(1,'Error generating blocks')
 	p.wait()
 	gmsg('Mined {} block{}'.format(blocks,suf(blocks,'s')))

mmgen/rpc.py

@@ -154,6 +154,7 @@ class BitcoinRPCConnection(object):
 		'getblockchaininfo',
 		'getblockcount',
 		'getblockhash',
+		'getmempoolinfo',
 		'getmempoolentry',
 		'getnettotals',
 		'getnetworkinfo',

mmgen/share/Opts.py

@@ -20,7 +20,7 @@
 Opts.py:  Generic options handling
 """
 
-import sys, getopt
+import sys,getopt
 # from mmgen.util import mdie,die,pdie,pmsg # DEBUG
 
 def usage(opts_data):
@@ -91,7 +91,6 @@ def process_opts(argv,opts_data,short_opts,long_opts,defer_help=False):
 
 	return opts,args,do_help
 
-
 def parse_opts(argv,opts_data,opt_filter=None,defer_help=False):
 
 	import re

mmgen/tool.py

@@ -78,7 +78,7 @@ cmd_data = OrderedDict([
 	('Mn_printlist', ["wordlist [str='electrum']"]),
 
 	('Listaddress',['<{} address> [str]'.format(pnm),'minconf [int=1]','pager [bool=False]','showempty [bool=True]''showbtcaddr [bool=True]']),
-	('Listaddresses',["addrs [str='']",'minconf [int=1]','showempty [bool=False]','pager [bool=False]','showbtcaddrs [bool=False]']),
+	('Listaddresses',["addrs [str='']",'minconf [int=1]','showempty [bool=False]','pager [bool=False]','showbtcaddrs [bool=False]','all_labels [bool=False]']),
 	('Getbalance',   ['minconf [int=1]','quiet [bool=False]']),
 	('Txview',       ['<{} TX file(s)> [str]'.format(pnm),'pager [bool=False]','terse [bool=False]',"sort [str='mtime'] (options: 'ctime','atime')",'MARGS']),
 	('Twview',       ["sort [str='age']",'reverse [bool=False]','show_days [bool=True]','show_mmid [bool=True]','minconf [int=1]','wide [bool=False]','pager [bool=False]']),
@@ -96,91 +96,6 @@ cmd_data = OrderedDict([
 	('Regtest_setup',[]),
 ])
 
-stdin_msg = """
-To force a command to read from STDIN in place of its first argument (for
-supported commands), use '-' as the first argument.
-""".strip()
-
-cmd_help = """
-Bitcoin address/key operations (compressed public keys supported):
-  addr2hexaddr   - convert Bitcoin address from base58 to hex format
-  hex2wif        - convert a private key from hex to WIF format
-  hexaddr2addr   - convert Bitcoin address from hex to base58 format
-  privhex2addr   - generate Bitcoin address from private key in hex format
-  privhex2pubhex - generate a hex public key from a hex private key
-  pubhex2addr    - convert a hex pubkey to an address
-  pubhex2redeem_script - convert a hex pubkey to a witness redeem script
-  wif2redeem_script - convert a WIF private key to a witness redeem script
-  wif2segwit_pair - generate both a Segwit redeem script and address from WIF
-  pubkey2addr    - convert Bitcoin public key to address
-  randpair       - generate a random private key/address pair
-  randwif        - generate a random private key in WIF format
-  wif2addr       - generate a Bitcoin address from a key in WIF format
-  wif2hex        - convert a private key from WIF to hex format
-
-Wallet/TX operations (bitcoind must be running):
-  getbalance    - like 'bitcoin-cli getbalance' but shows confirmed/unconfirmed,
-                  spendable/unspendable balances for individual {pnm} wallets
-  listaddress   - list the specified {pnm} address and its balance
-  listaddresses - list {pnm} addresses and their balances
-  txview        - show raw/signed {pnm} transaction in human-readable form
-  twview        - view tracking wallet
-
-General utilities:
-  hexdump      - encode data into formatted hexadecimal form (file or stdin)
-  unhexdump    - decode formatted hexadecimal data (file or stdin)
-  bytespec     - convert a byte specifier such as '1GB' into an integer
-  hexlify      - display string in hexadecimal format
-  hexreverse   - reverse bytes of a hexadecimal string
-  rand2file    - write 'n' bytes of random data to specified file
-  randhex      - print 'n' bytes (default 32) of random data in hex format
-  hash256      - compute sha256(sha256(data)) (double sha256)
-  hash160      - compute ripemd160(sha256(data)) (converts hexpubkey to hexaddr)
-  b58randenc   - generate a random 32-byte number and convert it to base 58
-  b58tostr     - convert a base 58 number to a string
-  strtob58     - convert a string to base 58
-  b58tohex     - convert a base 58 number to hexadecimal
-  hextob58     - convert a hexadecimal number to base 58
-  b32tohex     - convert a base 32 number to hexadecimal
-  hextob32     - convert a hexadecimal number to base 32
-
-File encryption:
-  encrypt      - encrypt a file
-  decrypt      - decrypt a file
-    {pnm} encryption suite:
-      * Key: Scrypt (user-configurable hash parameters, 32-byte salt)
-      * Enc: AES256_CTR, 16-byte rand IV, sha256 hash + 32-byte nonce + data
-      * The encrypted file is indistinguishable from random data
-
-{pnm}-specific operations:
-  add_label    - add descriptive label for {pnm} address in tracking wallet
-  remove_label - remove descriptive label for {pnm} address in tracking wallet
-  addrfile_chksum    - compute checksum for {pnm} address file
-  keyaddrfile_chksum - compute checksum for {pnm} key-address file
-  passwdfile_chksum  - compute checksum for {pnm} password file
-  find_incog_data    - Use an Incog ID to find hidden incognito wallet data
-  id6          - generate 6-character {pnm} ID for a file (or stdin)
-  id8          - generate 8-character {pnm} ID for a file (or stdin)
-  str2id6      - generate 6-character {pnm} ID for a string, ignoring spaces
-
-Mnemonic operations (choose 'electrum' (default), 'tirosh' or 'all'
-  wordlists):
-  mn_rand128   - generate random 128-bit mnemonic
-  mn_rand192   - generate random 192-bit mnemonic
-  mn_rand256   - generate random 256-bit mnemonic
-  mn_stats     - show stats for mnemonic wordlist
-  mn_printlist - print mnemonic wordlist
-  hex2mn       - convert a 16, 24 or 32-byte number in hex format to a mnemonic
-  mn2hex       - convert a 12, 18 or 24-word mnemonic to a number in hex format
-
-  IMPORTANT NOTE: Though {pnm} mnemonics use the Electrum wordlist, they're
-  computed using a different algorithm and are NOT Electrum-compatible!
-
-Miscellaneous
-  regtest_setup - setup a regtest environment for testing MMGen scripts
-  {sm}
-""".format(pnm=pnm,sm='\n  '.join(stdin_msg.split('\n')))
-
 def usage(command):
 
 	for v in cmd_data.values():
@@ -192,12 +107,14 @@ def usage(command):
 		Msg('Usage information for mmgen-tool commands:')
 		for k,v in cmd_data.items():
 			Msg('  {:18} {}'.format(k.lower(),' '.join(v)))
+		from mmgen.main_tool import stdin_msg
 		Msg('\n  '+'\n  '.join(stdin_msg.split('\n')))
 		sys.exit(0)
 
 	Command = command.capitalize()
 	if Command in cmd_data:
 		import re
+		from mmgen.main_tool import cmd_help
 		for line in cmd_help.split('\n'):
 			if re.match(r'\s+{}\s+'.format(command),line):
 				c,h = line.split('-',1)
@@ -437,32 +354,18 @@ def Listaddress(addr,minconf=1,pager=False,showempty=True,showbtcaddr=True):
 	return Listaddresses(addrs=addr,minconf=minconf,pager=pager,showempty=showempty,showbtcaddrs=showbtcaddr)
 
 # List MMGen addresses and their balances.  TODO: move this code to AddrList
-def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=False):
+def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=False,all_labels=False):
 
-	c = bitcoin_connection()
+	c = rpc_connection()
 
-	def check_dup_mmid(accts):
-		help_msg = """
-    Your tracking wallet is corrupted or has been altered by a non-{pnm} program.
-
-    You might be able to salvage your wallet by determining which of the offending
-    addresses doesn't belong to {pnm} ID {mid} and then typing:
-
-        bitcoin-cli importaddress <offending address> "" false
-	"""
-		m_prev = None
-
-		for m in sorted(b.mmid for b in [a for a in accts if a]):
-			if m == m_prev:
-				msg('Duplicate MMGen ID ({}) discovered in tracking wallet!\n'.format(m))
-				bad_accts = MMGenList([l for l in accts if l.mmid == m])
-				msg('  Affected Bitcoin RPC accounts:\n    {}\n'.format('\n    '.join(bad_accts)))
-				bad_addrs = [a[0] for a in c.getaddressesbyaccount([[a] for a in bad_accts],batch=True)]
-				if len(set(bad_addrs)) != 1:
-					msg('  Offending addresses:\n    {}'.format('\n    '.join(bad_addrs)))
-					msg(help_msg.format(mid=m,pnm=pnm))
-				die(3,red('Exiting on error'))
-			m_prev = m
+	def check_dup_mmid(acct_labels):
+		mmid_prev,err = None,False
+		for mmid in sorted(a.mmid for a in acct_labels if a):
+			if mmid == mmid_prev:
+				err = True
+				msg('Duplicate MMGen ID ({}) discovered in tracking wallet!\n'.format(mmid))
+			mmid_prev = mmid
+		if err: rdie(3,'Tracking wallet is corrupted!')
 
 	def check_addr_array_lens(acct_pairs):
 		err = False
@@ -505,7 +408,7 @@ def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=Fa
 			total += d['amount']
 
 	# We use listaccounts only for empty addresses, as it shows false positive balances
-	if showempty:
+	if showempty or all_labels:
 		# for compatibility with old mmids, must use raw RPC rather than native data for matching
 		# args: minconf,watchonly, MUST use keys() so we get list, not dict
 		acct_list = c.listaccounts(0,True).keys() # raw list, no 'L'
@@ -517,6 +420,7 @@ def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=Fa
 		check_addr_array_lens(addr_pairs)
 		for label,addr_arr in addr_pairs:
 			if not label: continue
+			if all_labels and not showempty and not label.comment: continue
 			if usr_addr_list and (label.mmid not in usr_addr_list): continue
 			if label.mmid not in addrs:
 				addrs[label.mmid] = { 'amt':BTCAmt('0'), 'lbl':label, 'addr':'' }
@@ -550,7 +454,7 @@ def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=Fa
 			if al_id_save:
 				out.append('')
 				al_id_save = None
-			mmid_disp = mmid.type
+			mmid_disp = 'Non-MMGen'
 		out.append(fs.format(
 			mid = MMGenID.fmtc(mmid_disp,width=max_mmid_len,color=True),
 			addr=(addrs[mmid]['addr'].fmt(color=True) if showbtcaddrs else None),
@@ -563,7 +467,7 @@ def Listaddresses(addrs='',minconf=1,showempty=False,pager=False,showbtcaddrs=Fa
 
 def Getbalance(minconf=1,quiet=False):
 	accts = {}
-	for d in bitcoin_connection().listunspent(0):
+	for d in rpc_connection().listunspent(0):
 		ma = split2(d['account'] if 'account' in d else '')[0] # include coinbase outputs if spendable
 		keys = ['TOTAL']
 		if d['spendable']: keys += ['SPENDABLE']
@@ -577,7 +481,7 @@ def Getbalance(minconf=1,quiet=False):
 				accts[key][j] += d['amount']
 
 	if quiet:
-		Msg('{}'.format(accts['TOTAL'][2]))
+		Msg('{}'.format(accts['TOTAL'][2] if accts else BTCAmt('0')))
 	else:
 		fs = '{:13} {} {} {}'
 		mc,lbl = str(minconf),'confirms'
@@ -597,7 +501,7 @@ def Txview(*infiles,**kwargs):
 	flist = MMGenFileList(infiles,ftype=MMGenTX)
 	flist.sort_by_age(key=sort_key) # in-place sort
 	from mmgen.term import get_terminal_size
-	sep = u'—'*get_terminal_size()[0]+'\n'
+	sep = u'—'*77+'\n'
 	out = sep.join([MMGenTX(fn).format_view(terse=terse) for fn in flist.names()])
 	(Msg,do_pager)[pager](out.rstrip())
 

mmgen/tw.py

@@ -80,7 +80,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		if g.bogus_wallet_data: # for debugging purposes only
 			us_rpc = eval(get_data_from_file(g.bogus_wallet_data))
 		else:
-			us_rpc = bitcoin_connection().listunspent(self.minconf)
+			us_rpc = rpc_connection().listunspent(self.minconf)
 #		write_data_to_file('bogus_unspent.json', repr(us), 'bogus unspent data')
 #		sys.exit(0)
 
@@ -336,7 +336,7 @@ Display options: show [D]ays, [g]roup, show [m]mgen addr, r[e]draw screen
 			msg("Address '{}' not in tracking wallet".format(btcaddr))
 			return False
 
-		c = bitcoin_connection()
+		c = rpc_connection()
 		if not btcaddr.is_for_current_chain():
 			msg("Address '{}' not valid for chain {}".format(btcaddr,g.chain.upper()))
 			return False

mmgen/tx.py

@@ -27,7 +27,7 @@ from mmgen.common import *
 from mmgen.obj import *
 
 def segwit_is_active(exit_on_error=False):
-	d = bitcoin_connection().getblockchaininfo()
+	d = rpc_connection().getblockchaininfo()
 	if d['chain'] == 'regtest':
 		return True
 	if 'segwit' in d['bip9_softforks'] and d['bip9_softforks']['segwit']['status'] == 'active':
@@ -307,7 +307,7 @@ class MMGenTX(MMGenObject):
 
 	def get_relay_fee(self):
 		assert self.estimate_size()
-		kb_fee = BTCAmt(bitcoin_connection().getnetworkinfo()['relayfee'])
+		kb_fee = BTCAmt(rpc_connection().getnetworkinfo()['relayfee'])
 		vmsg('Relay fee: {} {}/kB'.format(kb_fee,g.coin))
 		return kb_fee * self.estimate_size() / 1024
 
@@ -640,7 +640,7 @@ class MMGenTX(MMGenObject):
 
 # 	def is_rbf_fromhex(self,color=False):
 # 		try:
-# 			dec_tx = bitcoin_connection().decoderawtransaction(self.hex)
+# 			dec_tx = rpc_connection().decoderawtransaction(self.hex)
 # 		except:
 # 			return yellow('Unknown') if color else None
 # 		rbf = bool(dec_tx['vin'][0]['sequence'] == g.max_int - 2)
@@ -655,7 +655,7 @@ class MMGenTX(MMGenObject):
 
 	def format_view(self,terse=False):
 		try:
-			blockcount = bitcoin_connection().getblockcount()
+			blockcount = rpc_connection().getblockcount()
 		except:
 			blockcount = None
 

mmgen/txcreate.py

@@ -119,7 +119,7 @@ def mmaddr2baddr(c,mmaddr,ad_w,ad_f):
 
 def get_fee_from_estimate_or_user(tx,estimate_fail_msg_shown=[]):
 
-	c = bitcoin_connection()
+	c = rpc_connection()
 
 	if opt.tx_fee:
 		desc = 'User-selected'
@@ -218,7 +218,7 @@ def txcreate(cmd_args,do_info=False,caller='txcreate'):
 
 	if opt.comment_file: tx.add_comment(opt.comment_file)
 
-	c = bitcoin_connection()
+	c = rpc_connection()
 
 	if not do_info: get_outputs_from_cmdline(cmd_args,tx)
 

mmgen/util.py

@@ -62,7 +62,7 @@ def Die(ev=0,s=''):
 	sys.exit(ev)
 
 def rdie(ev=0,s=''): die(ev,red(s))
-def wdie(ev=0,s=''): die(ev,yellow(s))
+def ydie(ev=0,s=''): die(ev,yellow(s))
 def hi(): sys.stdout.write(yellow('hi'))
 
 def pformat(d):
@@ -805,9 +805,9 @@ def get_bitcoind_auth_cookie():
 	else:
 		return ''
 
-def bitcoin_connection():
+def rpc_connection():
 
-	def	check_coin_mismatch(c):
+	def check_coin_mismatch(c):
 		if c.getblockcount() == 0:
 			msg('Warning: no blockchain, so skipping block mismatch check')
 			return
@@ -816,9 +816,9 @@ def bitcoin_connection():
 		if c.getblockchaininfo()['blocks'] <= 478558 or c.getblockhash(478559) == fb:
 			if g.coin == 'BCH': err = 'BCH','BTC'
 		elif g.coin == 'BTC': err = 'BTC','BCH'
-		if err: wdie(2,"'{}' requested, but this is the {} chain!".format(*err))
+		if err: ydie(2,"'{}' requested, but this is the {} chain!".format(*err))
 
-	def	check_chain_mismatch():
+	def check_chain_mismatch():
 		err = None
 		if g.regtest and g.chain != 'regtest':
 			err = '--regtest option'

scripts/traceback.py

@@ -10,10 +10,7 @@ try:
 	sys.argv.pop(0)
 	execfile(sys.argv[0])
 except SystemExit:
-	try:
-		sys.exit(int(str(sys.exc_info()[1])))
-	except:
-		sys.exit(1)
+	sys.exit(int(str(sys.exc_info()[1])))
 except:
 	l = traceback.format_exception(*sys.exc_info())
 	exc = l.pop()

test/test.py

@@ -790,7 +790,7 @@ def create_fake_unspent_data(adata,tx_data,non_mmgen_input=''):
 #	msg('\n'.join([repr(o) for o in out])); sys.exit(0)
 	return out
 
-def	write_fake_data_to_file(d):
+def write_fake_data_to_file(d):
 	unspent_data_file = os.path.join(cfg['tmpdir'],'unspent.json')
 	write_data_to_file(unspent_data_file,d,'Unspent outputs',silent=True)
 	os.environ['MMGEN_BOGUS_WALLET_DATA'] = unspent_data_file