Data directory, config file and default wallet support:

 * Under Linux, data directory is '~/.mmgen'; config file is 'mmgen.cfg'.
 * If default wallet is enabled, specifying the wallet on the command line is
   no longer necessary.  If not, all MMGen commands behave as previously.
 * The datadir structure mirrors that of Bitcoin Core: mainnet and testnet
   share a common config file, but testnet puts its other files including its
   default wallet in a separate subdirectory 'testnet3'.

New routines have been added to the test suite to test these features.
Tested only under Linux.

data_files/mmgen.cfg

@@ -8,6 +8,9 @@
 # Uncomment to suppress the GPL license prompt:
 # no_license true
 
+# Uncomment to enable quieter output:
+# quiet true
+
 # Uncomment to disable color output:
 # color false
 

mmgen/crypto.py

@@ -24,7 +24,6 @@ from binascii import hexlify
 from hashlib import sha256
 
 from mmgen.common import *
-from mmgen.term import get_char
 
 crmsg = {
 	'usr_rand_notice': """
@@ -157,6 +156,7 @@ def _get_random_data_from_user(uchars):
 	msg_r(prompt % uchars)
 
 	import time
+	from mmgen.term import get_char
 	# time.clock() always returns zero, so we'll use time.time()
 	saved_time = time.time()
 

mmgen/filename.py

@@ -21,7 +21,8 @@ filename.py:  Filename class and methods for the MMGen suite
 """
 import sys,os
 from mmgen.obj import *
-from mmgen.util import die,get_extension,check_infile
+from mmgen.util import die,get_extension
+from mmgen.seed import *
 
 class Filename(MMGenObject):
 
@@ -63,3 +64,26 @@ class Filename(MMGenObject):
 				os.close(fd)
 		else:
 			self.size = os.stat(fn).st_size
+
+def find_files_in_dir(ftype,fdir,no_dups=False):
+	if type(ftype) != type:
+		die(3,"'{}': not a type".format(ftype))
+
+	from mmgen.seed import SeedSource
+	if not issubclass(ftype,SeedSource):
+		die(3,"'{}': not a recognized file type".format(ftype))
+
+	try: dirlist = os.listdir(fdir)
+	except: die(3,"ERROR: unable to read directory '{}'".format(fdir))
+
+	matches = [l for l in dirlist if l[-len(ftype.ext)-1:]=='.'+ftype.ext]
+
+	if no_dups:
+		if len(matches) > 1:
+			die(1,"ERROR: more than one {} file in directory '{}'".format(ftype.__name__,fdir))
+		return os.path.join(fdir,matches[0]) if len(matches) else None
+	else:
+		return [os.path.join(fdir,m) for m in matches]
+
+def find_file_in_dir(ftype,fdir,no_dups=True):
+	return find_files_in_dir(ftype,fdir,no_dups=no_dups)

mmgen/globalvars.py

@@ -59,13 +59,14 @@ class g(object):
 	# Constants - some of these might be overriden, but they don't change thereafter
 
 	debug                = False
+	quiet                = False
 	no_license           = False
 	hold_protect         = True
 	color                = (False,True)[sys.stdout.isatty()]
 	testnet              = False
-	bogus_wallet_data    = ''
 	rpc_host             = 'localhost'
 	testnet_name         = 'testnet3'
+	bogus_wallet_data    = '' # for debugging, used by test suite
 
 	for k in ('win','linux'):
 		if sys.platform[:len(k)] == k:
@@ -81,18 +82,18 @@ class g(object):
 		m = ('$HOME is not set','Neither $HOME nor %HOMEPATH% are set')[platform=='win']
 		die(2,m + '\nUnable to determine home directory')
 
-	data_dir = (os.path.join(home_dir,'Application Data',proj_name),
-				os.path.join(home_dir,'.'+proj_name.lower()))[bool(os.getenv('HOME'))]
+	data_dir_root = None
+	data_dir = None
+	cfg_file = None
 	bitcoin_data_dir = (os.path.join(home_dir,'Application Data','Bitcoin'),
 				os.path.join(home_dir,'.bitcoin'))[bool(os.getenv('HOME'))]
-	cfg_file = os.path.join(data_dir,'{}.cfg'.format(proj_name.lower()))
 
-	common_opts = ['color','no_license','rpc_host','testnet']
-	required_opts = [
+	common_opts = ('color','no_license','rpc_host','testnet')
+	required_opts = (
 		'quiet','verbose','debug','outdir','echo_passphrase','passwd_file','stdout',
 		'show_hash_presets','label','keep_passphrase','keep_hash_preset',
 		'brain_params','b16','usr_randchars'
-	]
+	)
 	incompatible_opts = (
 		('quiet','verbose'),
 		('label','keep_label'),
@@ -100,9 +101,14 @@ class g(object):
 		('tx_id','terse_info'),
 		('batch','rescan'),
 	)
+	cfg_file_opts = (
+		'color','debug','hash_preset','http_timeout','no_license','rpc_host',
+		'quiet','tx_fee','tx_fee_adj','usr_randchars','testnet'
+	)
 	env_opts = (
 		'MMGEN_BOGUS_WALLET_DATA',
 		'MMGEN_DEBUG',
+		'MMGEN_QUIET',
 		'MMGEN_DISABLE_COLOR',
 		'MMGEN_DISABLE_HOLD_PROTECT',
 		'MMGEN_MIN_URANDCHARS',
@@ -116,7 +122,7 @@ class g(object):
 
 	# Global var sets user opt:
 	global_sets_opt = ['minconf','seed_len','hash_preset','usr_randchars','debug',
-                       'tx_confs','tx_fee_adj','tx_fee','key_generator']
+						'quiet','tx_confs','tx_fee_adj','tx_fee','key_generator']
 
 	keyconv_exec = 'keyconv'
 
@@ -149,68 +155,3 @@ class g(object):
 		'6': [17, 8, 20],
 		'7': [18, 8, 24],
 	}
-
-def create_data_dir(g):
-	from mmgen.util import msg,die
-	try:
-		os.listdir(g.data_dir)
-	except:
-		try:
-			os.mkdir(g.data_dir,0700)
-		except:
-			die(2,"ERROR: unable to read or create '{}'".format(g.data_dir))
-
-def get_data_from_config_file(g):
-	from mmgen.util import msg,die
-	# https://wiki.debian.org/Python:
-	#   Debian (Ubuntu) sys.prefix is '/usr' rather than '/usr/local, so add 'local'
-	# TODO - test for Windows
-	# This must match the configuration in setup.py
-	data = u''
-	try:
-		with open(g.cfg_file,'rb') as f: data = f.read().decode('utf8')
-	except:
-		cfg_template = os.path.join(*([sys.prefix]
-					+ ([''],['local','share'])[bool(os.getenv('HOME'))]
-					+ [g.proj_name.lower(),os.path.basename(g.cfg_file)]))
-		try:
-			with open(cfg_template,'rb') as f: template_data = f.read()
-		except:
-			msg("WARNING: configuration template not found at '{}'".format(cfg_template))
-		else:
-			try:
-				with open(g.cfg_file,'wb') as f: f.write(template_data)
-				os.chmod(g.cfg_file,0600)
-			except:
-				die(2,"ERROR: unable to write to datadir '{}'".format(g.data_dir))
-	return data
-
-def override_from_cfg_file(g,cfg_data):
-	from mmgen.util import die,strip_comments,set_for_type
-	cvars = ('color','debug','hash_preset','http_timeout','no_license','rpc_host',
-			'testnet','tx_fee','tx_fee_adj','usr_randchars')
-	import re
-	for n,l in enumerate(cfg_data.splitlines(),1): # DOS-safe
-		l = strip_comments(l)
-		if l == '': continue
-		m = re.match(r'(\w+)\s+(\S+)$',l)
-		if not m: die(2,"Parse error in file '{}', line {}".format(g.cfg_file,n))
-		name,val = m.groups()
-		if name in cvars:
-			setattr(g,name,set_for_type(val,getattr(g,name),name,src=g.cfg_file))
-		else:
-			die(2,"'{}': unrecognized option in '{}'".format(name,g.cfg_file))
-
-def override_from_env(g):
-	from mmgen.util import set_for_type
-	for name in g.env_opts:
-		idx,invert_bool = ((6,False),(14,True))[name[:14]=='MMGEN_DISABLE_']
-		val = os.getenv(name) # os.getenv() returns None if env var is unset
-		if val:
-			gname = name[idx:].lower()
-			setattr(g,gname,set_for_type(val,getattr(g,gname),name,invert_bool))
-
-create_data_dir(g)
-cfg_data = get_data_from_config_file(g)
-override_from_cfg_file(g,cfg_data)
-override_from_env(g)

mmgen/main_addrgen.py

@@ -44,7 +44,7 @@ opts_data = {
 	'sets': [('print_checksum',True,'quiet',True)],
 	'desc': """Generate a range or list of {what} from an {pnm} wallet,
                   mnemonic, seed or password""".format(what=gen_what,pnm=g.proj_name),
-	'usage':'[opts] [infile] <address range or list>',
+	'usage':'[opts] [infile] <range or list of address indexes>',
 	'options': """
 -h, --help            Print this help message
 --, --longhelp        Print help message for long options (common options)
@@ -80,8 +80,7 @@ opts_data = {
 ),
 	'notes': """
 
-Addresses are given in a comma-separated list.  Hyphen-separated ranges are
-also allowed.
+Address indexes are given in a comma-separated list and/or hyphen-separated ranges.
 
 {n}
 
@@ -100,18 +99,14 @@ FMT CODES:
 
 cmd_args = opts.init(opts_data,add_opts=['b16'],opt_filter=opt_filter)
 
-nargs = 2
-if len(cmd_args) < nargs and not (opt.hidden_incog_input_params or opt.in_fmt):
-	opts.usage()
-elif len(cmd_args) > nargs - int(bool(opt.hidden_incog_input_params)):
-	opts.usage()
+if len(cmd_args) < 1: opts.usage()
+idxs = AddrIdxList(fmt_str=cmd_args.pop())
 
-addridxlist_str = cmd_args.pop()
-idxs = AddrIdxList(fmt_str=addridxlist_str)
+sf = get_seed_file(cmd_args,1)
 
 do_license_msg()
 
-ss = SeedSource(*cmd_args) # *(cmd_args[0] if cmd_args else [])
+ss = SeedSource(sf)
 
 i = (gen_what=='addresses') or bool(opt.no_addresses)*2
 al = (KeyAddrList,AddrList,KeyList)[i](seed=ss.seed,addr_idxs=idxs)

mmgen/main_addrimport.py

@@ -47,7 +47,7 @@ opts_data = {
 This command can also be used to update the comment fields of addresses already
 in the tracking wallet.
 
-The --batch option cannot be used with the --rescan option.
+The --batch and --rescan options cannot be used together.
 """
 }
 

mmgen/main_txcreate.py

@@ -46,8 +46,8 @@ opts_data = {
 """.format(g=g),
 	'notes': """
 
-Transaction inputs are chosen from a list of the user's unpent outputs
-via an interactive menu.
+The transaction's outputs are specified on the command line, while its inputs
+are chosen from a list of the user's unpent outputs via an interactive menu.
 
 If the transaction fee is not specified by the user, it will be calculated
 using bitcoind's "estimatefee" function for the default (or user-specified)

mmgen/main_txsign.py

@@ -69,37 +69,34 @@ opts_data = {
 		kg=g.key_generator),
 	'notes': """
 
-Transactions with either {pnm} or non-{pnm} input addresses may be signed.
-For non-{pnm} inputs, the bitcoind wallet.dat is used as the key source.
-For {pnm} inputs, key data is generated from your seed as with the
-{pnl}-addrgen and {pnl}-keygen utilities.
-
-Data for the --from-<what> options will be taken from a file if a second
-file is specified on the command line.  Otherwise, the user will be
-prompted to enter the data.
-
-In cases of transactions with mixed {pnm} and non-{pnm} inputs, non-{pnm}
-keys must be supplied in a separate file (WIF format, one key per line)
-using the '--keys-from-file' option.  Alternatively, one may get keys from
-a running bitcoind using the '--force-wallet-dat' option.  First import the
-required {pnm} keys using 'bitcoind importprivkey'.
-
-For transaction outputs that are {pnm} addresses, {pnm}-to-Bitcoin address
-mappings are verified.  Therefore, seed material or a key-address file for
-these addresses must be supplied on the command line.
-
-Seed data supplied in files must have the following extensions:
-   wallet:      '.{w.ext}'
-   seed:        '.{s.ext}'
-   mnemonic:    '.{m.ext}'
-   brainwallet: '.{b.ext}'
-
-FMT CODES:
+Transactions may contain both {pnm} or non-{pnm} input addresses.
+
+To sign non-{pnm} inputs, a bitcoind wallet dump or flat key list is used
+as the key source ('--keys-from-file' option).
+
+To sign {pnm} inputs, key data is generated from a seed as with the
+{pnl}-addrgen and {pnl}-keygen commands.  Alternatively, a key-address file
+may be used (--mmgen-keys-from-file option).
+
+Multiple wallets or other seed files can be listed on the command line in
+any order.  If the seeds required to sign the transaction's inputs are not
+found in these files (or in the default wallet), the user will be prompted
+for seed data interactively.
+
+To prevent an attacker from crafting transactions with bogus {pnm}-to-Bitcoin
+address mappings, all outputs to {pnm} addresses are verified with a seed
+source.  Therefore, seed files or a key-address file for all {pnm} outputs
+must also be supplied on the command line if the data can't be found in the
+default wallet.
+
+Seed source files must have the canonical extensions listed in the 'FileExt'
+column below:
+
   {f}
 """.format(
 		f='\n  '.join(SeedSource.format_fmt_codes().splitlines()),
 		pnm=pnm,pnl=pnm.lower(),
-		w=Wallet,s=SeedFile,m=Mnemonic,b=Brainwallet
+		w=Wallet,s=SeedFile,m=Mnemonic,b=Brainwallet,x=IncogWalletHex,h=IncogWallet
 	)
 }
 
@@ -166,39 +163,6 @@ def add_keys(tx,src,infiles=None,saved_seeds=None,keyaddr_list=None):
 		vmsg('Added %s wif key%s from %s' % (len(new_keys),suf(new_keys,'k'),desc))
 	return new_keys
 
-# # functions unneeded - use bitcoin-cli walletdump instead
-# def get_bitcoind_passphrase(prompt):
-# 	if opt.passwd_file:
-# 		pwfile_reuse_warning()
-# 		return get_data_from_file(opt.passwd_file,'passphrase').strip('\r\n')
-# 	else:
-# 		return my_raw_input(prompt, echo=opt.echo_passphrase)
-#
-# def sign_tx_with_bitcoind_wallet(c,tx,tx_num_str,keys):
-# 	ok = tx.sign(c,tx_num_str,keys) # returns false on failure
-# 	if ok:
-# 		return ok
-# 	else:
-# 		msg('Using keys in wallet.dat as per user request')
-# 		prompt = 'Enter passphrase for bitcoind wallet: '
-# 		while True:
-# 			passwd = get_bitcoind_passphrase(prompt)
-# 			ret = c.walletpassphrase(passwd, 9999,on_fail='return')
-# 			if rpc_error(ret):
-# 				if rpc_errmsg(ret,'unencrypted wallet, but walletpassphrase was called'):
-# 					msg('Wallet is unencrypted'); break
-# 			else:
-# 				msg('Passphrase OK'); break
-#
-# 		ok = tx.sign(c,tx_num_str,keys)
-#
-# 		msg('Locking wallet')
-# 		ret = c.walletlock(on_fail='return')
-# 		if rpc_error(ret):
-# 			msg('Failed to lock wallet')
-#
-# 		return ok
-
 # main(): execution begins here
 
 infiles = opts.init(opts_data,add_opts=['b16'])
@@ -212,6 +176,10 @@ saved_seeds = {}
 tx_files   = [i for i in infiles if get_extension(i) == MMGenTX.raw_ext]
 seed_files = [i for i in infiles if get_extension(i) in SeedSource.get_extensions()]
 
+from mmgen.filename import find_file_in_dir
+wf = find_file_in_dir(Wallet,g.data_dir)
+if wf: seed_files.append(wf)
+
 if not tx_files:
 	die(1,'You must specify a raw transaction file!')
 if not (seed_files or opt.mmgen_keys_from_file or opt.keys_from_file): # or opt.use_wallet_dat):

mmgen/main_wallet.py

@@ -23,7 +23,8 @@ mmgen/main_wallet:  Entry point for MMGen wallet-related scripts
 import os,re
 
 from mmgen.common import *
-from mmgen.seed import SeedSource
+from mmgen.seed import SeedSource,Wallet
+from mmgen.filename import find_file_in_dir
 from mmgen.obj import MMGenWalletLabel
 
 bn = os.path.basename(sys.argv[0])
@@ -36,6 +37,7 @@ oaction = 'convert'
 bw_note = opts.bw_note
 pw_note = opts.pw_note
 
+# full: defhHiJkKlLmoOpPqrSvz-
 if invoked_as == 'gen':
 	desc = 'Generate an {pnm} wallet from a random seed'
 	opt_filter = 'ehdoJlLpPqrSvz-'
@@ -44,14 +46,14 @@ if invoked_as == 'gen':
 	nargs = 0
 elif invoked_as == 'conv':
 	desc = 'Convert an {pnm} wallet from one format to another'
-	opt_filter = None
+	opt_filter = 'dehHiJkKlLmoOpPqrSvz-'
 elif invoked_as == 'chk':
 	desc = 'Check validity of an {pnm} wallet'
 	opt_filter = 'ehiHOlpPqrvz-'
 	iaction = 'input'
 elif invoked_as == 'passchg':
-	desc = 'Change the password, hash preset or label of an {pnm} wallet'
-	opt_filter = 'ehdiHkKOlLmpPqrSvz-'
+	desc = 'Change the passphrase, hash preset or label of an {pnm} wallet'
+	opt_filter = 'efhdiHkKOlLmpPqrSvz-'
 	iaction = 'input'
 	bw_note = ''
 else:
@@ -67,6 +69,7 @@ opts_data = {
 --, --longhelp        Print help message for long options (common options)
 -d, --outdir=      d  Output files to directory 'd' instead of working dir
 -e, --echo-passphrase Echo passphrases and other user input to screen
+-f, --force-update    Force update of wallet even if nothing has changed
 -i, --in-fmt=      f  {iaction} from wallet format 'f' (see FMT CODES below)
 -o, --out-fmt=     f  {oaction} to wallet format 'f' (see FMT CODES below)
 -H, --hidden-incog-input-params=f,o  Read hidden incognito data from file
@@ -114,30 +117,56 @@ cmd_args = opts.init(opts_data,opt_filter=opt_filter)
 if opt.label:
 	opt.label = MMGenWalletLabel(opt.label,msg="Error in option '--label'")
 
-if len(cmd_args) < nargs \
-		and not opt.hidden_incog_input_params and not opt.in_fmt:
-	die(1,'An input file or input format must be specified')
-elif len(cmd_args) > nargs \
-		or (len(cmd_args) == nargs and opt.hidden_incog_input_params):
-	msg('No input files may be specified' if invoked_as == 'gen'
-			else 'Too many input files specified')
-	opts.usage()
-
-if cmd_args: check_infile(cmd_args[0])
+sf = get_seed_file(cmd_args,nargs,invoked_as=invoked_as)
 
 if not invoked_as == 'chk': do_license_msg()
 
-if invoked_as in ('conv','passchg'): msg(green('Processing input wallet'))
+dw_msg = ('',yellow(' (default wallet)'))[bool(sf and os.path.dirname(sf)==g.data_dir)]
+
+if invoked_as in ('conv','passchg'):
+	msg(green('Processing input wallet')+dw_msg)
 
-ss_in = None if invoked_as == 'gen' \
-			else SeedSource(*cmd_args,passchg=invoked_as=='passchg')
+ss_in = None if invoked_as == 'gen' else SeedSource(sf,passchg=(invoked_as=='passchg'))
 
 if invoked_as == 'chk': sys.exit()
 
-if invoked_as in ('conv','passchg'): msg(green('Processing output wallet'))
+if invoked_as in ('conv','passchg'):
+	msg(green('Processing output wallet'))
 
 ss_out = SeedSource(ss=ss_in,passchg=invoked_as=='passchg')
 
-if invoked_as == 'gen': qmsg("This wallet's Seed ID: %s" % ss_out.seed.sid.hl())
-
-ss_out.write_to_file()
+if invoked_as == 'gen':
+	qmsg("This wallet's Seed ID: %s" % ss_out.seed.sid.hl())
+
+if invoked_as == 'passchg':
+	if not (opt.force_update or [k for k in 'passwd','hash_preset','label'
+		if getattr(ss_out.ssdata,k) != getattr(ss_in.ssdata,k)]):
+		die(1,'Password, hash preset and label are unchanged.  Taking no action')
+
+m1 = yellow('Confirmation of default wallet update')
+m2 = 'update the default wallet'
+m3 = 'Make this wallet your default and move it to the data directory?'
+
+if invoked_as == 'passchg' and ss_in.infile.dirname == g.data_dir:
+	confirm_or_exit(m1,m2,exit_msg='Password not changed')
+	ss_out.write_to_file(desc='New wallet',outdir=g.data_dir)
+	msg('Deleting old wallet')
+	from subprocess import check_call
+	try:
+		check_call(['wipe','-s',ss_in.infile.name])
+	except:
+		msg('WARNING: wipe failed, using regular file delete instead')
+		os.unlink(ss_in.infile.name)
+elif invoked_as == 'gen' and not find_file_in_dir(Wallet,g.data_dir) \
+	and not opt.stdout and keypress_confirm(m3,default_yes=True):
+	ss_out.write_to_file(outdir=g.data_dir)
+else:
+	ss_out.write_to_file()
+
+if invoked_as == 'passchg':
+	if ss_out.ssdata.passwd == ss_in.ssdata.passwd:
+		msg('New and old passphrases are the same')
+	else:
+		msg('Wallet passphrase has changed')
+	if ss_out.ssdata.hash_preset != ss_in.ssdata.hash_preset:
+		msg("Hash preset has been changed to '{}'".format(ss_out.ssdata.hash_preset))

mmgen/opts.py

@@ -19,7 +19,7 @@
 """
 opts.py:  MMGen-specific options processing after generic processing by share.Opts
 """
-import sys
+import sys,os
 
 class opt(object): pass
 
@@ -41,16 +41,13 @@ with brainwallets.  For a brainwallet passphrase to generate the correct
 seed, the same seed length and hash preset parameters must always be used.
 """.strip()
 
-def usage():
-	Msg('USAGE: %s %s' % (g.prog_name, usage_txt))
-	sys.exit(2)
-
-def print_version_info():
-	Msg("""
+version_info = """
 {pgnm_uc} version {g.version}
 Part of the {pnm} suite, a Bitcoin cold-storage solution for the command line.
 Copyright (C) {g.Cdates} {g.author} {g.email}
-""".format(pnm=g.proj_name, g=g, pgnm_uc=g.prog_name.upper()).strip())
+""".format(pnm=g.proj_name, g=g, pgnm_uc=g.prog_name.upper()).strip()
+
+def usage(): Die(2,'USAGE: %s %s' % (g.prog_name, usage_txt))
 
 def die_on_incompatible_opts(incompat_list):
 	for group in incompat_list:
@@ -58,35 +55,6 @@ def die_on_incompatible_opts(incompat_list):
 		if len(bad) > 1:
 			die(1,'Conflicting options: %s' % ', '.join([fmt_opt(b) for b in bad]))
 
-# TODO - delete
-# def _typeconvert_from_dfl(key):
-#
-# 	global opt
-#
-# 	gval = g.__dict__[key]
-# 	uval = opt.__dict__[key]
-# 	gtype = type(gval)
-#
-# 	try:
-# 		setattr(opt,key,gtype(uval))
-# 	except:
-# 		d = {
-# 			'int':   'an integer',
-# 			'str':   'a string',
-# 			'float': 'a float',
-# 			'bool':  'a boolean value',
-# 		}
-# 		die(1, "'%s': invalid parameter for '--%s' option (not %s)" % (
-# 			uval,
-# 			key.replace('_','-'),
-# 			d[gtype.__name__]
-# 		))
-#
-# 	if g.debug:
-# 		Msg('Opt overriden by user:\n    %-18s: %s' % (
-# 				key, ('%s -> %s' % (gval,uval))
-# 			))
-#
 def fmt_opt(o): return '--' + o.replace('_','-')
 
 def _show_hash_presets():
@@ -98,34 +66,115 @@ def _show_hash_presets():
 	msg('N = memory usage (power of two), p = iterations (rounds)')
 
 common_opts_data = """
---, --color=b      Set 'b' to '0' to disable color output, '1' to enable
---, --no-license   Suppress the GPL license prompt
---, --rpc-host=h   Communicate with bitcoind running on host 'h'
---, --testnet      Use testnet instead of mainnet
-"""
-
-def init(opts_data,add_opts=[],opt_filter=None):
+--, --color=c       Set to '0' to disable color output, '1' to enable
+--, --data-dir=d    Specify the location of {pnm}'s data directory
+--, --no-license    Suppress the GPL license prompt
+--, --rpc-host=h    Communicate with bitcoind running on host 'h'
+--, --skip-cfg-file Skip reading the configuration file
+--, --testnet       Use testnet instead of mainnet
+--, --version       Print version information and exit
+""".format(pnm=g.proj_name)
+
+def opt_preproc_debug(short_opts,long_opts,skipped_opts,uopts,args):
+	d = (
+		('Cmdline',            ' '.join(sys.argv)),
+		('Short opts',         short_opts),
+		('Long opts',          long_opts),
+		('Skipped opts',       skipped_opts),
+		('User-selected opts', uopts),
+		('Cmd args',           args),
+	)
+	Msg('\n=== opts.py debug ===')
+	for e in d: Msg('    {:<20}: {}'.format(*e))
+
+def opt_postproc_debug():
+	opt.verbose,opt.quiet = True,None
+	a = [k for k in dir(opt) if k[:2] != '__' and getattr(opt,k) != None]
+	b = [k for k in dir(opt) if k[:2] != '__' and getattr(opt,k) == None]
+	Msg('    Opts after processing:')
+	for k in a:
+		v = getattr(opt,k)
+		Msg('        %-18s: %-6s [%s]' % (k,v,type(v).__name__))
+	Msg("    Opts set to 'None':")
+	Msg('        %s\n' % '\n        '.join(b))
+	Msg('    Global vars:')
+	for e in [d for d in dir(g) if d[:2] != '__']:
+		Msg('        {:<20}: {}'.format(e, getattr(g,e)))
+	Msg('\n=== end opts.py debug ===')
+
+def opt_postproc_actions():
+	from mmgen.term import set_terminal_vars
+	set_terminal_vars()
+	# testnet data_dir differs from data_dir_root, so check or create
+	from mmgen.util import msg,die,check_or_create_dir
+	check_or_create_dir(g.data_dir) # dies on error
+
+def	set_data_dir_root():
+
+	g.data_dir_root = os.path.normpath(os.path.expanduser(opt.data_dir)) if opt.data_dir else \
+		(os.path.join(g.home_dir,'Application Data',g.proj_name),
+			os.path.join(g.home_dir,'.'+g.proj_name.lower()))[bool(os.getenv('HOME'))]
+
+	# mainnet and testnet share cfg file, as with Core
+	g.cfg_file = os.path.join(g.data_dir_root,'{}.cfg'.format(g.proj_name.lower()))
+
+def get_data_from_config_file():
+	from mmgen.util import msg,die,check_or_create_dir
+	check_or_create_dir(g.data_dir_root) # dies on error
+
+	# https://wiki.debian.org/Python:
+	#   Debian (Ubuntu) sys.prefix is '/usr' rather than '/usr/local, so add 'local'
+	# TODO - test for Windows
+	# This must match the configuration in setup.py
+	data = u''
+	try:
+		with open(g.cfg_file,'rb') as f: data = f.read().decode('utf8')
+	except:
+		cfg_template = os.path.join(*([sys.prefix]
+					+ ([''],['local','share'])[bool(os.getenv('HOME'))]
+					+ [g.proj_name.lower(),os.path.basename(g.cfg_file)]))
+		try:
+			with open(cfg_template,'rb') as f: template_data = f.read()
+		except:
+			msg("WARNING: configuration template not found at '{}'".format(cfg_template))
+		else:
+			try:
+				with open(g.cfg_file,'wb') as f: f.write(template_data)
+				os.chmod(g.cfg_file,0600)
+			except:
+				die(2,"ERROR: unable to write to datadir '{}'".format(g.data_dir))
+	return data
+
+def override_from_cfg_file(cfg_data):
+	from mmgen.util import die,strip_comments,set_for_type
+	import re
+	for n,l in enumerate(cfg_data.splitlines(),1): # DOS-safe
+		l = strip_comments(l)
+		if l == '': continue
+		m = re.match(r'(\w+)\s+(\S+)$',l)
+		if not m: die(2,"Parse error in file '{}', line {}".format(g.cfg_file,n))
+		name,val = m.groups()
+		if name in g.cfg_file_opts:
+			setattr(g,name,set_for_type(val,getattr(g,name),name,src=g.cfg_file))
+		else:
+			die(2,"'{}': unrecognized option in '{}'".format(name,g.cfg_file))
 
-	if len(sys.argv) == 2 and sys.argv[1] == '--version':
-		print_version_info()
-		sys.exit()
+def override_from_env():
+	from mmgen.util import set_for_type
+	for name in g.env_opts:
+		idx,invert_bool = ((6,False),(14,True))[name[:14]=='MMGEN_DISABLE_']
+		val = os.getenv(name) # os.getenv() returns None if env var is unset
+		if val: # exclude empty string values too
+			gname = name[idx:].lower()
+			setattr(g,gname,set_for_type(val,getattr(g,gname),name,invert_bool))
 
+def init(opts_data,add_opts=[],opt_filter=None):
 	opts_data['long_options'] = common_opts_data
 
 	uopts,args,short_opts,long_opts,skipped_opts = \
 		mmgen.share.Opts.parse_opts(sys.argv,opts_data,opt_filter=opt_filter)
 
-	if g.debug:
-		d = (
-			('Cmdline',            ' '.join(sys.argv)),
-			('Short opts',         short_opts),
-			('Long opts',          long_opts),
-			('Skipped opts',       skipped_opts),
-			('User-selected opts', uopts),
-			('Cmd args',           args),
-		)
-		Msg('\n=== opts.py debug ===')
-		for e in d: Msg('    {:<20}: {}'.format(*e))
+	if g.debug: opt_preproc_debug(short_opts,long_opts,skipped_opts,uopts,args)
 
 	# Save this for usage()
 	global usage_txt
@@ -137,15 +186,30 @@ def init(opts_data,add_opts=[],opt_filter=None):
 		if k in opts_data: del opts_data[k]
 
 	# Transfer uopts into opt, setting program's opts + required opts to None if not set by user
-	for o in [s.rstrip('=') for s in long_opts] + \
-			g.required_opts + add_opts + skipped_opts + g.common_opts:
+	for o in tuple([s.rstrip('=') for s in long_opts] + add_opts + skipped_opts) + \
+				g.required_opts + g.common_opts:
 		setattr(opt,o,uopts[o] if o in uopts else None)
 
+	if opt.version: Die(0,version_info)
+
+	# === Interaction with global vars begins here ===
+
+	# cfg file is in g.data_dir_root, wallet and other data are in g.data_dir
+	# Must set g.data_dir_root and g.cfg_file from cmdline before processing cfg file
+	set_data_dir_root()
+	if not opt.skip_cfg_file:
+		cfg_data = get_data_from_config_file()
+		override_from_cfg_file(cfg_data)
+	override_from_env()
+
 	# User opt sets global var - do these here, before opt is set from g.global_sets_opt
 	for k in g.common_opts:
 		val = getattr(opt,k)
 		if val != None: setattr(g,k,set_for_type(val,getattr(g,k),'--'+k))
 
+#	Global vars are now final, including g.testnet, so we can set g.data_dir
+	g.data_dir=os.path.normpath(os.path.join(g.data_dir_root,('',g.testnet_name)[g.testnet]))
+
 	# If user opt is set, convert its type based on value in mmgen.globalvars (g)
 	# If unset, set it to default value in mmgen.globalvars (g)
 	setattr(opt,'set_by_user',[])
@@ -165,21 +229,15 @@ def init(opts_data,add_opts=[],opt_filter=None):
 		_show_hash_presets()
 		sys.exit()
 
-	if g.debug:
-		opt.verbose = True
-		a = [k for k in dir(opt) if k[:2] != '__' and getattr(opt,k) != None]
-		b = [k for k in dir(opt) if k[:2] != '__' and getattr(opt,k) == None]
-		Msg('    Opts after processing:')
-		for k in a:
-			v = getattr(opt,k)
-			Msg('        %-18s: %-6s [%s]' % (k,v,type(v).__name__))
-		Msg("    Opts set to 'None':")
-		Msg('        %s\n' % '\n        '.join(b))
+	if g.debug: opt_postproc_debug()
+
+	if opt.verbose: opt.quiet = None
 
 	die_on_incompatible_opts(g.incompatible_opts)
 
-	return args
+	opt_postproc_actions()
 
+	return args
 
 def check_opts(usr_opts):       # Returns false if any check fails
 
@@ -271,7 +329,6 @@ def check_opts(usr_opts):       # Returns false if any check fails
 				check_infile(a[0],blkdev_ok=True)
 				key2 = 'in_fmt'
 			else:
-				import os
 				try: os.stat(a[0])
 				except:
 					b = os.path.dirname(a[0])

mmgen/rpc.py

@@ -81,7 +81,6 @@ class BitcoinRPCConnection(object):
 
 		dmsg('=== rpc.py debug ===')
 		dmsg('    RPC POST data ==> %s\n' % p)
-
 		caller = self
 		class MyJSONEncoder(json.JSONEncoder):
 			def default(self, obj):
@@ -94,16 +93,19 @@ class BitcoinRPCConnection(object):
 # 			dump = json.dumps(p,cls=MyJSONEncoder,ensure_ascii=False)
 # 			print(dump)
 
+		dmsg('    RPC AUTHORIZATION data ==> [Basic {}]\n'.format(base64.b64encode(self.auth_str)))
 		try:
 			hc.request('POST', '/', json.dumps(p,cls=MyJSONEncoder), {
 				'Host': self.host,
-				'Authorization': 'Basic ' + base64.b64encode(self.auth_str)
+				'Authorization': 'Basic {}'.format(base64.b64encode(self.auth_str))
 			})
 		except Exception as e:
 			return die_maybe(None,2,'%s\nUnable to connect to bitcoind' % e)
 
 		r = hc.getresponse() # returns HTTPResponse instance
 
+		dmsg('    RPC GETRESPONSE data ==> %s\n' % r.__dict__)
+
 		if r.status != 200:
 			msgred('RPC Error: {} {}'.format(r.status,r.reason))
 			e1 = r.read()

mmgen/seed.py

@@ -204,15 +204,15 @@ class SeedSource(MMGenObject):
 		self._format()
 		return self.fmt_data
 
-	def write_to_file(self):
+	def write_to_file(self,outdir='',desc=''):
 		self._format()
 		kwargs = {
-			'desc':     self.desc,
+			'desc':     desc or self.desc,
 			'ask_tty':  self.ask_tty,
 			'no_tty':   self.no_tty,
 			'binary':   self.file_mode == 'binary'
 		}
-		write_data_to_file(self._filename(),self.fmt_data,**kwargs)
+		write_data_to_file(os.path.join(outdir,self._filename()),self.fmt_data,**kwargs)
 
 class SeedSourceUnenc(SeedSource):
 
@@ -661,7 +661,7 @@ class Wallet (SeedSourceEnc):
 	def _decrypt(self):
 		d = self.ssdata
 		# Needed for multiple transactions with {}-txsign
-		suf = ('',self.infile.name)[bool(opt.quiet)]
+		suf = ('',os.path.basename(self.infile.name))[bool(opt.quiet)]
 		self._get_passphrase(desc_suf=suf)
 		key = make_key(d.passwd, d.salt, d.hash_preset)
 		ret = decrypt_seed(d.enc_seed, key, d.seed_id, d.key_id)

mmgen/term.py

@@ -23,6 +23,17 @@ term.py:  Terminal-handling routines for the MMGen suite
 import os,struct
 from mmgen.common import *
 
+try:
+	import tty,termios
+	from select import select
+	_platform = 'linux'
+except:
+	try:
+		import msvcrt,time
+		_platform = 'win'
+	except:
+		die(2,'Unable to set terminal mode')
+
 def _kb_hold_protect_unix():
 
 	fd = sys.stdin.fileno()
@@ -64,7 +75,6 @@ def _get_keypress_unix(prompt='',immed_chars='',prehold_protect=True):
 	termios.tcsetattr(fd, termios.TCSADRAIN, old)
 	return ch
 
-
 def _get_keypress_unix_raw(prompt='',immed_chars='',prehold_protect=None):
 
 	msg_r(prompt)
@@ -155,7 +165,6 @@ def _get_terminal_size_linux():
 
 	return int(cr[1]), int(cr[0])
 
-
 def _get_terminal_size_mswin():
 	try:
 		from ctypes import windll,create_string_buffer
@@ -176,67 +185,15 @@ def _get_terminal_size_mswin():
 
 def mswin_dummy_flush(fd,termconst): pass
 
-try:
-	import tty,termios
-	from select import select
-	if g.hold_protect:
-		get_char = _get_keypress_unix
-		kb_hold_protect = _kb_hold_protect_unix
+def set_terminal_vars():
+	global get_char,kb_hold_protect,get_terminal_size
+	if _platform == 'linux':
+		get_char = (_get_keypress_unix_raw,_get_keypress_unix)[g.hold_protect]
+		kb_hold_protect = (_kb_hold_protect_unix_raw,_kb_hold_protect_unix)[g.hold_protect]
+		get_terminal_size = _get_terminal_size_linux
+		myflush = termios.tcflush   # call: myflush(sys.stdin, termios.TCIOFLUSH)
 	else:
-		get_char = _get_keypress_unix_raw
-		kb_hold_protect = _kb_hold_protect_unix_raw
-	get_terminal_size = _get_terminal_size_linux
-	myflush = termios.tcflush
-# call: myflush(sys.stdin, termios.TCIOFLUSH)
-except:
-	try:
-		import msvcrt,time
-		if g.hold_protect:
-			get_char = _get_keypress_mswin
-			kb_hold_protect = _kb_hold_protect_mswin
-		else:
-			get_char = _get_keypress_mswin_raw
-			kb_hold_protect = _kb_hold_protect_mswin_raw
+		get_char = (_get_keypress_mswin_raw,_get_keypress_mswin)[g.hold_protect]
+		kb_hold_protect = (_kb_hold_protect_mswin_raw,_kb_hold_protect_mswin)[g.hold_protect]
 		get_terminal_size = _get_terminal_size_mswin
 		myflush = mswin_dummy_flush
-	except:
-		msg('Unable to set terminal mode')
-		sys.exit(2)
-
-def do_pager(text):
-
-	pagers = ['less','more']
-	shell = False
-
-	from os import environ
-
-# Hack for MS Windows command line (i.e. non CygWin) environment
-# When 'shell' is true, Windows aborts the calling program if executable
-# not found.
-# When 'shell' is false, an exception is raised, invoking the fallback
-# 'print' instead of the pager.
-# We risk assuming that 'more' will always be available on a stock
-# Windows installation.
-	if g.platform == 'win':
-		if 'HOME' not in environ: # native Windows terminal
-			shell = True
-			pagers = ['more']
-		else:                     # MSYS
-			environ['LESS'] = '-cR -#1' # disable buggy line chopping
-	else:
-		environ['LESS'] = '-RS -#1' # raw, chop, scroll right 1 char
-
-	if 'PAGER' in environ and environ['PAGER'] != pagers[0]:
-		pagers = [environ['PAGER']] + pagers
-
-	for pager in pagers:
-		end = ('\n(end of text)\n','')[pager=='less']
-		try:
-			from subprocess import Popen,PIPE,STDOUT
-			p = Popen([pager], stdin=PIPE, shell=shell)
-		except: pass
-		else:
-			p.communicate(text+end+'\n')
-			msg_r('\r')
-			break
-	else: Msg(text+end)

mmgen/test.py

@@ -40,11 +40,11 @@ def getrandstr(num_chars,no_space=False):
 	if no_space: n,m = 94,33
 	return ''.join([chr(ord(i)%n+m) for i in list(os.urandom(num_chars))])
 
-def mk_tmpdir(cfg):
-	try: os.mkdir(cfg['tmpdir'],0755)
+def mk_tmpdir(d):
+	try: os.mkdir(d,0755)
 	except OSError as e:
 		if e.errno != 17: raise
-	else: msg("Created directory '%s'" % cfg['tmpdir'])
+	else: msg("Created directory '%s'" % d)
 
 def mk_tmpdir_path(path,cfg):
 	try:

mmgen/tw.py

@@ -22,7 +22,6 @@ tw: Tracking wallet methods for the MMGen suite
 
 from mmgen.common import *
 from mmgen.obj import *
-from mmgen.term import get_char
 from mmgen.tx import is_mmgen_id
 
 CUR_HOME,ERASE_ALL = '\033[H','\033[0J'
@@ -237,7 +236,6 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 							return n,s
 
 	def view_and_sort(self):
-		from mmgen.term import do_pager
 		prompt = """
 Sort options: [t]xid, [a]mount, a[d]dress, [A]ge, [r]everse, [M]mgen addr
 Display options: show [D]ays, [g]roup, show [m]mgen addr, r[e]draw screen
@@ -245,6 +243,7 @@ Display options: show [D]ays, [g]roup, show [m]mgen addr, r[e]draw screen
 		self.display()
 		msg(prompt)
 
+		from mmgen.term import get_char
 		p = "'q'=quit view, 'p'=print to file, 'v'=pager view, 'w'=wide view, 'l'=add label:\b"
 		while True:
 			reply = get_char(p, immed_chars='atDdAMrgmeqpvw')

mmgen/tx.py

@@ -25,7 +25,6 @@ from stat import *
 from binascii import unhexlify
 from mmgen.common import *
 from mmgen.obj import *
-from mmgen.term import do_pager
 
 def is_mmgen_seed_id(s): return SeedID(sid=s,on_fail='silent')
 def is_mmgen_idx(s):     return AddrIdx(s,on_fail='silent')

mmgen/util.py

@@ -124,6 +124,15 @@ def parse_nbytes(nbytes):
 
 	die(1,"'%s': invalid byte specifier" % nbytes)
 
+def check_or_create_dir(path):
+	try:
+		os.listdir(path)
+	except:
+		try:
+			os.makedirs(path,0700)
+		except:
+			die(2,"ERROR: unable to read or create path '{}'".format(path))
+
 from mmgen.opts import opt
 
 def qmsg(s,alt=False):
@@ -383,6 +392,26 @@ def check_outdir(f):
 def make_full_path(outdir,outfile):
 	return os.path.normpath(os.path.join(outdir, os.path.basename(outfile)))
 
+def get_seed_file(cmd_args,nargs,invoked_as=None):
+	from mmgen.filename import find_file_in_dir
+	from mmgen.seed import Wallet
+	wf = find_file_in_dir(Wallet,g.data_dir)
+
+	wd_from_opt = bool(opt.hidden_incog_input_params or opt.in_fmt) # have wallet data from opt?
+
+	import mmgen.opts as opts
+	if len(cmd_args) + (wd_from_opt or bool(wf)) < nargs:
+		opts.usage()
+	elif len(cmd_args) > nargs:
+		opts.usage()
+	elif len(cmd_args) == nargs and wf and invoked_as != 'gen':
+		msg('Warning: overriding wallet in data directory with user-supplied wallet')
+
+	if cmd_args or wf:
+		check_infile(cmd_args[0] if cmd_args else wf)
+
+	return cmd_args[0] if cmd_args else (wf,None)[wd_from_opt]
+
 def get_new_passphrase(desc,passchg=False):
 
 	w = '{}passphrase for {}'.format(('','new ')[bool(passchg)], desc)
@@ -405,18 +434,14 @@ def get_new_passphrase(desc,passchg=False):
 	if pw == '': qmsg('WARNING: Empty passphrase')
 	return pw
 
-
-def confirm_or_exit(message, question, expect='YES'):
-
+def confirm_or_exit(message,question,expect='YES',exit_msg='Exiting at user request'):
 	m = message.strip()
 	if m: msg(m)
-
 	a = question+'  ' if question[0].isupper() else \
 			'Are you sure you want to %s?\n' % question
 	b = "Type uppercase '%s' to confirm: " % expect
-
 	if my_raw_input(a+b).strip() != expect:
-		die(2,'Exiting at user request')
+		die(2,exit_msg)
 
 
 # New function
@@ -469,7 +494,8 @@ def write_data_to_file(
 
 		sys.stdout.write(data)
 	else:
-		if opt.outdir: outfile = make_full_path(opt.outdir,outfile)
+		if opt.outdir and not os.path.isabs(outfile):
+			outfile = make_full_path(opt.outdir,outfile)
 
 		if ask_write:
 			if not ask_write_prompt: ask_write_prompt = 'Save %s?' % desc
@@ -620,6 +646,42 @@ def prompt_and_get_char(prompt,chars,enter_ok=False,verbose=False):
 		if verbose: msg('\nInvalid reply')
 		else: msg_r('\r')
 
+def do_pager(text):
+
+	pagers = ['less','more']
+	shell = False
+
+# Hack for MS Windows command line (i.e. non CygWin) environment
+# When 'shell' is true, Windows aborts the calling program if executable
+# not found.
+# When 'shell' is false, an exception is raised, invoking the fallback
+# 'print' instead of the pager.
+# We risk assuming that 'more' will always be available on a stock
+# Windows installation.
+	if g.platform == 'win':
+		if 'HOME' not in os.environ: # native Windows terminal
+			shell = True
+			pagers = ['more']
+		else:                     # MSYS
+			os.environ['LESS'] = '-cR -#1' # disable buggy line chopping
+	else:
+		os.environ['LESS'] = '-RS -#1' # raw, chop, scroll right 1 char
+
+	if 'PAGER' in os.environ and os.environ['PAGER'] != pagers[0]:
+		pagers = [os.environ['PAGER']] + pagers
+
+	for pager in pagers:
+		end = ('\n(end of text)\n','')[pager=='less']
+		try:
+			from subprocess import Popen,PIPE,STDOUT
+			p = Popen([pager], stdin=PIPE, shell=shell)
+		except: pass
+		else:
+			p.communicate(text+end+'\n')
+			msg_r('\r')
+			break
+	else: Msg(text+end)
+
 def do_license_msg(immed=False):
 
 	if opt.quiet or g.no_license: return
@@ -630,7 +692,7 @@ def do_license_msg(immed=False):
 	msg(gpl.warning)
 	prompt = '%s ' % p.strip()
 
-	from mmgen.term import get_char,do_pager
+	from mmgen.term import get_char
 
 	while True:
 		reply = get_char(prompt, immed_chars=('','wc')[bool(immed)])

scripts/deinstall.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-CMD='rm -rf /usr/local/bin/mmgen-* /usr/local/lib/python2.7/dist-packages/mmgen*'
+CMD='rm -rf /usr/local/share/mmgen /usr/local/bin/mmgen-* /usr/local/lib/python2.7/dist-packages/mmgen*'
 
 if [ "$EUID" = 0 ]; then
 	set -x; $CMD

setup.py

@@ -37,7 +37,7 @@ class my_build_ext(build_ext):
 class my_install_data(install_data):
 	def run(self):
 		for f in 'mmgen.cfg','mnemonic.py','mn_wordlist.c':
-			os.chmod('data_files/'+f,0644)
+			os.chmod(os.path.join('data_files',f),0644)
 		install_data.run(self)
 
 module1 = Extension(
@@ -49,6 +49,11 @@ module1 = Extension(
 	include_dirs = ['/usr/local/include'],
 	)
 
+cmd_overrides = {
+	'linux': { 'build_ext': my_build_ext, 'install_data': my_install_data },
+	'win':   { 'install_data': my_install_data }
+}
+
 from mmgen.globalvars import g
 setup(
 		name         = 'mmgen',
@@ -60,9 +65,9 @@ setup(
 		license      = 'GNU GPL v3',
 		platforms    = 'Linux, MS Windows, Raspberry PI',
 		keywords     = 'Bitcoin, wallet, cold storage, offline storage, open-source, command-line, Python, Bitcoin Core, bitcoind, hd, deterministic, hierarchical, secure, anonymous',
-		cmdclass     = { 'build_ext': my_build_ext, 'install_data': my_install_data },
+		cmdclass     = cmd_overrides[g.platform],
 		# disable building of secp256k1 extension module on Windows
-		ext_modules = [module1] if sys.platform[:5] == 'linux' else [],
+		ext_modules = ([],[module1])[g.platform=='linux'],
 		data_files = [('share/mmgen', [
 				'data_files/mmgen.cfg',     # source files must have 0644 mode
 				'data_files/mn_wordlist.c',

test/gentest.py

@@ -62,6 +62,9 @@ EXAMPLES:
     (compare addrs generated with secp256k1 library to bitcoind wallet dump)
 """.format(prog='gentest.py',pnm=g.proj_name,snum=rounds)
 }
+
+sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
+
 cmd_args = opts.init(opts_data,add_opts=['exact_output'])
 
 if not 1 <= len(cmd_args) <= 2: opts.usage()
@@ -167,6 +170,8 @@ elif a and dump:
 	for n,[wif,a_addr] in enumerate(dump,1):
 		msg_r('\rKey %s/%s ' % (n,len(dump)))
 		sec = wif2hex(wif)
+		if sec == False:
+			die(2,'\nInvalid {}net WIF address in dump file: {}'.format(('main','test')[g.testnet],wif))
 		compressed = wif[0] != ('5','9')[g.testnet]
 		b_addr = gen_a(sec,compressed)
 		if a_addr != b_addr:

test/test.py

@@ -46,48 +46,12 @@ sys.path.__setitem__(0,os.path.abspath(os.curdir))
 from mmgen.common import *
 from mmgen.test import *
 
+g.quiet = False # if 'quiet' was set in config file, disable here
+os.environ['MMGEN_QUIET'] = '0' # and for the spawned scripts
+
 tb_cmd = 'scripts/traceback.py'
 log_file = 'test.py_log'
 
-opts_data = {
-#	'sets': [('non_interactive',bool,'verbose',None)],
-	'desc': 'Test suite for the MMGen suite',
-	'usage':'[options] [command(s) or metacommand(s)]',
-	'options': """
--h, --help          Print this help message.
--b, --buf-keypress  Use buffered keypresses as with real human input.
--d, --debug-scripts Turn on debugging output in executed scripts.
--D, --direct-exec   Bypass pexpect and execute a command directly (for
-                    debugging only).
--e, --exact-output  Show the exact output of the MMGen script(s) being run.
--l, --list-cmds     List and describe the commands in the test suite.
--L, --log           Log commands to file {lf}
--n, --names         Display command names instead of descriptions.
--I, --non-interactive Non-interactive operation (MS Windows mode)
--p, --pause         Pause between tests, resuming on keypress.
--P, --profile       Record the execution time of each script.
--q, --quiet         Produce minimal output.  Suppress dependency info.
--r, --resume=c      Resume at command 'c' after interrupted run
--s, --system        Test scripts and modules installed on system rather
-                    than those in the repo root.
--S, --skip-deps     Skip dependency checking for command
--u, --usr-random    Get random data interactively from user
---, --testnet       Run on testnet rather than mainnet
--t, --traceback     Run the command inside the '{tb_cmd}' script.
--v, --verbose       Produce more verbose output.
-""".format(tb_cmd=tb_cmd,lf=log_file),
-	'notes': """
-
-If no command is given, the whole suite of tests is run.
-"""
-}
-
-cmd_args = opts.init(opts_data)
-
-tn_desc = ('','.testnet')[g.testnet]
-
-start_mscolor()
-
 scripts = (
 	'addrgen', 'addrimport', 'keygen',
 	'passchg', 'tool',
@@ -128,7 +92,88 @@ tool_enc_passwd = "Scrypt it, don't hash it!"
 sample_text = \
 	'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks\n'
 
+# Laggy flash media cause pexpect to crash, so create a temporary directory
+# under '/dev/shm' and put datadir and temp files here.
+if g.platform == 'win':
+	data_dir = os.path.join('test','data_dir')
+else:
+	d,pfx = '/dev/shm','mmgen-test-'
+	try:
+		import subprocess
+		subprocess.call('rm -rf %s/%s*'%(d,pfx),shell=True)
+	except Exception as e:
+		die(2,'Unable to delete directory tree %s/%s* (%s)'%(d,pfx,e))
+	try:
+		import tempfile
+		shm_dir = tempfile.mkdtemp('',pfx,d)
+	except Exception as e:
+		die(2,'Unable to create temporary directory in %s (%s)'%(d,e))
+	data_dir = os.path.join(shm_dir,'data_dir')
+
+os.mkdir(data_dir,0755)
+
+opts_data = {
+#	'sets': [('non_interactive',bool,'verbose',None)],
+	'desc': 'Test suite for the MMGen suite',
+	'usage':'[options] [command(s) or metacommand(s)]',
+	'options': """
+-h, --help          Print this help message
+--, --longhelp      Print help message for long options (common options)
+-b, --buf-keypress  Use buffered keypresses as with real human input
+-d, --debug-scripts Turn on debugging output in executed scripts
+-D, --direct-exec   Bypass pexpect and execute a command directly (for
+                    debugging only)
+-e, --exact-output  Show the exact output of the MMGen script(s) being run
+-l, --list-cmds     List and describe the commands in the test suite
+-L, --log           Log commands to file {lf}
+-n, --names         Display command names instead of descriptions
+-I, --non-interactive Non-interactive operation (MS Windows mode)
+-p, --pause         Pause between tests, resuming on keypress
+-P, --profile       Record the execution time of each script
+-q, --quiet         Produce minimal output.  Suppress dependency info
+-r, --resume=c      Resume at command 'c' after interrupted run
+-s, --system        Test scripts and modules installed on system rather
+                    than those in the repo root
+-S, --skip-deps     Skip dependency checking for command
+-u, --usr-random    Get random data interactively from user
+--, --testnet       Run on testnet rather than mainnet
+-t, --traceback     Run the command inside the '{tb_cmd}' script
+-v, --verbose       Produce more verbose output
+""".format(tb_cmd=tb_cmd,lf=log_file),
+	'notes': """
+
+If no command is given, the whole suite of tests is run.
+"""
+}
+
+sys.argv = [sys.argv[0]] + ['--data-dir',data_dir] + sys.argv[1:]
+
+cmd_args = opts.init(opts_data)
+
+tn_desc = ('','.testnet')[g.testnet]
+
 cfgs = {
+	'15': {
+		'tmpdir':        os.path.join('test','tmp15'),
+		'wpasswd':       'Dorian',
+		'kapasswd':      'Grok the blockchain',
+		'addr_idx_list': '12,99,5-10,5,12', # 8 addresses
+		'dep_generators':  {
+			pwfile:        'walletgen_dfl_wallet',
+			'addrs':       'addrgen_dfl_wallet',
+			'rawtx':       'txcreate_dfl_wallet',
+			'sigtx':       'txsign_dfl_wallet',
+			'mmseed':      'export_seed_dfl_wallet',
+		},
+	},
+	'16': {
+		'tmpdir':        os.path.join('test','tmp16'),
+		'wpasswd':       'My changed password',
+		'hash_preset':   '2',
+		'dep_generators': {
+			pwfile:        'passchg_dfl_wallet',
+		},
+	},
 	'1': {
 		'tmpdir':        os.path.join('test','tmp1'),
 		'wpasswd':       'Dorian',
@@ -307,6 +352,8 @@ cfgs = {
 	},
 }
 
+start_mscolor()
+
 from copy import deepcopy
 for a,b in ('6','11'),('7','12'),('8','13'):
 	cfgs[b] = deepcopy(cfgs[a])
@@ -323,6 +370,14 @@ cmd_group['help'] = OrderedDict([
 ])
 
 cmd_group['main'] = OrderedDict([
+	['walletgen_dfl_wallet', (15,'wallet generation (default wallet)',[[[],15]],15)],
+	['addrgen_dfl_wallet',(15,'address generation (default wallet)',[[[pwfile],15]],15)],
+	['txcreate_dfl_wallet',(15,'transaction creation (default wallet)',[[['addrs'],15]],15)],
+	['txsign_dfl_wallet',(15,'transaction signing (default wallet)',[[['rawtx',pwfile],15]],15)],
+	['export_seed_dfl_wallet',(15,'seed export to mmseed format (default wallet)',[[[pwfile],15]])],
+	['passchg_dfl_wallet',(16,'password, label and hash preset change (default wallet)',[[[pwfile],15]],15)],
+	['walletchk_newpass_dfl_wallet',(16,'wallet check with new pw, label and hash preset',[[[pwfile],16]],15)],
+	['delete_dfl_wallet',(15,'delete default wallet',[[[pwfile],15]],15)],
 	['walletgen',       (1,'wallet generation',        [[[],1]],1)],
 #	['walletchk',       (1,'wallet check',             [[['mmdat'],1]])],
 	['passchg',         (5,'password, label and hash preset change',[[['mmdat',pwfile],1]],1)],
@@ -510,6 +565,7 @@ add_spawn_args = ' '.join(['{} {}'.format(
 	'--'+k.replace('_','-'),
 	getattr(opt,k) if getattr(opt,k) != True else ''
 	) for k in 'testnet','rpc_host' if getattr(opt,k)]).split()
+add_spawn_args += ['--data-dir',data_dir]
 
 if opt.profile: opt.names = True
 if opt.resume: opt.skip_deps = True
@@ -1073,7 +1129,7 @@ class MMGenTestSuite(object):
 
 	def longhelpscreens(self,name): self.helpscreens(name,arg='--longhelp')
 
-	def walletgen(self,name,seed_len=None):
+	def walletgen(self,name,seed_len=None,make_dfl_rsp='n'):
 		write_to_tmpfile(cfg,pwfile,cfg['wpasswd']+'\n')
 		add_args = ([usr_rand_arg],
 			['-q','-r0','-L','NI Wallet','-P',get_tmpfile_fn(cfg,pwfile)])[bool(ni)]
@@ -1085,9 +1141,13 @@ class MMGenTestSuite(object):
 		t.usr_rand(usr_rand_chars)
 		t.passphrase_new('new MMGen wallet',cfg['wpasswd'])
 		t.label()
+		t.expect('move it to the data directory? (Y/n): ',make_dfl_rsp)
 		t.written_to_file('MMGen wallet')
 		ok()
 
+	def walletgen_dfl_wallet(self,name,seed_len=None):
+		self.walletgen(name,seed_len=seed_len,make_dfl_rsp='y')
+
 	def brainwalletgen_ref(self,name):
 		sl_arg = '-l%s' % cfg['seed_len']
 		hp_arg = '-p%s' % ref_wallet_hash_preset
@@ -1120,7 +1180,7 @@ class MMGenTestSuite(object):
 		end_silence()
 		add_args = ([usr_rand_arg],['-q','-r0','-P',pf])[bool(ni)]
 		t = MMGenExpect(name,'mmgen-passchg', add_args +
-				['-d',cfg['tmpdir'],'-p','2','-L','New Label',wf])
+				['-d',cfg['tmpdir'],'-p','2','-L','New Label'] + ([],[wf])[bool(wf)])
 		if ni: return
 		t.license()
 		t.passphrase('MMGen wallet',cfgs['1']['wpasswd'],pwtype='old')
@@ -1130,9 +1190,18 @@ class MMGenTestSuite(object):
 		t.usr_rand(usr_rand_chars)
 		t.expect_getend('Label changed to ')
 #		t.expect_getend('Key ID changed: ')
-		t.written_to_file('MMGen wallet')
+		if not wf:
+			t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
+			t.written_to_file('New wallet')
+			t.expect('Okay to WIPE 1 regular file ? (Yes/No)','Yes\n')
+			t.expect_getend('has been changed to ')
+		else:
+			t.written_to_file('MMGen wallet')
 		ok()
 
+	def passchg_dfl_wallet(self,name,pf):
+		return self.passchg(name=name,wf=None,pf=pf)
+
 	def walletchk(self,name,wf,pf,desc='MMGen wallet',
 			add_args=[],sid=None,pw=False,extra_desc=''):
 		args = ([],['-P',pf,'-q'])[bool(ni and pf)]
@@ -1148,7 +1217,7 @@ class MMGenTestSuite(object):
 				msg(grnbg('%s %s' % (m,cyan(sid))))
 			return
 		if desc != 'hidden incognito data':
-			t.expect("Getting %s from file '%s'" % (desc,wf))
+			t.expect("Getting %s from file '" % (desc))
 		if pw:
 			t.passphrase(desc,cfg['wpasswd'])
 			t.expect(
@@ -1159,13 +1228,22 @@ class MMGenTestSuite(object):
 		if sid: cmp_or_die(chk,sid)
 		else: ok()
 
-	def walletchk_newpass (self,name,wf,pf):
+	def walletchk_newpass(self,name,wf,pf):
 		return self.walletchk(name,wf,pf,pw=True)
 
+	def walletchk_newpass_dfl_wallet(self,name,pf):
+		return self.walletchk_newpass(name,wf=None,pf=pf)
+
+	def delete_dfl_wallet(self,name,pf):
+		for wf in [f for f in os.listdir(g.data_dir) if f[-6:]=='.mmdat']:
+			os.unlink(os.path.join(g.data_dir,wf))
+		MMGenExpect(name,'true')
+		ok()
+
 	def addrgen(self,name,wf,pf=None,check_ref=False):
 		add_args = ([],['-q'] + ([],['-P',pf])[bool(pf)])[ni]
 		t = MMGenExpect(name,'mmgen-addrgen', add_args +
-				['-d',cfg['tmpdir'],wf,cfg['addr_idx_list']])
+				['-d',cfg['tmpdir']] + ([],[wf])[bool(wf)] + [cfg['addr_idx_list']])
 		if ni: return
 		t.license()
 		t.passphrase('MMGen wallet',cfg['wpasswd'])
@@ -1177,6 +1255,9 @@ class MMGenTestSuite(object):
 		t.written_to_file('Addresses',oo=True)
 		ok()
 
+	def addrgen_dfl_wallet(self,name,wf,pf=None,check_ref=False):
+		return self.addrgen(name,wf=None,pf=pf,check_ref=check_ref)
+
 	def refaddrgen(self,name,wf,pf):
 		d = ' (%s-bit seed)' % cfg['seed_len']
 		self.addrgen(name,wf,pf=pf,check_ref=True)
@@ -1196,6 +1277,9 @@ class MMGenTestSuite(object):
 	def txcreate(self,name,addrfile):
 		self.txcreate_common(name,sources=['1'])
 
+	def txcreate_dfl_wallet(self,name,addrfile):
+		self.txcreate_common(name,sources=['15'])
+
 	def txcreate_common(self,name,sources=['1'],non_mmgen_input='',do_label=False):
 		if opt.verbose or opt.exact_output:
 			sys.stderr.write(green('Generating fake tracking wallet info\n'))
@@ -1300,7 +1384,7 @@ class MMGenTestSuite(object):
 		if ni:
 			m = '\nAnswer the interactive prompts as follows:\n  ENTER, ENTER, ENTER'
 			msg(grnbg(m))
-		t = MMGenExpect(name,'mmgen-txsign', add_args+['-d',cfg['tmpdir'],txfile,wf])
+		t = MMGenExpect(name,'mmgen-txsign', add_args+['-d',cfg['tmpdir'],txfile]+([],[wf])[bool(wf)])
 		if ni: return
 		t.license()
 		t.tx_view()
@@ -1313,6 +1397,9 @@ class MMGenTestSuite(object):
 			t.close()
 		ok()
 
+	def txsign_dfl_wallet(self,name,txfile,pf='',save=True,has_label=False):
+		return self.txsign(name,txfile,wf=None,pf=pf,save=save,has_label=has_label)
+
 	def txsend(self,name,sigfile):
 		t = MMGenExpect(name,'mmgen-txsend', ['-d',cfg['tmpdir'],sigfile])
 		t.license()
@@ -1325,7 +1412,7 @@ class MMGenTestSuite(object):
 		ok()
 
 	def walletconv_export(self,name,wf,desc,uargs=[],out_fmt='w',pw=False):
-		opts = ['-d',cfg['tmpdir'],'-o',out_fmt] + uargs + [wf]
+		opts = ['-d',cfg['tmpdir'],'-o',out_fmt] + uargs + ([],[wf])[bool(wf)]
 		t = MMGenExpect(name,'mmgen-walletconv',opts)
 		t.license()
 		t.passphrase('MMGen wallet',cfg['wpasswd'])
@@ -1355,6 +1442,9 @@ class MMGenTestSuite(object):
 		end_silence()
 		ok()
 
+	def export_seed_dfl_wallet(self,name,pw,desc='seed data',out_fmt='seed'):
+		return self.export_seed(name,wf=None,desc=desc,out_fmt=out_fmt)
+
 	def export_mnemonic(self,name,wf):
 		self.export_seed(name,wf,desc='mnemonic data',out_fmt='words')
 
@@ -1851,7 +1941,7 @@ class MMGenTestSuite(object):
 			desc=desc,sid=cfg['seed_id'],pw=pw,
 			add_args=add_args,
 			extra_desc='(check)')
-
+	# END methods
 	for k in (
 			'ref_wallet_conv',
 			'ref_mn_conv',
@@ -1881,6 +1971,20 @@ class MMGenTestSuite(object):
 
 	for k in ('walletgen','addrgen','keyaddrgen'): locals()[k+'14'] = locals()[k]
 
+# create temporary dirs
+if g.platform == 'win':
+	for cfg in sorted(cfgs):
+		mk_tmpdir(cfgs[cfg]['tmpdir'])
+else:
+	for cfg in sorted(cfgs):
+		src = os.path.join(shm_dir,cfgs[cfg]['tmpdir'].split('/')[-1])
+		mk_tmpdir(src)
+		try:
+			os.unlink(cfgs[cfg]['tmpdir'])
+		except OSError as e:
+			if e.errno != 2: raise
+		finally:
+			os.symlink(src,cfgs[cfg]['tmpdir'])
 
 # main()
 if opt.pause:
@@ -1894,25 +1998,6 @@ if opt.pause:
 start_time = int(time.time())
 ts = MMGenTestSuite()
 
-# Laggy flash media cause pexpect to crash, so read and write all temporary
-# files to volatile memory in '/dev/shm'
-if not opt.skip_deps:
-	if g.platform == 'win':
-		for cfg in sorted(cfgs): mk_tmpdir(cfgs[cfg])
-	else:
-		d,pfx = '/dev/shm','mmgen-test-'
-		try:
-			import subprocess
-			subprocess.call('rm -rf %s/%s*'%(d,pfx),shell=True)
-		except Exception as e:
-			die(2,'Unable to delete directory tree %s/%s* (%s)'%(d,pfx,e))
-		try:
-			import tempfile
-			shm_dir = tempfile.mkdtemp('',pfx,d)
-		except Exception as e:
-			die(2,'Unable to create temporary directory in %s (%s)'%(d,e))
-		for cfg in sorted(cfgs): mk_tmpdir_path(shm_dir,cfgs[cfg])
-
 try:
 	if cmd_args:
 		for arg in cmd_args:

test/tooltest.py

@@ -91,6 +91,7 @@ cmd_data = OrderedDict([
 				('addrfile_chksum', ()),
 				('getbalance',      ()),
 				('listaddresses',   ()),
+				('twview',          ()),
 				('txview',          ()),
 			])
 		}
@@ -125,6 +126,8 @@ If no command is given, the whole suite of tests is run.
 """
 }
 
+sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
+
 cmd_args = opts.init(opts_data,add_opts=['exact_output','profile'])
 add_spawn_args = ' '.join(['{} {}'.format(
 	'--'+k.replace('_','-'),
@@ -358,6 +361,8 @@ class MMGenToolTestSuite(object):
 		self.run_cmd_out(name,literal=True)
 	def listaddresses(self,name):
 		self.run_cmd_out(name,literal=True)
+	def twview(self,name):
+		self.run_cmd_out(name,literal=True)
 	def txview(self,name):
 		fn = os.path.join(cfg['refdir'],cfg['txfile'])
 		self.run_cmd_out(name,fn,literal=True)
@@ -369,7 +374,7 @@ class MMGenToolTestSuite(object):
 import time
 start_time = int(time.time())
 ts = MMGenToolTestSuite()
-mk_tmpdir(cfg)
+mk_tmpdir(cfg['tmpdir'])
 
 if cmd_args:
 	if len(cmd_args) != 1: