Require UTF-8 for brainwallet; other UTF-8 fixes

- This commit introduces a backwards incompatibility.  Users with non-UTF-8
  brainwallets (if there are any, which is very unlikely) must export them to
  another MMGen wallet format using an older version of MMGen.

doc/README.mswin

@@ -0,0 +1,10 @@
+MMGen MS Windows Notes
+
+The following MMGen features are unsupported or broken on the MSWin/MinGW platform:
+
+- Autosign (not supported)
+- Zcash z-address generation (requires libsodium)
+- Monero wallet creation/syncing* (IO stream issues with pexpect and the password prompt)
+- UTF-8 label, filename and path support (may work on versions of Windows with native UTF-8 support)
+
+*Monero address and viewkey generation works fine.

mmgen/crypto.py

@@ -111,6 +111,7 @@ def scrypt_hash_passphrase(passwd,salt,hash_preset,buflen=32):
 	# Buflen arg is for brainwallets only, which use this function to generate
 	# the seed directly.
 	N,r,p = get_hash_params(hash_preset)
+	if type(passwd) == unicode: passwd = passwd.encode('utf8')
 	return scrypt.hash(passwd,salt,2**N,r,p,buflen=buflen)
 
 def make_key(passwd,salt,hash_preset,desc='encryption key',from_what='passphrase',verbose=False):

mmgen/globalvars.py

@@ -100,14 +100,14 @@ class g(object):
 		die(1,"'{}': platform not supported by {}\n".format(sys.platform,proj_name))
 
 	if os.getenv('HOME'):                             # Linux or MSYS
-		home_dir = os.getenv('HOME')
+		home_dir = os.getenv('HOME').decode('utf8')
 	elif platform == 'win': # Windows native:
 		die(1,'$HOME not set!  {} for Windows must be run in MSYS environment'.format(proj_name))
 	else:
 		die(2,'$HOME is not set!  Unable to determine home directory')
 
 	data_dir_root,data_dir,cfg_file = None,None,None
-	daemon_data_dir = '' # set by user or protocol
+	daemon_data_dir = u'' # set by user or protocol
 
 	# User opt sets global var:
 	common_opts = (

mmgen/main.py

@@ -26,7 +26,7 @@ def launch(what):
 		try:
 			return a.decode('utf8')
 		except:
-			sys.stderr.write("Argument '{}' is not a valid UTF-8 string".format(a))
+			sys.stderr.write("Argument {!r} is not a valid UTF-8 string".format(a))
 			sys.exit(2)
 
 	import sys

mmgen/main_autosign.py

@@ -23,11 +23,11 @@ mmgen-autosign: Auto-sign MMGen transactions
 import sys,os,subprocess,time,signal,shutil
 from stat import *
 
-mountpoint   = '/mnt/tx'
-tx_dir       = '/mnt/tx/tx'
-part_label   = 'MMGEN_TX'
-wallet_dir   = '/dev/shm/autosign'
-key_fn       = 'autosign.key'
+mountpoint   = u'/mnt/tx'
+tx_dir       = u'/mnt/tx/tx'
+part_label   = u'MMGEN_TX'
+wallet_dir   = u'/dev/shm/autosign'
+key_fn       = u'autosign.key'
 
 from mmgen.common import *
 prog_name = os.path.basename(sys.argv[0])
@@ -129,7 +129,7 @@ def check_daemons_running():
 				ydie(1,'{} daemon not running or not listening on port {}'.format(coin,g.proto.rpc_port))
 
 def get_wallet_files():
-	wfs = [f for f in os.listdir(wallet_dir) if f[-6:] == '.mmdat']
+	wfs = filter(lambda x: x[-6:] == '.mmdat',os.listdir(wallet_dir))
 	if not wfs:
 		die(1,'No wallet files present!')
 	return [os.path.join(wallet_dir,w) for w in wfs]
@@ -193,7 +193,7 @@ def decrypt_wallets():
 	opt.passwd_file = os.path.join(tx_dir,key_fn)
 #	opt.passwd_file = '/tmp/key'
 	from mmgen.seed import SeedSource
-	msg("Unlocking wallet{} with key from '{}'".format(suf(wfs),opt.passwd_file))
+	msg(u"Unlocking wallet{} with key from '{}'".format(suf(wfs),opt.passwd_file))
 	fails = 0
 	for wf in wfs:
 		try:
@@ -225,14 +225,14 @@ def wipe_existing_key():
 	try: os.stat(fn)
 	except: pass
 	else:
-		msg('\nWiping existing key {}'.format(fn))
+		msg(u'\nWiping existing key {}'.format(fn))
 		subprocess.call(['wipe','-cf',fn])
 
 def create_key():
 	from binascii import hexlify
 	kdata = hexlify(os.urandom(32))
 	fn = os.path.join(tx_dir,key_fn)
-	desc = 'key file {}'.format(fn)
+	desc = u'key file {}'.format(fn)
 	msg('Creating ' + desc)
 	try:
 		with open(fn,'w') as f: f.write(kdata+'\n')
@@ -311,7 +311,7 @@ def set_led(cmd):
 
 def get_insert_status():
 	if os.getenv('MMGEN_TEST_SUITE'): return True
-	try: os.stat(os.path.join('/dev/disk/by-label/',part_label))
+	try: os.stat(os.path.join(u'/dev/disk/by-label',part_label))
 	except: return False
 	else: return True
 

mmgen/main_txbump.py

@@ -22,7 +22,6 @@ mmgen-txbump: Increase the fee on a replaceable (replace-by-fee) MMGen
 """
 
 from mmgen.common import *
-from mmgen.seed import SeedSource
 
 opts_data = lambda: {
 	'desc': 'Increase the fee on a replaceable (RBF) {g.proj_name} transaction, creating a new transaction, and optionally sign and send the new transaction'.format(g=g),

mmgen/main_txdo.py

@@ -21,7 +21,6 @@ mmgen-txdo: Create, sign and broadcast an online MMGen transaction
 """
 
 from mmgen.common import *
-from mmgen.seed import SeedSource
 
 opts_data = lambda: {
 	'desc': 'Create, sign and send an {g.proj_name} transaction'.format(g=g),

mmgen/main_txsign.py

@@ -21,7 +21,6 @@ mmgen-txsign: Sign a transaction generated by 'mmgen-txcreate'
 """
 
 from mmgen.common import *
-from mmgen.seed import SeedSource
 
 # -w, --use-wallet-dat (keys from running coin daemon) removed: use walletdump rpc instead
 opts_data = lambda: {

mmgen/main_wallet.py

@@ -158,6 +158,7 @@ if invoked_as == 'passchg' and ss_in.infile.dirname == g.data_dir:
 else:
 	try:
 		assert invoked_as == 'gen','dw'
+		assert not opt.outdir,'dw'
 		assert not opt.stdout,'dw'
 		assert not find_file_in_dir(Wallet,g.data_dir),'dw'
 		m = 'Make this wallet your default and move it to the data directory?'

mmgen/opts.py

@@ -28,7 +28,7 @@ from mmgen.globalvars import g
 import mmgen.share.Opts
 from mmgen.util import *
 
-def usage(): Die(2,'USAGE: {} {}'.format((g.prog_name,usage_txt)))
+def usage(): Die(2,'USAGE: {} {}'.format(g.prog_name,usage_txt))
 
 def die_on_incompatible_opts(incompat_list):
 	for group in incompat_list:
@@ -120,7 +120,7 @@ def get_data_from_cfg_file():
 			with open(fn,'wb') as f: f.write(template_data)
 			os.chmod(fn,0600)
 		except:
-			die(2,"ERROR: unable to write to datadir '{}'".format(g.data_dir))
+			die(2,u"ERROR: unable to write to datadir '{}'".format(g.data_dir))
 
 	for k,suf in (('cfg',''),('sample','.sample')):
 		try:

mmgen/seed.py

@@ -60,6 +60,7 @@ class SeedSource(MMGenObject):
 	ask_tty = True
 	no_tty  = False
 	op = None
+	require_utf8_input = False
 	_msg = {}
 
 	class SeedSourceData(MMGenObject): pass
@@ -131,7 +132,7 @@ class SeedSource(MMGenObject):
 	def _get_data(self):
 		if hasattr(self,'infile'):
 			self.fmt_data = get_data_from_file(self.infile.name,self.desc,
-								binary=self.file_mode=='binary')
+								binary=self.file_mode=='binary',require_utf8=self.require_utf8_input)
 		else:
 			self.fmt_data = self._get_data_from_user(self.desc)
 
@@ -384,13 +385,13 @@ class Mnemonic (SeedSourceUnenc):
 		longest_word = max(len(w) for w in wl)
 		from string import ascii_lowercase
 
-		m  = 'Enter your {}-word mnemonic, hitting ENTER or SPACE after each word.\n'
-		m += "Optionally, you may use pad characters.  Anything you type that's not a\n"
-		m += 'lowercase letter will be treated as a “pad character”, i.e. it will simply\n'
-		m += 'be discarded.  Pad characters may be typed before, after, or in the middle\n'
-		m += "of words.  For each word, once you've typed {} characters total (including\n"
-		m += 'pad characters) a pad character will enter the word.'
-		msg(m.decode('utf8').format(mn_len,longest_word))
+		m  = u'Enter your {}-word mnemonic, hitting ENTER or SPACE after each word.\n'
+		m += u"Optionally, you may use pad characters.  Anything you type that's not a\n"
+		m += u'lowercase letter will be treated as a “pad character”, i.e. it will simply\n'
+		m += u'be discarded.  Pad characters may be typed before, after, or in the middle\n'
+		m += u"of words.  For each word, once you've typed {} characters total (including\n"
+		m += u'pad characters) a pad character will enter the word.'
+		msg(m.format(mn_len,longest_word))
 
 		def get_word():
 			s,pad = '',0
@@ -580,6 +581,7 @@ class Wallet (SeedSourceEnc):
 	fmt_codes = 'wallet','w'
 	desc = g.proj_name + ' wallet'
 	ext = 'mmdat'
+	require_utf8_input = True # label is UTF-8
 
 	def _get_label_from_user(self,old_lbl=''):
 		d = u"to reuse the label '{}'".format(old_lbl.hl()) if old_lbl else 'for no label'
@@ -740,6 +742,7 @@ class Brainwallet (SeedSourceEnc):
 	fmt_codes = 'mmbrain','brainwallet','brain','bw','b'
 	desc = 'brainwallet'
 	ext = 'mmbrain'
+	require_utf8_input = True # brainwallet is user input, so require UTF-8
 	# brainwallet warning message? TODO
 
 	def get_bw_params(self):
@@ -765,8 +768,7 @@ class Brainwallet (SeedSourceEnc):
 			seed_len = opt.seed_len
 		qmsg_r('Hashing brainwallet data.  Please wait...')
 		# Use buflen arg of scrypt.hash() to get seed of desired length
-		seed = scrypt_hash_passphrase(self.brainpasswd, '',
-					d.hash_preset, buflen=seed_len/8)
+		seed = scrypt_hash_passphrase(self.brainpasswd,'',d.hash_preset,buflen=seed_len/8)
 		qmsg('Done')
 		self.seed = Seed(seed)
 		msg('Seed ID: {}'.format(self.seed.sid))

mmgen/test.py

@@ -25,24 +25,20 @@ from binascii import hexlify
 
 from mmgen.common import *
 
-def path_join(*args,**kwargs):
-	if not 'decode' in kwargs: kwargs['decode'] = True
-	assert type(kwargs['decode']) == bool
-	assert kwargs.keys() == ['decode']
-	ret = os.path.join(*[a.encode('utf8') for a in args])
-	return ret.decode('utf8') if kwargs['decode'] else ret
-
+# Windows uses non-UTF8 encodings in filesystem, so use raw bytes here
 def cleandir(d):
-	from shutil import rmtree
-	try:    files = [f.decode('utf8') for f in os.listdir(d)]
+	d_enc = d.encode('utf8')
+
+	try:    files = os.listdir(d_enc)
 	except: return
 
+	from shutil import rmtree
 	gmsg(u"Cleaning directory '{}'".format(d))
 	for f in files:
 		try:
-			os.unlink(path_join(d,f,decode=False))
+			os.unlink(os.path.join(d_enc,f))
 		except:
-			rmtree(path_join(d,f,decode=False))
+			rmtree(os.path.join(d_enc,f))
 
 def getrandnum(n): return int(hexlify(os.urandom(n)),16)
 def getrandhex(n): return hexlify(os.urandom(n))

mmgen/tool.py

@@ -523,8 +523,8 @@ def monero_wallet_ops(infile,op,blockheight=None,addrs=None):
 	def create(n,d,fn):
 		try: os.stat(fn)
 		except: pass
-		else: die(1,"Wallet '{}' already exists!".format(fn))
-		p = pexpect.spawn('monero-wallet-cli --generate-from-spend-key {}'.format(fn))
+		else: die(1,u"Wallet '{}' already exists!".format(fn))
+		p = pexpect.spawn('monero-wallet-cli --generate-from-spend-key {}'.format(fn.encode('utf8')))
 		if g.debug: p.logfile = sys.stdout
 		my_expect(p,'Awaiting initial prompt','Secret spend key: ')
 		my_sendline(p,'',d.sec,65)
@@ -559,8 +559,8 @@ def monero_wallet_ops(infile,op,blockheight=None,addrs=None):
 
 	def sync(n,d,fn):
 		try: os.stat(fn)
-		except: die(1,"Wallet '{}' does not exist!".format(fn))
-		p = pexpect.spawn('monero-wallet-cli --wallet-file={}'.format(fn))
+		except: die(1,u"Wallet '{}' does not exist!".format(fn))
+		p = pexpect.spawn('monero-wallet-cli --wallet-file={}'.format(fn.encode('utf8')))
 		if g.debug: p.logfile = sys.stdout
 		my_expect(p,'Awaiting password prompt','Wallet password: ')
 		my_sendline(p,'Sending password',d.wallet_passwd,33)
@@ -601,11 +601,12 @@ def monero_wallet_ops(infile,op,blockheight=None,addrs=None):
 		assert dl,"No addresses in addrfile within range '{}'".format(addrs)
 		gmsg('\n{}ing {} wallet{}'.format(m[op][0],dl,suf(dl)))
 		for n,d in enumerate(data): # [d.sec,d.wallet_passwd,d.viewkey,d.addr]
-			fn = '{}{}-{}-MoneroWallet'.format(
-				(opt.outdir+'/' if opt.outdir else ''),
-				al.al_id.sid,
-				d.idx)
-			gmsg('\n{}ing wallet {}/{} ({})'.format(m[op][1],n+1,dl,fn))
+			fn = os.path.join(
+				opt.outdir or u'',u'{}-{}-MoneroWallet{}'.format(
+					al.al_id.sid,
+					d.idx,
+					u'-α' if g.debug_utf8 else ''))
+			gmsg(u'\n{}ing wallet {}/{} ({})'.format(m[op][1],n+1,dl,fn))
 			m[op][2](n,d,fn)
 		gmsg('\n{} wallet{} {}ed'.format(dl,suf(dl),m[op][0].lower()))
 		if op == 'sync':

mmgen/tw.py

@@ -125,9 +125,9 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		while True:
 			self.cols = get_terminal_size()[0]
 			if self.cols >= g.min_screen_width: break
-			m1 = 'Screen too narrow to display the tracking wallet'
+			m1 = 'Screen too narrow to display the tracking wallet\n'
 			m2 = 'Please resize your screen to at least {} characters and hit ENTER '
-			my_raw_input(m1+'\n'+m2.format(g.min_screen_width))
+			my_raw_input((m1+m2).format(g.min_screen_width))
 
 	def display(self):
 		if not opt.no_blank: msg(CUR_HOME+ERASE_ALL)
@@ -280,7 +280,7 @@ Display options: show [D]ays, [g]roup, show [m]mgen addr, r[e]draw screen
 				idx,lbl = self.get_idx_and_label_from_user()
 				if idx:
 					e = self.unspent[idx-1]
-					if type(self).add_label(e.twmmid,lbl.decode('utf8'),addr=e.addr):
+					if type(self).add_label(e.twmmid,lbl,addr=e.addr):
 						self.get_unspent_data()
 						self.do_sort()
 						msg(u'{}\n{}\n{}'.format(self.fmt_display,prompt,p))

mmgen/util.py

@@ -116,12 +116,11 @@ def parse_nbytes(nbytes):
 	die(1,"'{}': invalid byte specifier".format(nbytes))
 
 def check_or_create_dir(path):
-	path_enc = path.encode('utf8')
 	try:
-		os.listdir(path_enc)
+		os.listdir(path)
 	except:
 		try:
-			os.makedirs(path_enc,0700)
+			os.makedirs(path,0700)
 		except:
 			die(2,u"ERROR: unable to read or create path '{}'".format(path))
 
@@ -178,6 +177,7 @@ def make_chksum_8(s,sep=False):
 	return '{} {}'.format(s[:4],s[4:]) if sep else s
 def make_chksum_6(s):
 	from mmgen.obj import HexStr
+	if type(s) == unicode: s = s.encode('utf8')
 	return HexStr(sha256(s).hexdigest()[:6])
 def is_chksum_6(s): return len(s) == 6 and is_hex_str_lc(s)
 
@@ -619,15 +619,15 @@ def write_data_to_file(
 def get_words_from_user(prompt):
 	# split() also strips
 	words = my_raw_input(prompt, echo=opt.echo_passphrase).split()
-	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
+	dmsg(u'Sanitized input: [{}]'.format(' '.join(words)))
 	return words
 
 def get_words_from_file(infile,desc,silent=False):
 	if not silent:
 		qmsg(u"Getting {} from file '{}'".format(desc,infile))
 	f = open_file_or_exit(infile, 'r')
-	# split() also strips
-	words = f.read().split()
+	try: words = f.read().decode('utf8').split() # split() also strips
+	except: die(1,'{} data must be UTF-8 encoded.'.format(capfirst(desc)))
 	f.close()
 	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
 	return words
@@ -655,19 +655,22 @@ def get_lines_from_file(fn,desc='',trim_comments=False,silent=False):
 	dmsg(u"Got {} lines from file '{}'".format(len(ret),fn))
 	return ret
 
-def get_data_from_user(desc='data',silent=False):
-	p = ('','Enter {}: '.format(desc))[g.stdin_tty]
+def get_data_from_user(desc='data',silent=False): # user input MUST be UTF-8
+	p = ('',u'Enter {}: '.format(desc))[g.stdin_tty]
 	data = my_raw_input(p,echo=opt.echo_passphrase)
-	dmsg('User input: [{}]'.format(data))
+	dmsg(u'User input: [{}]'.format(data))
 	return data
 
-def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False):
+def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,require_utf8=False):
 	if dash and infile == '-': return sys.stdin.read()
 	if not opt.quiet and not silent and desc:
 		qmsg(u"Getting {} from file '{}'".format(desc,infile))
 	f = open_file_or_exit(infile,('r','rb')[bool(binary)],silent=silent)
 	data = f.read()
 	f.close()
+	if require_utf8:
+		try: data = data.decode('utf8')
+		except: die(1,'{} data must be UTF-8 encoded.'.format(capfirst(desc)))
 	return data
 
 def pwfile_reuse_warning():
@@ -703,10 +706,13 @@ def my_raw_input(prompt,echo=True,insert_txt='',use_readline=True):
 		reply = raw_input(prompt.encode('utf8'))
 	else:
 		from getpass import getpass
-		reply = getpass(prompt)
+		reply = getpass(prompt.encode('utf8'))
 	kb_hold_protect()
 
-	return reply.strip()
+	try:
+		return reply.strip().decode('utf8')
+	except:
+		die(1,'User input must be UTF-8 encoded.')
 
 def keypress_confirm(prompt,default_yes=False,verbose=False,no_nl=False):
 
@@ -755,8 +761,6 @@ def do_pager(text):
 	if 'PAGER' in os.environ and os.environ['PAGER'] != pagers[0]:
 		pagers = [os.environ['PAGER']] + pagers
 
-	text = text.encode('utf8')
-
 	for pager in pagers:
 		end = ('\n(end of text)\n','')[pager=='less']
 		try:
@@ -764,7 +768,7 @@ def do_pager(text):
 			p = Popen([pager], stdin=PIPE, shell=shell)
 		except: pass
 		else:
-			p.communicate(text+end+'\n')
+			p.communicate(text.encode('utf8')+end+'\n')
 			msg_r('\r')
 			break
 	else: Msg(text+end)

scripts/test-release.sh

@@ -136,8 +136,7 @@ f_misc_ni='Miscellaneous non-interactive tests complete'
 
 i_alts='Gen-only altcoin'
 s_alts='The following tests will test generation operations for all supported altcoins'
-if [ "$MINGW" ]; then
-	t_alts=(
+t_alts=(
 	"$scrambletest_py"
 	"$test_py -n altcoin_ref"
 	"$gentest_py --coin=btc 2 $rounds"
@@ -148,25 +147,10 @@ if [ "$MINGW" ]; then
 	"$gentest_py --coin=ltc --type=compressed 2 $rounds"
 	"$gentest_py --coin=ltc --type=segwit 2 $rounds"
 	"$gentest_py --coin=ltc --type=bech32 2 $rounds"
-	"$gentest_py --coin=zec 2 $rounds"
 	"$gentest_py --coin=etc 2 $rounds"
-	"$gentest_py --coin=eth 2 $rounds")
-else
-	t_alts=(
-	"$scrambletest_py"
-	"$test_py -n altcoin_ref"
-	"$gentest_py --coin=btc 2 $rounds"
-	"$gentest_py --coin=btc --type=compressed 2 $rounds"
-	"$gentest_py --coin=btc --type=segwit 2 $rounds"
-	"$gentest_py --coin=btc --type=bech32 2 $rounds"
-	"$gentest_py --coin=ltc 2 $rounds"
-	"$gentest_py --coin=ltc --type=compressed 2 $rounds"
-	"$gentest_py --coin=ltc --type=segwit 2 $rounds"
-	"$gentest_py --coin=ltc --type=bech32 2 $rounds"
+	"$gentest_py --coin=eth 2 $rounds"
 	"$gentest_py --coin=zec 2 $rounds"
 	"$gentest_py --coin=zec --type=zcash_z 2 $rounds_spec"
-	"$gentest_py --coin=etc 2 $rounds"
-	"$gentest_py --coin=eth 2 $rounds"
 
 	"$gentest_py --coin=btc 2:ext $rounds"
 	"$gentest_py --coin=btc --type=compressed 2:ext $rounds"
@@ -183,14 +167,22 @@ else
 	"$gentest_py --all 2:pyethereum $rounds_low"
 	"$gentest_py --all 2:keyconv $rounds_low"
 	"$gentest_py --all 2:zcash_mini $rounds_low")
+if [ "$MINGW" ]; then
+	t_alts[13]="# MSWin platform: skipping zcash z-addr generation and altcoin verification with third-party tools"
+	i=14 end=${#t_alts[*]}
+	while [ $i -lt $end ]; do unset t_alts[$i]; let i++; done
 fi
 f_alts='Gen-only altcoin tests completed'
 
-TMPDIR='/tmp/mmgen-test-release-'$(cat /dev/urandom | base32 - | head -n1 | cut -b 1-16)
+if [ "$MINGW" ]; then
+	TMPDIR='/tmp/mmgen-test-release'
+else
+	TMPDIR='/tmp/mmgen-test-release-'$(cat /dev/urandom | base32 - | head -n1 | cut -b 1-16)
+fi
 mkdir -p $TMPDIR
 
 i_monero='Monero'
-s_monero='Testing generation and wallet creation operations for Monero'
+s_monero='Testing key-address file generation and wallet creation and sync operations for Monero'
 s_monero='The monerod (mainnet) daemon must be running for the following tests'
 t_monero=(
 "mmgen-walletgen -q -r0 -p1 -Llabel --outdir $TMPDIR -o words"
@@ -207,10 +199,14 @@ t_monero=(
 "$mmgen_tool -q --accept-defaults --outdir $TMPDIR syncmonerowallets $TMPDIR/*-XMR*.akeys addrs=23-29"
 "$mmgen_tool -q --accept-defaults --outdir $TMPDIR syncmonerowallets $TMPDIR/*-XMR*.akeys"
 )
-[ "$MINGW" ] && t_monero=("$t_monero")
+[ "$MINGW" ] && {
+	t_monero[2]="# MSWin platform: skipping Monero wallet creation and sync tests; NOT verifying key-addr list"
+	i=3 end=${#t_monero[*]}
+	while [ $i -lt $end ]; do unset t_monero[$i]; let i++; done
+}
 f_monero='Monero tests completed'
 
-i_misc='Miscellaneous operations (interactive)' # includes autosign!
+i_misc='Miscellaneous operations (autosign)'
 s_misc='The bitcoin, bitcoin-abc and litecoin (mainnet) daemons must be running for the following tests'
 t_misc=(
 	"$test_py -On misc")

scripts/tx-v1-to-v3.py

@@ -117,6 +117,7 @@ tx.outputs = tx.MMGenTxOutputList(
 				MMGenTX.MMGenTxOutput(addr=i['scriptPubKey']['addresses'][0],
 						amt=g.proto.coin_amt(i['value']))
 			for i in dec_tx['vout'])
+
 for e in tx.outputs:
 	if e.addr in outputs:
 		f = outputs[e.addr]

test/mmgen_pexpect.py

@@ -126,7 +126,8 @@ class MMGenPexpect(object):
 				clr1,clr2,eol = ((green,cyan,'\n'),(nocolor,nocolor,' '))[bool(opt.print_cmdline)]
 				sys.stderr.write(green('Testing: {}\n'.format(desc)))
 				if not msg_only:
-					sys.stderr.write(clr1(u'Executing {}{}'.format(clr2(cmd_str),eol)))
+					s = repr(cmd_str) if g.platform == 'win' else cmd_str
+					sys.stderr.write(clr1(u'Executing {}{}'.format(clr2(s),eol)))
 			else:
 				m = 'Testing {}: '.format(desc)
 				msg_r(m)

test/test.py

@@ -36,18 +36,18 @@ set_debug_all()
 g.quiet = False # if 'quiet' was set in config file, disable here
 os.environ['MMGEN_QUIET'] = '0' # and for the spawned scripts
 
-log_file = 'test.py_log'
+log_file = u'test.py_log'
 
 hincog_fn      = 'rand_data'
 hincog_bytes   = 1024*1024
 hincog_offset  = 98765
 hincog_seedlen = 256
 
-incog_id_fn  = 'incog_id'
-non_mmgen_fn = 'coinkey'
-pwfile = 'passwd_file'
+incog_id_fn  = u'incog_id'
+non_mmgen_fn = u'coinkey'
+pwfile = u'passwd_file'
 
-ref_dir = os.path.join('test','ref')
+ref_dir = os.path.join(u'test',u'ref')
 
 ref_wallet_brainpass = 'abc'
 ref_wallet_hash_preset = '1'
@@ -64,13 +64,13 @@ ref_tx_label_lat_cyr_gr = ''.join(map(unichr,
 									range(913,939) +   # greek
 									range(97,123)))[:MMGenTXLabel.max_len] # 72 chars
 ref_bw_hash_preset = '1'
-ref_bw_file        = 'wallet.mmbrain'
-ref_bw_file_spc    = 'wallet-spaced.mmbrain'
+ref_bw_file        = u'wallet.mmbrain'
+ref_bw_file_spc    = u'wallet-spaced.mmbrain'
 
 ref_kafile_pass        = 'kafile password'
 ref_kafile_hash_preset = '1'
 
-ref_enc_fn = 'sample-text.mmenc'
+ref_enc_fn = u'sample-text.mmenc'
 tool_enc_passwd = "Scrypt it, don't hash it!"
 sample_text = \
 	'The Times 03/Jan/2009 Chancellor on brink of second bailout for banks\n'
@@ -79,14 +79,13 @@ sample_text = \
 # under '/dev/shm' and put datadir and temp files here.
 shortopts = ''.join([e[1:] for e in sys.argv if len(e) > 1 and e[0] == '-' and e[1] != '-'])
 shortopts = ['-'+e for e in list(shortopts)]
-data_dir_basename = 'data_dir' + ('',u'-α')[bool(os.getenv('MMGEN_DEBUG_UTF8'))]
-data_dir = path_join('test',data_dir_basename)
-data_dir_enc = data_dir.encode('utf8')
-trash_dir = path_join('test','trash')
+data_dir_basename = u'data_dir' + ('',u'-α')[bool(os.getenv('MMGEN_DEBUG_UTF8'))]
+data_dir = os.path.join(u'test',data_dir_basename)
+trash_dir = os.path.join(u'test',u'trash')
 
 if not any(e in ('--skip-deps','--resume','-S','-r') for e in sys.argv+shortopts):
 	if g.platform == 'win':
-		for tdir in (data_dir_enc,trash_dir):
+		for tdir in (data_dir,trash_dir):
 			try: os.listdir(tdir)
 			except: pass
 			else:
@@ -106,11 +105,11 @@ if not any(e in ('--skip-deps','--resume','-S','-r') for e in sys.argv+shortopts
 			die(2,'Unable to delete directory tree {}/{}* ({})'.format(d,pfx,e))
 		try:
 			import tempfile
-			shm_dir = tempfile.mkdtemp('',pfx,d)
+			shm_dir = unicode(tempfile.mkdtemp('',pfx,d))
 		except Exception as e:
 			die(2,'Unable to create temporary directory in {} ({})'.format(d,e))
-		for tdir in (data_dir_enc,trash_dir):
-			dd = path_join(shm_dir,os.path.basename(tdir),decode=False)
+		for tdir in (data_dir,trash_dir):
+			dd = os.path.join(shm_dir,os.path.basename(tdir))
 			os.mkdir(dd,0755)
 			try: os.unlink(tdir)
 			except: pass
@@ -207,7 +206,7 @@ def restore_debug():
 
 cfgs = {
 	'15': {
-		'tmpdir':        os.path.join('test','tmp15'),
+		'tmpdir':        os.path.join(u'test',u'tmp15'),
 		'wpasswd':       'Dorian',
 		'kapasswd':      'Grok the blockchain',
 		'addr_idx_list': '12,99,5-10,5,12', # 8 addresses
@@ -222,7 +221,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'16': {
-		'tmpdir':        os.path.join('test','tmp16'),
+		'tmpdir':        os.path.join(u'test',u'tmp16'),
 		'wpasswd':       'My changed password',
 		'hash_preset':   '2',
 		'dep_generators': {
@@ -230,11 +229,11 @@ cfgs = {
 		},
 		'segwit': get_segwit_bool()
 	},
-	'17': { 'tmpdir': os.path.join('test','tmp17') },
-	'18': { 'tmpdir': os.path.join('test','tmp18') },
-	'19': { 'tmpdir': os.path.join('test','tmp19'), 'wpasswd':'abc' },
+	'17': { 'tmpdir': os.path.join(u'test',u'tmp17') },
+	'18': { 'tmpdir': os.path.join(u'test',u'tmp18') },
+	'19': { 'tmpdir': os.path.join(u'test',u'tmp19'), 'wpasswd':'abc' },
 	'1': {
-		'tmpdir':        os.path.join('test','tmp1'),
+		'tmpdir':        os.path.join(u'test',u'tmp1'),
 		'wpasswd':       'Dorian',
 		'kapasswd':      'Grok the blockchain',
 		'addr_idx_list': '12,99,5-10,5,12', # 8 addresses
@@ -250,14 +249,14 @@ cfgs = {
 			'mmhex':       'export_hex',
 			'mmincog':     'export_incog',
 			'mmincox':     'export_incog_hex',
-			hincog_fn:     'export_incog_hidden',
-			incog_id_fn:   'export_incog_hidden',
+			hincog_fn:     u'export_incog_hidden',
+			incog_id_fn:   u'export_incog_hidden',
 			'akeys.mmenc': 'keyaddrgen'
 		},
 		'segwit': get_segwit_bool()
 	},
 	'2': {
-		'tmpdir':        os.path.join('test','tmp2'),
+		'tmpdir':        os.path.join(u'test',u'tmp2'),
 		'wpasswd':       'Hodling away',
 		'addr_idx_list': '37,45,3-6,22-23',  # 8 addresses
 		'seed_len':      128,
@@ -271,7 +270,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'20': {
-		'tmpdir':        os.path.join('test','tmp20'),
+		'tmpdir':        os.path.join(u'test',u'tmp20'),
 		'wpasswd':       'Vsize it',
 		'addr_idx_list': '1-8',  # 8 addresses
 		'seed_len':      256,
@@ -284,7 +283,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'21': {
-		'tmpdir':        os.path.join('test','tmp21'),
+		'tmpdir':        os.path.join(u'test',u'tmp21'),
 		'wpasswd':       'Vsize it',
 		'addr_idx_list': '1-8',  # 8 addresses
 		'seed_len':      256,
@@ -297,7 +296,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'3': {
-		'tmpdir':        os.path.join('test','tmp3'),
+		'tmpdir':        os.path.join(u'test',u'tmp3'),
 		'wpasswd':       'Major miner',
 		'addr_idx_list': '73,54,1022-1023,2-5', # 8 addresses
 		'dep_generators': {
@@ -309,7 +308,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'4': {
-		'tmpdir':        os.path.join('test','tmp4'),
+		'tmpdir':        os.path.join(u'test',u'tmp4'),
 		'wpasswd':       'Hashrate good',
 		'addr_idx_list': '63,1004,542-544,7-9', # 8 addresses
 		'seed_len':      192,
@@ -321,13 +320,13 @@ cfgs = {
 			'sigtx':       'txsign4',
 			'txdo':        'txdo4',
 		},
-		'bw_filename': 'brainwallet.mmbrain',
+		'bw_filename': u'brainwallet.mmbrain',
 		'bw_params':   '192,1',
 		'segwit': get_segwit_bool()
 	},
 	'14': {
 		'kapasswd':      'Maxwell',
-		'tmpdir':        os.path.join('test','tmp14'),
+		'tmpdir':        os.path.join(u'test',u'tmp14'),
 		'wpasswd':       'The Halving',
 		'addr_idx_list': '61,998,502-504,7-9', # 8 addresses
 		'seed_len':      256,
@@ -339,7 +338,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'5': {
-		'tmpdir':        os.path.join('test','tmp5'),
+		'tmpdir':        os.path.join(u'test',u'tmp5'),
 		'wpasswd':       'My changed password',
 		'hash_preset':   '2',
 		'dep_generators': {
@@ -389,14 +388,14 @@ cfgs = {
 		'passfile32_chk':  '37B6 C218 2ABC 7508',
 		'passfilehex_chk': '523A F547 0E69 8323',
 		'wpasswd':         'reference password',
-		'ref_wallet':      'FE3C6545-D782B529[128,1].mmdat',
-		'ic_wallet':       'FE3C6545-E29303EA-5E229E30[128,1].mmincog',
-		'ic_wallet_hex':   'FE3C6545-BC4BE3F2-32586837[128,1].mmincox',
+		'ref_wallet':      u'FE3C6545-D782B529[128,1].mmdat',
+		'ic_wallet':       u'FE3C6545-E29303EA-5E229E30[128,1].mmincog',
+		'ic_wallet_hex':   u'FE3C6545-BC4BE3F2-32586837[128,1].mmincox',
 
 		'hic_wallet':       'FE3C6545-161E495F-BEB7548E[128,1].incog-offset123',
 		'hic_wallet_old':   'FE3C6545-161E495F-9860A85B[128,1].incog-old.offset123',
 
-		'tmpdir':        os.path.join('test','tmp6'),
+		'tmpdir':        os.path.join(u'test',u'tmp6'),
 		'kapasswd':      '',
 		'addr_idx_list': '1010,500-501,31-33,1,33,500,1011', # 8 addresses
 		'pass_idx_list': '1,4,9-11,1100',
@@ -449,14 +448,14 @@ cfgs = {
 		'passfile32_chk':  '2A28 C5C7 36EC 217A',
 		'passfilehex_chk': 'B11C AC6A 1464 608D',
 		'wpasswd':         'reference password',
-		'ref_wallet':      '1378FC64-6F0F9BB4[192,1].mmdat',
-		'ic_wallet':       '1378FC64-2907DE97-F980D21F[192,1].mmincog',
-		'ic_wallet_hex':   '1378FC64-4DCB5174-872806A7[192,1].mmincox',
+		'ref_wallet':      u'1378FC64-6F0F9BB4[192,1].mmdat',
+		'ic_wallet':       u'1378FC64-2907DE97-F980D21F[192,1].mmincog',
+		'ic_wallet_hex':   u'1378FC64-4DCB5174-872806A7[192,1].mmincox',
 
-		'hic_wallet':       '1378FC64-B55E9958-77256FC1[192,1].incog.offset123',
-		'hic_wallet_old':   '1378FC64-B55E9958-D85FF20C[192,1].incog-old.offset123',
+		'hic_wallet':      u'1378FC64-B55E9958-77256FC1[192,1].incog.offset123',
+		'hic_wallet_old':  u'1378FC64-B55E9958-D85FF20C[192,1].incog-old.offset123',
 
-		'tmpdir':        os.path.join('test','tmp7'),
+		'tmpdir':        os.path.join(u'test',u'tmp7'),
 		'kapasswd':      '',
 		'addr_idx_list': '1010,500-501,31-33,1,33,500,1011', # 8 addresses
 		'pass_idx_list': '1,4,9-11,1100',
@@ -509,11 +508,11 @@ cfgs = {
 		'passfile32_chk':  'F6C1 CDFB 97D9 FCAE',
 		'passfilehex_chk': 'BD4F A0AC 8628 4BE4',
 		'wpasswd':         'reference password',
-		'ref_wallet':      '98831F3A-{}[256,1].mmdat'.format(('27F2BF93','E2687906')[g.testnet]),
-		'ref_addrfile':    '98831F3A{}[1,31-33,500-501,1010-1011]{}.addrs',
-		'ref_segwitaddrfile':'98831F3A{}-S[1,31-33,500-501,1010-1011]{}.addrs',
-		'ref_bech32addrfile':'98831F3A{}-B[1,31-33,500-501,1010-1011]{}.addrs',
-		'ref_keyaddrfile': '98831F3A{}[1,31-33,500-501,1010-1011]{}.akeys.mmenc',
+		'ref_wallet':      u'98831F3A-{}[256,1].mmdat'.format(('27F2BF93','E2687906')[g.testnet]),
+		'ref_addrfile':    u'98831F3A{}[1,31-33,500-501,1010-1011]{}.addrs',
+		'ref_segwitaddrfile':u'98831F3A{}-S[1,31-33,500-501,1010-1011]{}.addrs',
+		'ref_bech32addrfile':u'98831F3A{}-B[1,31-33,500-501,1010-1011]{}.addrs',
+		'ref_keyaddrfile': u'98831F3A{}[1,31-33,500-501,1010-1011]{}.akeys.mmenc',
 		'ref_passwdfile':  u'98831F3A-фубар@crypto.org-b58-20[1,4,9-11,1100].pws',
 		'ref_addrfile_chksum': {
 			'btc': ('6FEF 6FB9 7B13 5D91','3C2C 8558 BB54 079E'),
@@ -551,13 +550,13 @@ cfgs = {
 			'b2x': '6A52BC-B2X[106.6789,tl=1320969600]{}.rawtx',
 			'ltc': '75F455-LTC[106.6789]{}.rawtx',
 		},
-		'ic_wallet':       '98831F3A-5482381C-18460FB1[256,1].mmincog',
-		'ic_wallet_hex':   '98831F3A-1630A9F2-870376A9[256,1].mmincox',
+		'ic_wallet':       u'98831F3A-5482381C-18460FB1[256,1].mmincog',
+		'ic_wallet_hex':   u'98831F3A-1630A9F2-870376A9[256,1].mmincox',
 
-		'hic_wallet':       '98831F3A-F59B07A0-559CEF19[256,1].incog.offset123',
-		'hic_wallet_old':   '98831F3A-F59B07A0-848535F3[256,1].incog-old.offset123',
+		'hic_wallet':       u'98831F3A-F59B07A0-559CEF19[256,1].incog.offset123',
+		'hic_wallet_old':   u'98831F3A-F59B07A0-848535F3[256,1].incog-old.offset123',
 
-		'tmpdir':        os.path.join('test','tmp8'),
+		'tmpdir':        os.path.join(u'test',u'tmp8'),
 		'kapasswd':      '',
 		'addr_idx_list': '1010,500-501,31-33,1,33,500,1011', # 8 addresses
 		'pass_idx_list': '1,4,9-11,1100',
@@ -571,7 +570,7 @@ cfgs = {
 		'segwit': get_segwit_bool()
 	},
 	'9': {
-		'tmpdir':        os.path.join('test','tmp9'),
+		'tmpdir':        os.path.join(u'test',u'tmp9'),
 		'tool_enc_infn':      'tool_encrypt.in',
 #		'tool_enc_ref_infn':  'tool_encrypt_ref.in',
 		'wpasswd':         'reference password',
@@ -587,7 +586,7 @@ cfgs = {
 from copy import deepcopy
 for a,b in (('6','11'),('7','12'),('8','13')):
 	cfgs[b] = deepcopy(cfgs[a])
-	cfgs[b]['tmpdir'] = os.path.join('test','tmp'+b)
+	cfgs[b]['tmpdir'] = os.path.join(u'test',u'tmp'+b)
 
 if g.debug_utf8:
 	for k in cfgs: cfgs[k]['tmpdir'] += u'-α'
@@ -1041,10 +1040,8 @@ NL = ('\r\n','\n')[g.platform=='linux' and bool(opt.popen_spawn)]
 
 def get_file_with_ext(ext,mydir,delete=True,no_dot=False,return_list=False):
 
-	ext_enc = ext.encode('utf8')
 	dot = ('.','')[bool(no_dot)]
-	flist = [os.path.join(mydir.encode('utf8'),f).decode('utf8') for f in os.listdir(mydir.encode('utf8'))
-				if f == ext_enc or f[-len(dot+ext_enc):] == dot+ext_enc]
+	flist = [os.path.join(mydir,f) for f in os.listdir(mydir) if f == ext or f[-len(dot+ext):] == dot+ext]
 
 	if not flist: return False
 	if return_list: return flist
@@ -1118,9 +1115,9 @@ def create_fake_unspent_entry(coinaddr,al_id=None,idx=None,lbl=None,non_mmgen=Fa
 	amt1,amt2 = {'btc':(10,40),'bch':(10,40),'ltc':(1000,4000)}[coin_sel]
 	return {
 		'account': '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) if non_mmgen \
-			else (u'{}:{}{}'.format(al_id,idx,lbl.decode('utf8'))),
+			else (u'{}:{}{}'.format(al_id,idx,lbl)),
 		'vout': int(getrandnum(4) % 8),
-		'txid': hexlify(os.urandom(32)).decode('utf8'),
+		'txid': unicode(hexlify(os.urandom(32))),
 		'amount': g.proto.coin_amt('{}.{}'.format(amt1 + getrandnum(4) % amt2, getrandnum(4) % 100000000)),
 		'address': coinaddr,
 		'spendable': False,
@@ -1132,8 +1129,8 @@ labels = [
 	"Automotive",
 	"Travel expenses",
 	"Healthcare",
-	ref_tx_label_jp[:40].encode('utf8'),
-	ref_tx_label_zh[:40].encode('utf8'),
+	ref_tx_label_jp[:40],
+	ref_tx_label_zh[:40],
 	"Alice's allowance",
 	"Bob's bequest",
 	"House purchase",
@@ -1155,11 +1152,11 @@ def get_label(do_shuffle=False):
 	from random import shuffle
 	global label_iter
 	try:
-		return next(label_iter)
+		return unicode(next(label_iter))
 	except:
 		if do_shuffle: shuffle(labels)
 		label_iter = iter(labels)
-		return next(label_iter)
+		return unicode(next(label_iter))
 
 def create_fake_unspent_data(adata,tx_data,non_mmgen_input='',non_mmgen_input_compressed=True):
 
@@ -1183,7 +1180,7 @@ def create_fake_unspent_data(adata,tx_data,non_mmgen_input='',non_mmgen_input_co
 	return out
 
 def write_fake_data_to_file(d):
-	unspent_data_file = path_join(cfg['tmpdir'],'unspent.json')
+	unspent_data_file = os.path.join(cfg['tmpdir'],u'unspent.json')
 	write_data_to_file(unspent_data_file,d,'Unspent outputs',silent=True)
 	os.environ['MMGEN_BOGUS_WALLET_DATA'] = unspent_data_file.encode('utf8')
 	bwd_msg = u'MMGEN_BOGUS_WALLET_DATA={}'.format(unspent_data_file)
@@ -1249,15 +1246,26 @@ def add_comments_to_addr_file(addrfile,outfile,use_labels=False):
 	write_data_to_file(outfile,a.fmt_data,silent=True)
 	end_silence()
 
+# 100 words chosen randomly from here:
+#   https://github.com/bitcoin/bips/pull/432/files/6332230d63149a950d05db78964a03bfd344e6b0
+rwords = [
+	'ампула','арест','арка','архив','атлас','афера','багаж','башмак','бежать','бидон','брюки','вена',
+	'взвод','виски','волна','вспышка','встреча','гавань','гамма','гора','горшок','депутат','динамика',
+	'доверие','доза','документ','жених','жюри','зависть','заслуга','зато','зацепка','заявка','здание',
+	'зеркало','зефир','зрачок','изнутри','исход','кедр','киоск','кирпич','комната','концерт','косой',
+	'кубок','лачуга','лужа','мелодия','металл','механизм','механизм','механизм','мост','мощность','мыло',
+	'некий','нижний','новый','няня','овощ','ограда','опыт','орел','падение','петля','пила','поцелуй',
+	'пощечина','проект','путем','пыль','роман','рюкзак','сауна','сбыт','север','сейчас','сержант','след',
+	'слуга','снижение','сокол','соус','стакан','статус','сущность','табак','тело','тень','техника','ужин',
+	'упор','уровень','фирма','франция','фуражка','чучело','шрифт','элемент']
+
 def make_brainwallet_file(fn):
 	# Print random words with random whitespace in between
-	from mmgen.mn_tirosh import words
-	wl = words.split()
 	nwords,ws_list,max_spaces = 10,'    \n',5
 	def rand_ws_seq():
 		nchars = getrandnum(1) % max_spaces + 1
 		return ''.join([ws_list[getrandnum(1)%len(ws_list)] for i in range(nchars)])
-	rand_pairs = [wl[getrandnum(4) % len(wl)] + rand_ws_seq() for i in range(nwords)]
+	rand_pairs = [rwords[getrandnum(4) % len(rwords)] + rand_ws_seq() for i in range(nwords)]
 	d = ''.join(rand_pairs).rstrip() + '\n'
 	if opt.verbose: msg_r('Brainwallet password:\n{}'.format(cyan(d)))
 	write_data_to_file(fn,d,'brainwallet password',silent=True)
@@ -1451,7 +1459,8 @@ class MMGenTestSuite(object):
 
 	def walletgen(self,name,del_dw_run='dummy',seed_len=None,gen_dfl_wallet=False):
 		write_to_tmpfile(cfg,pwfile,cfg['wpasswd']+'\n')
-		args = ['-d',cfg['tmpdir'],'-p1']
+		args = ['-p1']
+		if not gen_dfl_wallet: args += ['-d',cfg['tmpdir']]
 		if seed_len: args += ['-l',str(seed_len)]
 		t = MMGenExpect(name,'mmgen-walletgen', args + [usr_rand_arg])
 		t.license()
@@ -1459,9 +1468,9 @@ class MMGenTestSuite(object):
 		t.passphrase_new('new MMGen wallet',cfg['wpasswd'])
 		t.label()
 		global have_dfl_wallet
-		if not have_dfl_wallet:
-			t.expect('move it to the data directory? (Y/n): ',('n','y')[gen_dfl_wallet])
-			if gen_dfl_wallet: have_dfl_wallet = True
+		if not have_dfl_wallet and gen_dfl_wallet:
+			t.expect('move it to the data directory? (Y/n): ','y')
+			have_dfl_wallet = True
 		t.written_to_file('MMGen wallet')
 		t.ok()
 
@@ -1604,7 +1613,7 @@ class MMGenTestSuite(object):
 		self.addrgen(name,wf,pf=pf,check_ref=True,mmtype='compressed')
 
 	def addrimport(self,name,addrfile):
-		outfile = os.path.join(cfg['tmpdir'],'addrfile_w_comments')
+		outfile = os.path.join(cfg['tmpdir'],u'addrfile_w_comments')
 		add_comments_to_addr_file(addrfile,outfile)
 		t = MMGenExpect(name,'mmgen-addrimport', [outfile])
 		t.expect_getend(r'Checksum for address data .*\[.*\]: ',regex=True)
@@ -1710,8 +1719,9 @@ class MMGenTestSuite(object):
 			t.expect('Add a comment to transaction? (y/N): ','\n')
 			t.expect('Save transaction? (y/N): ','y')
 			t.written_to_file('Transaction')
-		os.unlink(txfile.encode('utf8')) # our tx file replaces the original
-		os.system('touch ' + path_join(cfg['tmpdir'],'txbump',decode=False))
+		os.unlink(txfile) # our tx file replaces the original
+		cmd = 'touch ' + os.path.join(cfg['tmpdir'],u'txbump')
+		os.system(cmd.encode('utf8'))
 		t.ok()
 
 	def txdo(self,name,addrfile,wallet):
@@ -1825,7 +1835,7 @@ class MMGenTestSuite(object):
 
 	# TODO: make outdir and hidden incog compatible (ignore --outdir and warn user?)
 	def export_incog_hidden(self,name,wf):
-		rf = path_join(cfg['tmpdir'],hincog_fn)
+		rf = os.path.join(cfg['tmpdir'],hincog_fn)
 		add_args = ['-J',u'{},{}'.format(rf,hincog_offset)]
 		self.export_incog(
 			name,wf,desc='hidden incognito data',out_fmt='hi',add_args=add_args)
@@ -1870,7 +1880,7 @@ class MMGenTestSuite(object):
 		self.addrgen_incog(name,wf,'',in_fmt='xi',desc='hex incognito data')
 
 	def addrgen_incog_hidden(self,name,wf,foo):
-		rf = path_join(cfg['tmpdir'],hincog_fn)
+		rf = os.path.join(cfg['tmpdir'],hincog_fn)
 		self.addrgen_incog(name,[],'',in_fmt='hi',desc='hidden incognito data',
 			args=['-H',u'{},{}'.format(rf,hincog_offset),'-l',str(hincog_seedlen)])
 
@@ -1991,7 +2001,8 @@ class MMGenTestSuite(object):
 		os.system('rm -f {}/*.sigtx'.format(cfg['tmpdir'].encode('utf8')))
 		self.txsign4(name,f7,f8,f9,f10,f11,f12,txdo_handle=t)
 		self.txsend(name,'',txdo_handle=t)
-		os.system('touch ' + path_join(cfg['tmpdir'],'txdo',decode=False))
+		cmd = 'touch ' + os.path.join(cfg['tmpdir'],u'txdo')
+		os.system(cmd.encode('utf8'))
 
 	def txbump4(self,name,f1,f2,f3,f4,f5,f6,f7,f8,f9): # f7:txfile,f9:'txdo'
 		non_mm_fn = os.path.join(cfg['tmpdir'],non_mmgen_fn)
@@ -2108,9 +2119,7 @@ class MMGenTestSuite(object):
 		# make a bad tx file
 		with open(os.path.join(cfg['tmpdir'],'tx','bad.rawtx'),'w') as f:
 			f.write('bad tx data')
-		ls = os.listdir(cfg['tmpdir'])
 		opts = ['--mountpoint='+cfg['tmpdir'],'--coins=btc,bch,ltc']
-#		opts += ['--quiet']
 		mn_fn = os.path.join(ref_dir,cfgs['8']['seed_id']+'.mmwords')
 		mn = read_from_file(mn_fn).strip().split()
 
@@ -2135,11 +2144,11 @@ class MMGenTestSuite(object):
 
 	# Saved reference file tests
 	def ref_wallet_conv(self,name):
-		wf = path_join(ref_dir,cfg['ref_wallet'])
+		wf = os.path.join(ref_dir,cfg['ref_wallet'])
 		self.walletconv_in(name,wf,'MMGen wallet',pw=True,oo=True)
 
 	def ref_mn_conv(self,name,ext='mmwords',desc='Mnemonic data'):
-		wf = path_join(ref_dir,cfg['seed_id']+'.'+ext)
+		wf = os.path.join(ref_dir,cfg['seed_id']+'.'+ext)
 		self.walletconv_in(name,wf,desc,oo=True)
 
 	def ref_seed_conv(self,name):
@@ -2188,7 +2197,7 @@ class MMGenTestSuite(object):
 		self.walletconv_out(name,'hex incognito data',out_fmt='xi',pw=True)
 
 	def ref_hincog_conv_out(self,name,extra_uopts=[]):
-		ic_f = path_join(cfg['tmpdir'],hincog_fn)
+		ic_f = os.path.join(cfg['tmpdir'],hincog_fn)
 		hi_parms = u'{},{}'.format(ic_f,ref_wallet_incog_offset)
 		sl_parm = '-l' + str(cfg['seed_len'])
 		self.walletconv_out(name,
@@ -2247,7 +2256,7 @@ class MMGenTestSuite(object):
 	def ref_addrfile_chk(self,name,ftype='addr',coin=None,subdir=None,pfx=None,mmtype=None,add_args=[]):
 		af_key = 'ref_{}file'.format(ftype)
 		af_fn = cfg[af_key].format(pfx or altcoin_pfx,'' if coin else tn_ext)
-		af = path_join(ref_dir,(subdir or ref_subdir,'')[ftype=='passwd'],af_fn)
+		af = os.path.join(ref_dir,(subdir or ref_subdir,'')[ftype=='passwd'],af_fn)
 		coin_arg = [] if coin == None else ['--coin='+coin]
 		tool_cmd = ftype.replace('segwit','').replace('bech32','')+'file_chksum'
 		t = MMGenExpect(name,'mmgen-tool',coin_arg+[tool_cmd,af]+add_args)
@@ -2371,7 +2380,7 @@ class MMGenTestSuite(object):
 
 	def walletconv_out(self,name,desc,out_fmt='w',uopts=[],uopts_chk=[],pw=False):
 		opts = ['-d',cfg['tmpdir'],'-p1','-o',out_fmt] + uopts
-		infile = path_join(ref_dir,cfg['seed_id']+'.mmwords')
+		infile = os.path.join(ref_dir,cfg['seed_id']+'.mmwords')
 		t = MMGenExpect(name,'mmgen-walletconv',[usr_rand_arg]+opts+[infile],extra_desc='(convert)')
 
 		add_args = ['-l{}'.format(cfg['seed_len'])]
@@ -2406,7 +2415,7 @@ class MMGenTestSuite(object):
 	def regtest_setup(self,name):
 		if g.testnet:
 			die(2,'--testnet option incompatible with regtest test suite')
-		try: shutil.rmtree(os.path.join(data_dir_enc,'regtest'))
+		try: shutil.rmtree(os.path.join(data_dir,'regtest'))
 		except: pass
 		os.environ['MMGEN_TEST_SUITE'] = '' # mnemonic is piped to stdin, so stop being a terminal
 		t = MMGenExpect(name,'mmgen-regtest',['-n','setup'])
@@ -2428,7 +2437,7 @@ class MMGenTestSuite(object):
 
 	@staticmethod
 	def regtest_user_dir(user,coin=None):
-		return path_join(data_dir,'regtest',coin or g.coin.lower(),user)
+		return os.path.join(data_dir,u'regtest',coin or g.coin.lower(),user)
 
 	def regtest_user_sid(self,user):
 		return os.path.basename(get_file_with_ext('mmdat',self.regtest_user_dir(user)))[:8]
@@ -2541,7 +2550,8 @@ class MMGenTestSuite(object):
 		t.expect('OK? (Y/n): ','y') # fee OK?
 		t.expect('OK? (Y/n): ','y') # change OK?
 		t.expect('Add a comment to transaction? (y/N): ',('\n','y')[do_label])
-		if do_label: t.expect('Comment: ',ref_tx_label_jp.encode('utf8')+'\n')
+		if do_label:
+			t.expect('Comment: ',ref_tx_label_jp.encode('utf8')+'\n')
 		t.expect('View decoded transaction\? .*?: ',('t','v')[full_tx_view],regex=True)
 		if not do_label: t.expect('to continue: ','\n')
 		t.passphrase('MMGen wallet',pw)
@@ -2665,8 +2675,8 @@ class MMGenTestSuite(object):
 
 	def regtest_bob_pre_import(self,name):
 		pairs = self.gen_pairs(5)
-		write_to_tmpfile(cfg,'non-mmgen.keys','\n'.join([a[0] for a in pairs])+'\n')
-		write_to_tmpfile(cfg,'non-mmgen.addrs','\n'.join([a[1] for a in pairs])+'\n')
+		write_to_tmpfile(cfg,u'non-mmgen.keys','\n'.join([a[0] for a in pairs])+'\n')
+		write_to_tmpfile(cfg,u'non-mmgen.addrs','\n'.join([a[1] for a in pairs])+'\n')
 		return self.regtest_user_txdo(name,'bob',rtFee[4],[pairs[0][1]],'3')
 
 	def regtest_user_import(self,name,user,args):
@@ -2678,15 +2688,15 @@ class MMGenTestSuite(object):
 		t.ok()
 
 	def regtest_bob_import_addr(self,name):
-		addr = read_from_tmpfile(cfg,'non-mmgen.addrs').split()[0]
+		addr = read_from_tmpfile(cfg,u'non-mmgen.addrs').split()[0]
 		return self.regtest_user_import(name,'bob',['--rescan','--address='+addr])
 
 	def regtest_bob_import_list(self,name):
-		fn = os.path.join(cfg['tmpdir'],'non-mmgen.addrs')
+		fn = os.path.join(cfg['tmpdir'],u'non-mmgen.addrs')
 		return self.regtest_user_import(name,'bob',['--addrlist',fn])
 
 	def regtest_bob_split2(self,name):
-		addrs = read_from_tmpfile(cfg,'non-mmgen.addrs').split()
+		addrs = read_from_tmpfile(cfg,u'non-mmgen.addrs').split()
 		amts = (1.12345678,2.87654321,3.33443344,4.00990099,5.43214321)
 		outputs1 = map('{},{}'.format,addrs,amts)
 		sid = self.regtest_user_sid('bob')