[mswin]: full test suite support with pexpect (PopenSpawn)

[mswin]: secure wallet deletion with sdelete
[mswin]: keyaddrfile binary mode write bugfix (main_addrgen.py)

mmgen/addr.py

@@ -263,10 +263,10 @@ Removed %s duplicate wif key%s from keylist (also in {pnm} key-address file
 		self.fmt_data = mmgen_encrypt(self.fmt_data,'new key list','')
 		self.ext += '.'+g.mmenc_ext
 
-	def write_to_file(self,ask_tty=True,ask_write_default_yes=False):
+	def write_to_file(self,ask_tty=True,ask_write_default_yes=False,binary=False):
 		fn = '{}.{}'.format(self.id_str,self.ext)
 		ask_tty = self.has_keys and not opt.quiet
-		write_data_to_file(fn,self.fmt_data,self.file_desc,ask_tty=ask_tty)
+		write_data_to_file(fn,self.fmt_data,self.file_desc,ask_tty=ask_tty,binary=binary)
 
 	def idxs(self):
 		return [e.idx for e in self.data]

mmgen/color.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution
+# Copyright (C)2013-2016 Philemon <mmgen-py@yandex.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+color.py:  color routines for the MMGen suite
+"""
+
+import os
+
+# If 88- or 256-color support is compiled, the following apply.
+#    P s = 3 8 ; 5 ; P s -> Set foreground color to the second P s .
+#    P s = 4 8 ; 5 ; P s -> Set background color to the second P s .
+if os.environ['TERM'][-8:] == '256color':
+	_blk,_red,_grn,_yel,_blu,_mag,_cya,_bright,_dim,_ybright,_ydim,_pnk,_orng,_gry = [
+	'\033[38;5;%s;1m' % c for c in 232,210,121,229,75,90,122,231,245,187,243,218,215,246]
+	_redbg = '\033[38;5;232;48;5;210;1m'
+	_grnbg = '\033[38;5;232;48;5;121;1m'
+	_grybg = '\033[38;5;231;48;5;240;1m'
+	_reset = '\033[0m'
+else:
+	_blk,_red,_grn,_yel,_blu,_mag,_cya,_reset,_grnbg = \
+		['\033[%sm' % c for c in '30;1','31;1','32;1','33;1','34;1','35;1','36;1','0','30;102']
+	_gry=_orng=_pnk=_redbg=_ybright=_ydim=_bright=_dim=_grybg=_mag  # TODO
+
+clr_red=clr_grn=clr_grnbg=clr_yel=clr_cya=clr_blu=clr_pnk=clr_orng=clr_gry=clr_mag=clr_reset=''
+
+def nocolor(s): return s
+def red(s):     return clr_red+s+clr_reset
+def green(s):   return clr_grn+s+clr_reset
+def grnbg(s):   return clr_grnbg+s+clr_reset
+def yellow(s):  return clr_yel+s+clr_reset
+def cyan(s):    return clr_cya+s+clr_reset
+def blue(s):    return clr_blu+s+clr_reset
+def pink(s):    return clr_pnk+s+clr_reset
+def orange(s):  return clr_orng+s+clr_reset
+def gray(s):    return clr_gry+s+clr_reset
+def magenta(s): return clr_mag+s+clr_reset
+
+def init_color(enable_color=True):
+	global clr_red,clr_grn,clr_grnbg,clr_yel,clr_cya,clr_blu,clr_pnk,clr_orng,clr_gry,clr_mag,clr_reset
+	if enable_color:
+		clr_red = _red   
+		clr_grn = _grn
+		clr_grnbg = _grnbg
+		clr_yel = _yel
+		clr_cya = _cya
+		clr_blu = _blu
+		clr_pnk = _pnk
+		clr_orng = _orng
+		clr_gry = _gry
+		clr_mag = _mag
+		clr_reset = _reset

mmgen/main.py

@@ -28,16 +28,14 @@ def launch(what):
 
 	try: import termios
 	except: # Windows
-		from mmgen.util import start_mscolor
-		start_mscolor()
 		__import__('mmgen.main_' + what)
 	else:
-		import sys,atexit
-		fd = sys.stdin.fileno()
-		old = termios.tcgetattr(fd)
-		def at_exit():
-			termios.tcsetattr(fd, termios.TCSADRAIN, old)
-		atexit.register(at_exit)
+		import sys,os,atexit
+		if not os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+			fd = sys.stdin.fileno()
+			old = termios.tcgetattr(fd)
+			def at_exit(): termios.tcsetattr(fd, termios.TCSADRAIN, old)
+			atexit.register(at_exit)
 		try: __import__('mmgen.main_' + what)
 		except KeyboardInterrupt:
 			sys.stderr.write('\nUser interrupt\n')

mmgen/main_addrgen.py

@@ -117,5 +117,6 @@ if al.gen_addrs and opt.print_checksum:
 
 if al.gen_keys and keypress_confirm('Encrypt key list?'):
 	al.encrypt()
-
-al.write_to_file()
+	al.write_to_file(binary=True)
+else:
+	al.write_to_file()

mmgen/main_wallet.py

@@ -150,12 +150,14 @@ m3 = 'Make this wallet your default and move it to the data directory?'
 if invoked_as == 'passchg' and ss_in.infile.dirname == g.data_dir:
 	confirm_or_exit(m1,m2,exit_msg='Password not changed')
 	ss_out.write_to_file(desc='New wallet',outdir=g.data_dir)
-	msg('Deleting old wallet')
-	from subprocess import check_call
+	msg('Securely deleting old wallet')
+	from subprocess import check_output,CalledProcessError
+	sd_cmd = (['wipe','-sf'],['sdelete','-p','20'])[g.platform=='win']
 	try:
-		check_call(['wipe','-s',ss_in.infile.name])
+		check_output(sd_cmd + [ss_in.infile.name])
 	except:
-		msg('WARNING: wipe failed, using regular file delete instead')
+		msg(yellow("WARNING: '%s' command failed, using regular file delete instead" % sd_cmd[0]))
+#		msg('Command output: {}\nReturn value {}'.format(e.output,e.returncode))
 		os.unlink(ss_in.infile.name)
 elif invoked_as == 'gen' and not find_file_in_dir(Wallet,g.data_dir) \
 	and not opt.stdout and keypress_confirm(m3,default_yes=True):

mmgen/obj.py

@@ -21,6 +21,7 @@ obj.py:  The MMGenObject class and methods
 """
 
 from decimal import *
+from mmgen.color import *
 lvl = 0
 
 class MMGenObject(object):
@@ -270,10 +271,8 @@ class Hilite(object):
 
 	@classmethod
 	def colorize(cls,s,color=True):
-		from mmgen.globalvars import g
-		from mmgen.util import red,blue,green,yellow,pink,cyan,gray,orange,magenta
 		k = color if type(color) is str else cls.color # hack: override color with str value
-		return locals()[k](s) if (color or cls.color_always) and g.color else s
+		return globals()[k](s) if (color or cls.color_always) else s
 
 class BTCAmt(Decimal,Hilite,InitErrors):
 	color = 'yellow'

mmgen/opts.py

@@ -107,13 +107,19 @@ def opt_postproc_debug():
 		Msg('        {:<20}: {}'.format(e, getattr(g,e)))
 	Msg('\n=== end opts.py debug ===\n')
 
-def opt_postproc_actions():
+def opt_postproc_initializations():
 	from mmgen.term import set_terminal_vars
 	set_terminal_vars()
+
 	# testnet data_dir differs from data_dir_root, so check or create
 	from mmgen.util import msg,die,check_or_create_dir
 	check_or_create_dir(g.data_dir) # dies on error
 
+	from mmgen.color import init_color
+	init_color(g.color)
+
+	if g.platform == 'win': start_mscolor()
+
 def	set_data_dir_root():
 	g.data_dir_root = os.path.normpath(os.path.expanduser(opt.data_dir)) if opt.data_dir else \
 			os.path.join(g.home_dir,'.'+g.proj_name.lower())
@@ -234,12 +240,11 @@ def init(opts_data,add_opts=[],opt_filter=None):
 		sys.exit()
 
 	if g.debug: opt_postproc_debug()
-
 	if opt.verbose: opt.quiet = None
 
 	die_on_incompatible_opts(g.incompatible_opts)
 
-	opt_postproc_actions()
+	opt_postproc_initializations()
 
 	return args
 

mmgen/term.py

@@ -33,6 +33,8 @@ except:
 		_platform = 'win'
 	except:
 		die(2,'Unable to set terminal mode')
+	if os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+		msvcrt.setmode(sys.stdin.fileno(),os.O_BINARY)
 
 def _kb_hold_protect_unix():
 
@@ -75,6 +77,10 @@ def _get_keypress_unix(prompt='',immed_chars='',prehold_protect=True):
 	termios.tcsetattr(fd, termios.TCSADRAIN, old)
 	return ch
 
+def _get_keypress_unix_stub(prompt='',immed_chars='',prehold_protect=None):
+	msg_r(prompt)
+	return sys.stdin.read(1)
+
 def _get_keypress_unix_raw(prompt='',immed_chars='',prehold_protect=None):
 
 	msg_r(prompt)
@@ -89,8 +95,6 @@ def _get_keypress_unix_raw(prompt='',immed_chars='',prehold_protect=None):
 
 	return ch
 
-
-
 def _kb_hold_protect_mswin():
 
 	timeout = float(0.5)
@@ -136,6 +140,9 @@ def _get_keypress_mswin_raw(prompt='',immed_chars='',prehold_protect=None):
 	if ord(ch) == 3: raise KeyboardInterrupt
 	return ch
 
+def _get_keypress_mswin_emu(prompt='',immed_chars='',prehold_protect=None):
+	msg_r(prompt)
+	return sys.stdin.read(1)
 
 def _get_terminal_size_linux():
 
@@ -166,21 +173,26 @@ def _get_terminal_size_linux():
 	return int(cr[1]), int(cr[0])
 
 def _get_terminal_size_mswin():
+	import sys,os,struct
+	x,y = 0,0
 	try:
 		from ctypes import windll,create_string_buffer
-		# stdin handle is -10
-		# stdout handle is -11
-		# stderr handle is -12
-		h = windll.kernel32.GetStdHandle(-12)
+		# handles - stdin: -10, stdout: -11, stderr: -12
 		csbi = create_string_buffer(22)
+		h = windll.kernel32.GetStdHandle(-12)
 		res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
 		if res:
 			(bufx, bufy, curx, cury, wattr, left, top, right, bottom,
 			maxx, maxy) = struct.unpack('hhhhHhhhhhh', csbi.raw)
-			sizex = right - left + 1
-			sizey = bottom - top + 1
-			return sizex, sizey
+			x = right - left + 1
+			y = bottom - top + 1
 	except:
+		pass
+
+	if x and y:
+		return x, y
+	else:
+		msg(yellow('Warning: could not get terminal size. Using fallback dimensions.'))
 		return 80,25
 
 def mswin_dummy_flush(fd,termconst): pass
@@ -190,10 +202,14 @@ def set_terminal_vars():
 	if _platform == 'linux':
 		get_char = (_get_keypress_unix_raw,_get_keypress_unix)[g.hold_protect]
 		kb_hold_protect = (_kb_hold_protect_unix_raw,_kb_hold_protect_unix)[g.hold_protect]
+		if os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+			get_char,kb_hold_protect = _get_keypress_unix_stub,_kb_hold_protect_unix_raw
 		get_terminal_size = _get_terminal_size_linux
-		myflush = termios.tcflush   # call: myflush(sys.stdin, termios.TCIOFLUSH)
+#		myflush = termios.tcflush   # call: myflush(sys.stdin, termios.TCIOFLUSH)
 	else:
 		get_char = (_get_keypress_mswin_raw,_get_keypress_mswin)[g.hold_protect]
 		kb_hold_protect = (_kb_hold_protect_mswin_raw,_kb_hold_protect_mswin)[g.hold_protect]
+		if os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+			get_char = _get_keypress_mswin_emu
 		get_terminal_size = _get_terminal_size_mswin
-		myflush = mswin_dummy_flush
+#		myflush = mswin_dummy_flush

mmgen/util.py

@@ -24,33 +24,7 @@ import sys,os,time,stat,re
 from hashlib import sha256
 from binascii import hexlify,unhexlify
 from string import hexdigits
-
-# If 88- or 256-color support is compiled, the following apply.
-#    P s = 3 8 ; 5 ; P s -> Set foreground color to the second P s .
-#    P s = 4 8 ; 5 ; P s -> Set background color to the second P s .
-if os.environ['TERM'][-8:] == '256color':
-	_blk,_red,_grn,_yel,_blu,_mag,_cya,_bright,_dim,_ybright,_ydim,_pnk,_orng,_gry = [
-	'\033[38;5;%s;1m' % c for c in 232,210,121,229,75,90,122,231,245,187,243,218,215,246]
-	_redbg = '\033[38;5;232;48;5;210;1m'
-	_grnbg = '\033[38;5;232;48;5;121;1m'
-	_grybg = '\033[38;5;231;48;5;240;1m'
-	_reset = '\033[0m'
-else:
-	_blk,_red,_grn,_yel,_blu,_mag,_cya,_reset,_grnbg = \
-		['\033[%sm' % c for c in '30;1','31;1','32;1','33;1','34;1','35;1','36;1','0','30;102']
-	_gry = _orng = _pnk = _redbg = _ybright = _ydim = _bright = _dim = _grybg = _mag  # TODO
-
-def red(s):     return _red+s+_reset
-def green(s):   return _grn+s+_reset
-def grnbg(s):   return _grnbg+s+_reset
-def yellow(s):  return _yel+s+_reset
-def cyan(s):    return _cya+s+_reset
-def blue(s):    return _blu+s+_reset
-def pink(s):    return _pnk+s+_reset
-def orange(s):  return _orng+s+_reset
-def gray(s):    return _gry+s+_reset
-def magenta(s): return _mag+s+_reset
-def nocolor(s): return s
+from mmgen.color import *
 
 def msg(s):    sys.stderr.write(s+'\n')
 def msg_r(s):  sys.stderr.write(s)
@@ -309,17 +283,11 @@ def remove_comments(lines):
 from mmgen.globalvars import g
 
 def start_mscolor():
-	if g.platform == 'win':
-		global red,green,yellow,cyan,nocolor
-		import os
-		if 'MMGEN_NOMSCOLOR' in os.environ:
-			red = green = yellow = cyan = grnbg = nocolor
-		else:
-			try:
-				import colorama
-				colorama.init(strip=True,convert=True)
-			except:
-				red = green = yellow = cyan = grnbg = nocolor
+	try:
+		import colorama
+		colorama.init(strip=True,convert=True)
+	except:
+		msg('Import of colorama module failed')
 
 def get_hash_params(hash_preset):
 	if hash_preset in g.hash_presets:
@@ -465,7 +433,7 @@ def write_data_to_file(
 	if ask_write_default_yes == False or ask_write_prompt:
 		ask_write = True
 
-	if opt.stdout or not sys.stdout.isatty() or outfile in ('','-'):
+	def do_stdout():
 		qmsg('Output to STDOUT requested')
 		if sys.stdout.isatty():
 			if no_tty:
@@ -493,7 +461,8 @@ def write_data_to_file(
 			msvcrt.setmode(sys.stdout.fileno(),os.O_BINARY)
 
 		sys.stdout.write(data)
-	else:
+
+	def do_file(outfile,ask_write_prompt):
 		if opt.outdir and not os.path.isabs(outfile):
 			outfile = make_full_path(opt.outdir,outfile)
 
@@ -522,6 +491,14 @@ def write_data_to_file(
 
 		return True
 
+	if opt.stdout or outfile in ('','-'):
+		do_stdout()
+	elif not sys.stdout.isatty() and not os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+		do_stdout()
+	else:
+		do_file(outfile,ask_write_prompt)
+
+
 def get_words_from_user(prompt):
 	# split() also strips
 	words = my_raw_input(prompt, echo=opt.echo_passphrase).split()
@@ -603,9 +580,8 @@ def my_raw_input(prompt,echo=True,insert_txt='',use_readline=True):
 		prompt = ''
 
 	from mmgen.term import kb_hold_protect
-
 	kb_hold_protect()
-	if echo:
+	if echo or os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
 		reply = raw_input(prompt)
 	else:
 		from getpass import getpass

scripts/deinstall.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-CMD='rm -rf /usr/local/share/mmgen /usr/local/bin/mmgen-* /usr/local/lib/python2.7/dist-packages/mmgen*'
+CMD='rm -rf /usr/local/share/mmgen /usr/local/bin/mmgen-* /usr/local/lib/python2.7/dist-packages/mmgen* /mingw/opt/bin/mmgen-* /mingw/opt/lib/python2.7/site-packages/mmgen*'
 
 if [ "$EUID" = 0 -o "$HOMEPATH" ]; then
 	set -x; $CMD

test/gentest.py

@@ -31,8 +31,6 @@ from binascii import hexlify
 from mmgen.common import *
 from mmgen.bitcoin import hex2wif,privnum2addr
 
-start_mscolor()
-
 rounds = 100
 opts_data = {
 	'desc': "Test address generation in various ways",

test/test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-#
+# -*- coding: UTF-8 -*-
 # mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution
 # Copyright (C)2013-2016 Philemon <mmgen-py@yandex.com>
 #
@@ -20,6 +20,7 @@
 test/test.py:  Test suite for the MMGen suite
 """
 
+
 import sys,os
 
 def run_in_tb():
@@ -118,7 +119,7 @@ else:
 	os.mkdir(data_dir,0755)
 
 opts_data = {
-#	'sets': [('non_interactive',bool,'verbose',None)],
+#	'sets': [('interactive',bool,'verbose',None)],
 	'desc': 'Test suite for the MMGen suite',
 	'usage':'[options] [command(s) or metacommand(s)]',
 	'options': """
@@ -127,13 +128,14 @@ opts_data = {
 -b, --buf-keypress  Use buffered keypresses as with real human input
 -c, --print-cmdline Print the command line of each spawned command
 -d, --debug-scripts Turn on debugging output in executed scripts
+-x, --debug-pexpect Produce debugging output for pexpect calls
 -D, --direct-exec   Bypass pexpect and execute a command directly (for
                     debugging only)
 -e, --exact-output  Show the exact output of the MMGen script(s) being run
 -l, --list-cmds     List and describe the commands in the test suite
 -L, --log           Log commands to file {lf}
 -n, --names         Display command names instead of descriptions
--I, --non-interactive Non-interactive operation (MS Windows mode)
+-I, --interactive   Interactive mode (without pexpect)
 -p, --pause         Pause between tests, resuming on keypress
 -P, --profile       Record the execution time of each script
 -q, --quiet         Produce minimal output.  Suppress dependency info
@@ -359,8 +361,6 @@ cfgs = {
 	},
 }
 
-start_mscolor()
-
 from copy import deepcopy
 for a,b in ('6','11'),('7','12'),('8','13'):
 	cfgs[b] = deepcopy(cfgs[a])
@@ -588,12 +588,11 @@ usr_rand_chars = (5,30)[bool(opt.usr_random)]
 usr_rand_arg = '-r%s' % usr_rand_chars
 
 if opt.system: sys.path.pop(0)
-ni = bool(opt.non_interactive)
+ia = bool(opt.interactive)
 
-# Disable MS color in spawned scripts due to bad interactions
-os.environ['MMGEN_NOMSCOLOR'] = '1'
-os.environ['MMGEN_NO_LICENSE'] = '1'
+# Disable color in spawned scripts so we can parse their output
 os.environ['MMGEN_DISABLE_COLOR'] = '1'
+os.environ['MMGEN_NO_LICENSE'] = '1'
 os.environ['MMGEN_MIN_URANDCHARS'] = '3'
 
 if opt.debug_scripts: os.environ['MMGEN_DEBUG'] = '1'
@@ -657,18 +656,32 @@ if opt.list_cmds:
 	sys.exit()
 
 import time,re
-try:
+if g.platform == 'linux':
 	import pexpect
-except: # Windows
-	m1 = green('MS Windows or missing pexpect module detected.  Skipping some tests and running in\n')
-	m2 = green('interactive mode.  User prompts and control values will be ')
-	m3 = grnbg('HIGHLIGHTED IN GREEN')
-	m4 = green('.\nControl values should be checked against the program output.')
-	m5 = green('\nContinue?')
-	ni = True
-	if not keypress_confirm(m1+m2+m3+m4+m5,default_yes=True):
-		errmsg('Exiting at user request')
-		sys.exit()
+	if os.getenv('MMGEN_PEXPECT_POPEN_SPAWN'):
+		import termios,atexit
+		def at_exit(): os.system('stty sane')
+		atexit.register(at_exit)
+		from pexpect.popen_spawn import PopenSpawn
+		use_popen_spawn,NL = True,'\n'
+	else:
+		use_popen_spawn,NL = False,'\r\n'
+else: # Windows
+	use_popen_spawn,NL = True,'\r\n'
+	try:
+		import pexpect
+		from pexpect.popen_spawn import PopenSpawn
+	except:
+		ia = True
+		m1 = ('Missing pexpect module detected.  Skipping some tests and running in'
+			'\ninteractive mode.  User prompts and control value checks will be ')
+		m2 = 'HIGHLIGHTED IN GREEN'
+		m3 = '.\nControl values should be checked against the program output.\nContinue?'
+		if not keypress_confirm(green(m1)+grnbg(m2)+green(m3),default_yes=True):
+			errmsg('Exiting at user request')
+			sys.exit()
+	else:
+		os.environ['MMGEN_PEXPECT_POPEN_SPAWN'] = '1'
 
 def my_send(p,t,delay=send_delay,s=False):
 	if delay: time.sleep(delay)
@@ -690,10 +703,12 @@ def my_expect(p,s,t='',delay=send_delay,regex=False,nonl=False):
 		if s == '': ret = 0
 		else:
 			f = (p.expect_exact,p.expect)[bool(regex)]
-			ret = f(s,timeout=60)
+			ret = f(s,timeout=(60,5)[bool(opt.debug_pexpect)])
 	except pexpect.TIMEOUT:
+		if opt.debug_pexpect: raise
 		errmsg(red('\nERROR.  Expect %s%s%s timed out.  Exiting' % (quo,s,quo)))
 		sys.exit(1)
+	debug_pexpect_msg(p)
 
 	if opt.debug or (opt.verbose and type(s) != str): msg_r(' ==> %s ' % ret)
 
@@ -747,6 +762,10 @@ def verify_checksum_or_exit(checksum,chk):
 		sys.exit(1)
 	vmsg(green('Checksums match: %s') % (cyan(chk)))
 
+def debug_pexpect_msg(p):
+	if opt.debug_pexpect:
+		errmsg('\n{}{}{}'.format(red('BEFORE ['),p.before,red(']')))
+		errmsg('{}{}{}'.format(red('MATCH ['),p.after,red(']')))
 
 class MMGenExpect(object):
 
@@ -768,11 +787,11 @@ class MMGenExpect(object):
 			sys.stderr.write(clr1('Executing {}{}'.format(clr2(cmd_str),eol)))
 		else:
 			m = 'Testing %s: ' % desc
-			msg_r((m,yellow(m))[ni])
+			msg_r((m,yellow(m))[ia])
 
 		if mmgen_cmd_arg == '': return
 
-		if opt.direct_exec or ni:
+		if opt.direct_exec or ia:
 			msg('')
 			from subprocess import call,check_output
 			f = (call,check_output)[bool(no_output)]
@@ -784,7 +803,12 @@ class MMGenExpect(object):
 			if opt.traceback:
 				cmd_args = [mmgen_cmd] + cmd_args
 				mmgen_cmd = tb_cmd
-			self.p = pexpect.spawn(mmgen_cmd,cmd_args)
+			if use_popen_spawn:
+				ca = [("'"+a+"'" if ' ' in a else a) for a in cmd_args]
+				cmd = '{} {}'.format(mmgen_cmd,' '.join(ca))
+				self.p = PopenSpawn('python ' + cmd)
+			else:
+				self.p = pexpect.spawn(mmgen_cmd,cmd_args)
 			if opt.exact_output: self.p.logfile = sys.stdout
 
 	def license(self):
@@ -841,7 +865,9 @@ class MMGenExpect(object):
 			return outfile
 # 			else:
 # 				ret = my_expect(self.p,s1)
-		outfile = self.p.readline().strip().strip("'")
+		self.expect(NL,nonl=True)
+		outfile = self.p.before.strip().strip("'")
+		if opt.debug_pexpect: msgred('Outfile [%s]' % outfile)
 		vmsg('%s file: %s' % (desc,cyan(outfile.replace("'",''))))
 		return outfile
 
@@ -854,7 +880,12 @@ class MMGenExpect(object):
 
 	def expect_getend(self,s,regex=False):
 		ret = self.expect(s,regex=regex,nonl=True)
-		end = self.readline().strip()
+		debug_pexpect_msg(self.p)
+#		end = self.readline().strip()
+		# readline() of partial lines doesn't work with PopenSpawn, so do this instead:
+		self.expect(NL,nonl=True)
+		debug_pexpect_msg(self.p)
+		end = self.p.before
 		vmsg(' ==> %s' % cyan(end))
 		return end
 
@@ -870,18 +901,18 @@ class MMGenExpect(object):
 	def send(self,*args,**kwargs):
 		return my_send(self.p,*args,**kwargs)
 
-	def readline(self):
-		return self.p.readline()
-
-	def close(self):
-		return self.p.close()
-
-	def readlines(self):
-		return [l.rstrip()+'\n' for l in self.p.readlines()]
+# 	def readline(self):
+# 		return self.p.readline()
+# 	def readlines(self):
+# 		return [l.rstrip()+'\n' for l in self.p.readlines()]
 
 	def read(self,n=None):
 		return self.p.read(n)
 
+	def close(self):
+		if not use_popen_spawn:
+			self.p.close()
+
 from mmgen.obj import BTCAmt
 from mmgen.bitcoin import verify_addr
 
@@ -1012,15 +1043,18 @@ def check_needs_rerun(
 
 	fdeps = ts.generate_file_deps(cmd)
 	cdeps = ts.generate_cmd_deps(fdeps)
+#	print 'cmd,fdeps,cdeps,fns: ',cmd,fdeps,cdeps,fns # DEBUG
 
 	for fn in fns:
 		my_age = os.stat(fn).st_mtime
 		for num,ext in fdeps:
 			f = get_file_with_ext(ext,cfgs[num]['tmpdir'],delete=build)
-			if f and os.stat(f).st_mtime > my_age: rerun = True
+			if f and os.stat(f).st_mtime > my_age:
+				rerun = True
 
 	for cdep in cdeps:
-		if check_needs_rerun(ts,cdep,build=build,root=False,dpy=cmd): rerun = True
+		if check_needs_rerun(ts,cdep,build=build,root=False,dpy=cmd):
+			rerun = True
 
 	if build:
 		if rerun:
@@ -1070,7 +1104,7 @@ def check_deps(cmds):
 
 
 def clean(usr_dirs=[]):
-	if opt.skip_deps and not ni: return
+	if opt.skip_deps and not ia: return
 	all_dirs = MMGenTestSuite().list_tmp_dirs()
 	dirs = (usr_dirs or all_dirs)
 	for d in sorted(dirs):
@@ -1101,7 +1135,7 @@ class MMGenTestSuite(object):
 
 	def do_cmd(self,cmd):
 
-		if ni and (len(cmd_data[cmd]) < 4 or cmd_data[cmd][3] != 1): return
+		if ia and (len(cmd_data[cmd]) < 4 or cmd_data[cmd][3] != 1): return
 
 		# delete files produced by this cmd
 # 		for ext,tmpdir in find_generated_exts(cmd):
@@ -1138,22 +1172,22 @@ class MMGenTestSuite(object):
 		for s in scripts:
 			t = MMGenExpect(name,('mmgen-'+s),[arg],
 				extra_desc='(mmgen-%s)'%s,no_output=True)
-			if not ni:
+			if not ia:
 				t.read(); ok()
 
 	def longhelpscreens(self,name): self.helpscreens(name,arg='--longhelp')
 
 	def walletgen(self,name,del_dw_run='dummy',seed_len=None,gen_dfl_wallet=False):
-		if ni:
+		if ia:
 			m = "\nAnswer '{}' at the the interactive prompt".format(('n','y')[gen_dfl_wallet])
 			msg(grnbg(m))
 		write_to_tmpfile(cfg,pwfile,cfg['wpasswd']+'\n')
 		add_args = ([usr_rand_arg],
-			['-q','-r0','-L','NI Wallet','-P',get_tmpfile_fn(cfg,pwfile)])[bool(ni)]
+			['-q','-r0','-L','Interactive Mode Wallet','-P',get_tmpfile_fn(cfg,pwfile)])[bool(ia)]
 		args = ['-d',cfg['tmpdir'],'-p1']
 		if seed_len: args += ['-l',str(seed_len)]
 		t = MMGenExpect(name,'mmgen-walletgen', args + add_args)
-		if ni: return
+		if ia: return
 		t.license()
 		t.usr_rand(usr_rand_chars)
 		t.passphrase_new('new MMGen wallet',cfg['wpasswd'])
@@ -1177,13 +1211,13 @@ class MMGenTestSuite(object):
 		args = ['-d',cfg['tmpdir'],hp_arg,sl_arg,'-ib','-L',label]
 		write_to_tmpfile(cfg,bf,ref_wallet_brainpass)
 		write_to_tmpfile(cfg,pwfile,cfg['wpasswd'])
-		if ni:
+		if ia:
 			add_args = ['-r0', '-q', '-P%s' % get_tmpfile_fn(cfg,pwfile),
 							get_tmpfile_fn(cfg,bf)]
 		else:
 			add_args = [usr_rand_arg]
 		t = MMGenExpect(name,'mmgen-walletconv', args + add_args)
-		if ni: return
+		if ia: return
 		t.license()
 		t.expect('Enter brainwallet: ', ref_wallet_brainpass+'\n')
 		t.passphrase_new('new MMGen wallet',cfg['wpasswd'])
@@ -1194,14 +1228,14 @@ class MMGenTestSuite(object):
 	def refwalletgen(self,name): self.brainwalletgen_ref(name)
 
 	def passchg(self,name,wf,pf):
-		# ni: reuse password, since there's no way to change it non-interactively
+		# ia: reuse password, since there's no way to change it non-interactively
 		silence()
 		write_to_tmpfile(cfg,pwfile,get_data_from_file(pf))
 		end_silence()
-		add_args = ([usr_rand_arg],['-q','-r0','-P',pf])[bool(ni)]
+		add_args = ([usr_rand_arg],['-q','-r0','-P',pf])[bool(ia)]
 		t = MMGenExpect(name,'mmgen-passchg', add_args +
 				['-d',cfg['tmpdir'],'-p','2','-L','Changed label'] + ([],[wf])[bool(wf)])
-		if ni: return
+		if ia: return
 		t.license()
 		t.passphrase('MMGen wallet',cfgs['1']['wpasswd'],pwtype='old')
 		t.expect_getend('Hash preset changed to ')
@@ -1214,27 +1248,29 @@ class MMGenTestSuite(object):
 		if not wf:
 			t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
 			t.written_to_file('New wallet')
-			t.expect('Okay to WIPE 1 regular file ? (Yes/No)','Yes\n')
+			t.expect('Securely deleting old wallet')
+#			t.expect('Okay to WIPE 1 regular file ? (Yes/No)','Yes\n')
+			t.expect('Wallet passphrase has changed')
 			t.expect_getend('has been changed to ')
 		else:
 			t.written_to_file('MMGen wallet')
 		ok()
 
 	def passchg_dfl_wallet(self,name,pf):
-		if ni:
+		if ia:
 			m = "\nAnswer 'YES'<ENTER> at the the interactive prompt"
 			msg(grnbg(m))
 		return self.passchg(name=name,wf=None,pf=pf)
 
 	def walletchk(self,name,wf,pf,desc='MMGen wallet',
 			add_args=[],sid=None,pw=False,extra_desc=''):
-		args = ([],['-P',pf,'-q'])[bool(ni and pf)]
+		args = ([],['-P',pf,'-q'])[bool(ia and pf)]
 		hp = cfg['hash_preset'] if 'hash_preset' in cfg else '1'
 		wf_arg = ([],[wf])[bool(wf)]
 		t = MMGenExpect(name,'mmgen-walletchk',
 				add_args+args+['-p',hp]+wf_arg,
 				extra_desc=extra_desc)
-		if ni:
+		if ia:
 			if sid:
 				n = (' should be','')[desc=='MMGen wallet']
 				m = grnbg('Seed ID%s:' % n)
@@ -1266,13 +1302,13 @@ class MMGenTestSuite(object):
 		MMGenExpect(name,'')
 		global have_dfl_wallet
 		have_dfl_wallet = False
-		if not ni: ok()
+		if not ia: ok()
 
 	def addrgen(self,name,wf,pf=None,check_ref=False):
-		add_args = ([],['-q'] + ([],['-P',pf])[bool(pf)])[ni]
+		add_args = ([],['-q'] + ([],['-P',pf])[bool(pf)])[ia]
 		t = MMGenExpect(name,'mmgen-addrgen', add_args +
 				['-d',cfg['tmpdir']] + ([],[wf])[bool(wf)] + [cfg['addr_idx_list']])
-		if ni: return
+		if ia: return
 		t.license()
 		t.passphrase('MMGen wallet',cfg['wpasswd'])
 		t.expect('Passphrase is OK')
@@ -1291,11 +1327,11 @@ class MMGenTestSuite(object):
 		self.addrgen(name,wf,pf=pf,check_ref=True)
 
 	def addrimport(self,name,addrfile):
-		add_args = ([],['-q','-t'])[ni]
+		add_args = ([],['-q','-t'])[ia]
 		outfile = os.path.join(cfg['tmpdir'],'addrfile_w_comments')
 		add_comments_to_addr_file(addrfile,outfile)
 		t = MMGenExpect(name,'mmgen-addrimport', add_args + [outfile])
-		if ni: return
+		if ia: return
 		t.expect_getend(r'Checksum for address data .*\[.*\]: ',regex=True)
 		t.expect_getend('Validating addresses...OK. ')
 		t.expect("Type uppercase 'YES' to confirm: ",'\n')
@@ -1357,13 +1393,13 @@ class MMGenTestSuite(object):
 		end_silence()
 		if opt.verbose or opt.exact_output: sys.stderr.write('\n')
 
-		add_args = ([],['-q'])[ni]
-		if ni:
+		add_args = ([],['-q'])[ia]
+		if ia:
 			m = '\nAnswer the interactive prompts as follows:\n' + \
 				" 'y', 'y', 'q', '1-9'<ENTER>, ENTER, ENTER, ENTER, ENTER, 'y'"
 			msg(grnbg(m))
 		t = MMGenExpect(name,'mmgen-txcreate',['-f','0.0001'] + add_args + cmd_args)
-		if ni: return
+		if ia: return
 		t.license()
 		for num in tx_data:
 			t.expect_getend('Getting address data from file ')
@@ -1408,12 +1444,12 @@ class MMGenTestSuite(object):
 		t.written_to_file('Signed transaction' + add, oo=True)
 
 	def txsign(self,name,txfile,wf,pf='',save=True,has_label=False):
-		add_args = ([],['-q','-P',pf])[ni]
-		if ni:
+		add_args = ([],['-q','-P',pf])[ia]
+		if ia:
 			m = '\nAnswer the interactive prompts as follows:\n  ENTER, ENTER, ENTER'
 			msg(grnbg(m))
 		t = MMGenExpect(name,'mmgen-txsign', add_args+['-d',cfg['tmpdir'],txfile]+([],[wf])[bool(wf)])
-		if ni: return
+		if ia: return
 		t.license()
 		t.tx_view()
 		t.passphrase('MMGen wallet',cfg['wpasswd'])
@@ -1443,7 +1479,7 @@ class MMGenTestSuite(object):
 		opts = ['-d',cfg['tmpdir'],'-o',out_fmt] + uargs + \
 			([],[wf])[bool(wf)] + ([],['-P',pf])[bool(pf)]
 		t = MMGenExpect(name,'mmgen-walletconv',opts)
-		if ni: return
+		if ia: return
 		t.license()
 		if not pf:
 			t.passphrase('MMGen wallet',cfg['wpasswd'])
@@ -1469,7 +1505,7 @@ class MMGenTestSuite(object):
 
 	def export_seed(self,name,wf,desc='seed data',out_fmt='seed',pf=None):
 		f = self.walletconv_export(name,wf,desc=desc,out_fmt=out_fmt,pf=pf)
-		if ni: return
+		if ia: return
 		silence()
 		msg('%s: %s' % (capfirst(desc),cyan(get_data_from_file(f,desc))))
 		end_silence()
@@ -1537,12 +1573,12 @@ class MMGenTestSuite(object):
 
 	def keyaddrgen(self,name,wf,pf=None,check_ref=False):
 		args = ['-d',cfg['tmpdir'],usr_rand_arg,wf,cfg['addr_idx_list']]
-		if ni:
+		if ia:
 			m = "\nAnswer 'n' at the interactive prompt"
 			msg(grnbg(m))
 			args = ['-q'] + ([],['-P',pf])[bool(pf)] + args
 		t = MMGenExpect(name,'mmgen-keygen', args)
-		if ni: return
+		if ia: return
 		t.license()
 		t.passphrase('MMGen wallet',cfg['wpasswd'])
 		chk = t.expect_getend(r'Checksum for key-address data .*?: ',regex=True)
@@ -1655,7 +1691,7 @@ class MMGenTestSuite(object):
 			tmp_fn = cfg['tool_enc_infn']
 			write_to_tmpfile(cfg,tmp_fn,d,binary=True)
 			infn = get_tmpfile_fn(cfg,tmp_fn)
-		if ni:
+		if ia:
 			pwfn = 'ni_pw'
 			write_to_tmpfile(cfg,pwfn,tool_enc_passwd+'\n')
 			pre = ['-P', get_tmpfile_fn(cfg,pwfn)]
@@ -1663,7 +1699,7 @@ class MMGenTestSuite(object):
 		else:
 			pre,app = [],[]
 		t = MMGenExpect(name,'mmgen-tool',pre+['-d',cfg['tmpdir'],usr_rand_arg,'encrypt',infn]+app)
-		if ni: return
+		if ia: return
 		t.usr_rand(usr_rand_chars)
 		t.hash_preset('user data','1')
 		t.passphrase_new('user data',tool_enc_passwd)
@@ -1678,26 +1714,26 @@ class MMGenTestSuite(object):
 
 	def tool_decrypt(self,name,f1,f2):
 		of = name + '.out'
-		if ni:
+		if ia:
 			pwfn = 'ni_pw'
 			pre = ['-P', get_tmpfile_fn(cfg,pwfn)]
 		else:
 			pre = []
 		t = MMGenExpect(name,'mmgen-tool',
 			pre+['-d',cfg['tmpdir'],'decrypt',f2,'outfile='+of,'hash_preset=1'])
-		if not ni:
+		if not ia:
 			t.passphrase('user data',tool_enc_passwd)
 			t.written_to_file('Decrypted data')
 		d1 = read_from_file(f1,binary=True)
 		d2 = read_from_file(get_tmpfile_fn(cfg,of),binary=True)
-		cmp_or_die(d1,d2,skip_ok=ni)
+		cmp_or_die(d1,d2,skip_ok=ia)
 
 	def tool_find_incog_data(self,name,f1,f2):
 		i_id = read_from_file(f2).rstrip()
 		vmsg('Incog ID: %s' % cyan(i_id))
 		t = MMGenExpect(name,'mmgen-tool',
 				['-d',cfg['tmpdir'],'find_incog_data',f1,i_id])
-		if ni: return
+		if ia: return
 		o = t.expect_getend('Incog data for ID %s found at offset ' % i_id)
 		os.unlink(f1)
 		cmp_or_die(hincog_offset,int(o))
@@ -1763,7 +1799,7 @@ class MMGenTestSuite(object):
 
 	def ref_wallet_chk(self,name):
 		wf = os.path.join(ref_dir,cfg['ref_wallet'])
-		if ni:
+		if ia:
 			write_to_tmpfile(cfg,pwfile,cfg['wpasswd'])
 			pf = get_tmpfile_fn(cfg,pwfile)
 		else:
@@ -1799,12 +1835,12 @@ class MMGenTestSuite(object):
 					)]
 			slarg = ['-l%s ' % cfg['seed_len']]
 			hparg = ['-p1']
-			if ni:
+			if ia:
 				write_to_tmpfile(cfg,pwfile,cfg['wpasswd'])
 				add_args = ['-q','-P%s' % get_tmpfile_fn(cfg,pwfile)]
 			else:
 				add_args = []
-			if ni and wtype == 'hic_wallet_old':
+			if ia and wtype == 'hic_wallet_old':
 				m = grnbg("Answer 'y' at the interactive prompt if Seed ID is")
 				n = cyan(cfg['seed_id'])
 				msg('\n%s %s' % (m,n))
@@ -1812,7 +1848,7 @@ class MMGenTestSuite(object):
 			t = MMGenExpect(name,'mmgen-walletchk',
 				add_args + slarg + hparg + of_arg + ic_arg,
 				extra_desc=edesc)
-			if ni: continue
+			if ia: continue
 			t.passphrase(desc,cfg['wpasswd'])
 			if wtype == 'hic_wallet_old':
 				t.expect('Is the Seed ID correct? (Y/n): ','\n')
@@ -1822,7 +1858,7 @@ class MMGenTestSuite(object):
 
 	def ref_addrfile_chk(self,name,ftype='addr'):
 		wf = os.path.join(ref_dir,cfg['ref_'+ftype+'file'])
-		if ni:
+		if ia:
 			m = "\nAnswer the interactive prompts as follows: '1'<ENTER>, ENTER"
 			msg(grnbg(m))
 			pfn = 'ref_kafile_passwd'
@@ -1831,7 +1867,7 @@ class MMGenTestSuite(object):
 		else:
 			aa = []
 		t = MMGenExpect(name,'mmgen-tool',aa+[ftype+'file_chksum',wf])
-		if ni:
+		if ia:
 			k = 'ref_%saddrfile_chksum' % ('','key')[ftype == 'keyaddr']
 			m = grnbg('Checksum should be:')
 			n = cyan(cfg[k])
@@ -1861,15 +1897,15 @@ class MMGenTestSuite(object):
 	def ref_tool_decrypt(self,name):
 		f = os.path.join(ref_dir,ref_enc_fn)
 		aa = []
-		if ni:
+		if ia:
 			pfn = 'tool_enc_passwd'
 			write_to_tmpfile(cfg,pfn,tool_enc_passwd)
 			aa = ['-P',get_tmpfile_fn(cfg,pfn)]
 		t = MMGenExpect(name,'mmgen-tool',
 				aa + ['-q','decrypt',f,'outfile=-','hash_preset=1'])
-		if ni: return
+		if ia: return
 		t.passphrase('user data',tool_enc_passwd)
-		t.readline()
+		t.expect(NL,nonl=True)
 		import re
 		o = re.sub('\r\n','\n',t.read())
 		cmp_or_die(sample_text,o)
@@ -1879,7 +1915,7 @@ class MMGenTestSuite(object):
 		opts = ['-d',cfg['tmpdir'],'-o','words',usr_rand_arg]
 		if_arg = [infile] if infile else []
 		d = '(convert)'
-		if ni:
+		if ia:
 			opts += ['-q']
 			msg('')
 			if pw:
@@ -1894,7 +1930,7 @@ class MMGenTestSuite(object):
 				n = cyan(cfg['seed_id'])
 				msg('\n%s %s' % (m,n))
 		t = MMGenExpect(name,'mmgen-walletconv',opts+uopts+if_arg,extra_desc=d)
-		if ni:
+		if ia:
 			m = grnbg('Seed ID should be:')
 			n = cyan(cfg['seed_id'])
 			msg(grnbg('%s %s' % (m,n)))
@@ -1922,7 +1958,7 @@ class MMGenTestSuite(object):
 
 	def walletconv_out(self,name,desc,out_fmt='w',uopts=[],uopts_chk=[],pw=False):
 		opts = ['-d',cfg['tmpdir'],'-p1','-o',out_fmt] + uopts
-		if ni:
+		if ia:
 			pfn = 'ni_passwd'
 			write_to_tmpfile(cfg,pfn,cfg['wpasswd'])
 			l = 'Non-Interactive Test Wallet'
@@ -1936,7 +1972,7 @@ class MMGenTestSuite(object):
 		t = MMGenExpect(name,'mmgen-walletconv',aa+opts+[infile],extra_desc='(convert)')
 
 		add_args = ['-l%s' % cfg['seed_len']]
-		if ni:
+		if ia:
 			pfn = 'ni_passwd'
 			write_to_tmpfile(cfg,pfn,cfg['wpasswd'])
 			pf = get_tmpfile_fn(cfg,pfn)
@@ -2035,7 +2071,7 @@ start_time = int(time.time())
 def end_msg():
 	t = int(time.time()) - start_time
 	m1 = 'All requested tests finished OK, elapsed time: {:02d}:{:02d}\n'
-	m2 = ('','Please re-check all {} control values against the program output.\n'.format(grnbg('HIGHLIGHTED')))[ni]
+	m2 = ('','Please re-check all {} control values against the program output.\n'.format(grnbg('HIGHLIGHTED')))[ia]
 	sys.stderr.write(green(m1.format(t/60,t%60)))
 	sys.stderr.write(m2)
 

test/tooltest.py

@@ -28,8 +28,6 @@ sys.path.__setitem__(0,os.path.abspath(os.curdir))
 # Import this _after_ local path's been added to sys.path
 from mmgen.common import *
 
-start_mscolor()
-
 from collections import OrderedDict
 cmd_data = OrderedDict([
 	('util', {