Support MSWin via MSYS2

This is a work in progress.  Currently, basic operations for BTC and ETH are
supported.

The successor to the MinGW64 project, MSYS2 features package management via
`pacman` and support for Python 3:

    https://sourceforge.net/projects/msys2
    https://www.msys2.org

mmgen/globalvars.py

@@ -102,6 +102,7 @@ class g(object):
 	debug_utf8           = False
 	traceback            = False
 	test_suite           = False
+	test_suite_popen_spawn = False
 
 	for k in ('linux','win','msys'):
 		if sys.platform[:len(k)] == k:
@@ -166,6 +167,7 @@ class g(object):
 		'MMGEN_DEBUG_ALL', # special: there is no g.debug_all var
 
 		'MMGEN_TEST_SUITE',
+		'MMGEN_TEST_SUITE_POPEN_SPAWN',
 		'MMGEN_BOGUS_WALLET_DATA',
 		'MMGEN_BOGUS_SEND',
 		'MMGEN_DEBUG',

mmgen/main.py

@@ -28,8 +28,13 @@ def launch(what):
 
 	import sys
 
-	try: import termios
-	except: # Windows
+	try:
+		import termios
+		platform = 'linux'
+	except:
+		platform = 'win'
+
+	if platform == 'win':
 		__import__('mmgen.main_' + what)
 	else:
 		import os,atexit

mmgen/regtest.py

@@ -235,14 +235,22 @@ def setup():
 
 def get_current_user_win(quiet=False):
 	if test_daemon() == 'stopped': return None
-	p = start_cmd('grep','Using wallet',os.path.join(daemon_dir,'debug.log'),quiet=True)
-	try: wallet_fn = p.stdout.readlines()[-1].split()[-1].decode()
-	except: return None
-	for k in ('miner','bob','alice'):
-		if wallet_fn == 'wallet.dat.'+k:
-			if not quiet: msg('Current user is {}'.format(k.capitalize()))
-			return k
-	return None
+	logfile = os.path.join(daemon_dir,'debug.log')
+	p = start_cmd('grep','Wallet completed loading in',logfile,quiet=True)
+	last_line = p.stdout.readlines()[-1].decode()
+
+	import re
+	m = re.search(r'\[wallet.dat.([a-z]+)\]',last_line)
+	if not m:
+		return None
+
+	user = m.group(1)
+	if user in ('miner','bob','alice'):
+		if not quiet:
+			msg('Current user is {}'.format(user.capitalize()))
+		return user
+	else:
+		return None
 
 def get_current_user_unix(quiet=False):
 	p = start_cmd('pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,rpc_port),quiet=True)
@@ -254,7 +262,7 @@ def get_current_user_unix(quiet=False):
 			return k
 	return None
 
-get_current_user = (get_current_user_win,get_current_user_unix)[g.platform=='linux']
+get_current_user = { 'win':get_current_user_win, 'linux':get_current_user_unix }[g.platform]
 
 def bob():   return user('bob',quiet=False)
 def alice(): return user('alice',quiet=False)

mmgen/term.py

@@ -133,12 +133,12 @@ def _get_keypress_mswin(prompt='',immed_chars='',prehold_protect=True,num_chars=
 def _get_keypress_mswin_raw(prompt='',immed_chars='',prehold_protect=None,num_chars=None):
 	msg_r(prompt)
 	ch = msvcrt.getch()
-	if ord(ch) == 3: raise KeyboardInterrupt
+	if ch == 3: raise KeyboardInterrupt
 	return ch
 
 def _get_keypress_mswin_stub(prompt='',immed_chars='',prehold_protect=None,num_chars=None):
 	msg_r(prompt)
-	return sys.stdin.read(1)
+	return os.read(0,1)
 
 def _get_terminal_size_linux():
 	try:

mmgen/util.py

@@ -634,9 +634,12 @@ def write_data_to_file( outfile,data,desc='data',
 					m = "{} in file '{}' has been altered by some other program!  Aborting file write"
 					die(3,m.format(desc,outfile))
 
-		f = open_file_or_exit(outfile,('w','wb')[bool(binary)])
+		# To maintain portability, always open files in binary mode
+		# If 'binary' option not set, encode/decode data before writing and after reading
+		f = open_file_or_exit(outfile,'wb')
+
 		try:
-			f.write(data)
+			f.write(data if binary else data.encode())
 		except:
 			die(2,"Failed to write {} to file '{}'".format(desc,outfile))
 		f.close
@@ -654,7 +657,6 @@ def write_data_to_file( outfile,data,desc='data',
 		do_file(outfile,ask_write_prompt)
 
 def get_words_from_user(prompt):
-	# split() also strips
 	words = my_raw_input(prompt, echo=opt.echo_passphrase).split()
 	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
 	return words
@@ -662,8 +664,8 @@ def get_words_from_user(prompt):
 def get_words_from_file(infile,desc,quiet=False):
 	if not quiet:
 		qmsg("Getting {} from file '{}'".format(desc,infile))
-	f = open_file_or_exit(infile, 'r')
-	try: words = f.read().split() # split() also strips
+	f = open_file_or_exit(infile, 'rb')
+	try: words = f.read().decode().split()
 	except: die(1,'{} data must be UTF-8 encoded.'.format(capfirst(desc)))
 	f.close()
 	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
@@ -687,7 +689,7 @@ def mmgen_decrypt_file_maybe(fn,desc='',quiet=False,silent=False):
 
 def get_lines_from_file(fn,desc='',trim_comments=False,quiet=False,silent=False):
 	dec = mmgen_decrypt_file_maybe(fn,desc,quiet=quiet,silent=silent)
-	ret = dec.decode('utf8').splitlines() # DOS-safe
+	ret = dec.decode().splitlines()
 	if trim_comments: ret = remove_comments(ret)
 	dmsg("Got {} lines from file '{}'".format(len(ret),fn))
 	return ret
@@ -703,12 +705,13 @@ def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,q
 	if not opt.quiet and not silent and not quiet and desc:
 		qmsg("Getting {} from file '{}'".format(desc,infile))
 
-	mode = ('r','rb')[bool(binary)]
-
 	if dash and infile == '-':
-		data = os.fdopen(0,mode).read(g.max_input_size+1)
+		data = os.fdopen(0,'rb').read(g.max_input_size+1)
 	else:
-		data = open_file_or_exit(infile,mode,silent=silent).read(g.max_input_size+1)
+		data = open_file_or_exit(infile,'rb',silent=silent).read(g.max_input_size+1)
+
+	if not binary:
+		data = data.decode()
 
 	if len(data) == g.max_input_size + 1:
 		raise MaxInputSizeExceeded('Too much input data!  Max input data size: {} bytes'.format(g.max_input_size))
@@ -744,11 +747,16 @@ def my_raw_input(prompt,echo=True,insert_txt='',use_readline=True):
 
 	from mmgen.term import kb_hold_protect
 	kb_hold_protect()
-	if echo or not sys.stdin.isatty():
+
+	if g.test_suite_popen_spawn:
+		msg(prompt)
+		reply = os.read(0,4096).decode()
+	elif echo or not sys.stdin.isatty():
 		reply = input(prompt)
 	else:
 		from getpass import getpass
 		reply = getpass(prompt)
+
 	kb_hold_protect()
 
 	try:

scripts/traceback_run.py

@@ -4,7 +4,7 @@
 # file, as all names will be seen by the exec'ed file.  To prevent name collisions, all names
 # defined here should begin with 'traceback_run_'
 
-import sys,time
+import sys,os,time
 
 def traceback_run_init():
 	import os
@@ -13,7 +13,7 @@ def traceback_run_init():
 	if 'TMUX' in os.environ: del os.environ['TMUX']
 	os.environ['MMGEN_TRACEBACK'] = '1'
 
-	of = os.path.join(os.environ['PWD'],'my.err')
+	of = 'my.err'
 	try: os.unlink(of)
 	except: pass
 
@@ -28,10 +28,12 @@ def traceback_run_process_exception():
 
 	exc = l.pop()
 	if exc[:11] == 'SystemExit:': l.pop()
-
-	red    = lambda s: '\033[31;1m{}\033[0m'.format(s)
-	yellow = lambda s: '\033[33;1m{}\033[0m'.format(s)
-	sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc)))
+	if os.getenv('MMGEN_DISABLE_COLOR'):
+		sys.stdout.write('{}{}'.format(''.join(l),exc))
+	else:
+		red    = lambda s: '\033[31;1m{}\033[0m'.format(s)
+		yellow = lambda s: '\033[33;1m{}\033[0m'.format(s)
+		sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc)))
 
 	open(traceback_run_outfile,'w').write(''.join(l+[exc]))
 
@@ -50,10 +52,5 @@ except Exception as e:
 	traceback_run_process_exception()
 	sys.exit(e.mmcode if hasattr(e,'mmcode') else e.code if hasattr(e,'code') else 1)
 
-blue = lambda s: '\033[34;1m{}\033[0m'.format(s)
+blue = lambda s: s if os.getenv('MMGEN_DISABLE_COLOR') else '\033[34;1m{}\033[0m'.format(s)
 sys.stdout.write(blue('Runtime: {:0.5f} secs\n'.format(time.time() - traceback_run_tstart)))
-
-# else:
-# 	print('else: '+repr(sys.exc_info()))
-# finally:
-# 	print('finally: '+repr(sys.exc_info()))

setup.py

@@ -26,43 +26,8 @@ if ver[0] < min_ver[0] or ver[1] < min_ver[1]:
 	sys.stderr.write(m.format(*ver,M=min_ver[0],m=min_ver[1]))
 	sys.exit(1)
 
-_gvi = subprocess.check_output(['gcc','--version']).decode().splitlines()[0]
-have_mingw64 = 'x86_64' in _gvi and 'MinGW' in _gvi
-have_arm     = subprocess.check_output(['uname','-m']).strip() == 'aarch64'
-have_msys2   = not have_mingw64 and os.getenv('MSYSTEM') == 'MINGW64'
-
-# Zipfile module under Windows (MinGW) can't handle UTF-8 filenames.
-# Move it so that distutils will use the 'zip' utility instead.
-def divert_zipfile_module():
-	msg1 = 'Unable to divert zipfile module. UTF-8 filenames may be broken in the Python archive.'
-	def return_warn(m):
-		sys.stderr.write('WARNING: {}\n'.format(m))
-		return False
-
-	dirname = os.path.dirname(sys.modules['os'].__file__)
-	if not dirname: return return_warn(msg1)
-	stem = os.path.join(dirname,'zipfile')
-	a,b = stem+'.py',stem+'-is-broken.py'
-
-	try: os.stat(a)
-	except: return
-
-	try:
-		sys.stderr.write('moving {} -> {}\n'.format(a,b))
-		os.rename(a,b)
-	except:
-		return return_warn(msg1)
-	else:
-		try:
-			os.unlink(stem+'.pyc')
-			os.unlink(stem+'.pyo')
-		except:
-			pass
-
-if have_mingw64:
-# 	import zipfile
-# 	sys.exit()
-	divert_zipfile_module()
+have_arm   = subprocess.check_output(['uname','-m']).strip() == b'aarch64'
+have_msys2 = subprocess.check_output(['uname','-s']).strip()[:7] == b'MSYS_NT'
 
 from distutils.core import setup,Extension
 from distutils.command.build_ext import build_ext
@@ -92,7 +57,6 @@ module1 = Extension(
 	libraries    = ['secp256k1'],
 	library_dirs = ['/usr/local/lib',r'c:\msys\local\lib'],
 	# mingw32 needs this, Linux can use it, but it breaks mingw64
-	extra_link_args = (['-lgmp'],[])[have_mingw64],
 	include_dirs = ['/usr/local/include',r'c:\msys\local\include'],
 	)
 

test/pexpect.py

@@ -37,12 +37,7 @@ def debug_pexpect_msg(p):
 		msg('\n{}{}{}'.format(red('BEFORE ['),p.before,red(']')))
 		msg('{}{}{}'.format(red('MATCH ['),p.after,red(']')))
 
-if g.platform == 'linux' and not opt.pexpect_spawn:
-	import atexit
-	atexit.register(lambda: os.system('stty sane'))
-	NL = '\n'
-else:
-	NL = '\r\n'
+NL = '\n'
 
 class MMGenPexpect(object):
 

test/test.py

@@ -42,6 +42,7 @@ def create_shm_dir(data_dir,trash_dir):
 						time.sleep(2)
 						shutil.rmtree(tdir)
 			os.mkdir(tdir,0o755)
+		shm_dir = 'test'
 	else:
 		tdir,pfx = '/dev/shm','mmgen-test-'
 		try:
@@ -566,6 +567,9 @@ class TestSuiteRunner(object):
 
 		args = [cmd] + passthru_opts + ['--data-dir='+self.data_dir] + args
 
+		if opt.traceback:
+			args = ['scripts/traceback_run.py'] + args
+
 		if g.platform == 'win':
 			args = ['python3'] + args
 
@@ -576,8 +580,6 @@ class TestSuiteRunner(object):
 
 		if opt.coverage:
 			args = ['python3','-m','trace','--count','--coverdir='+self.coverdir,'--file='+self.accfile] + args
-		elif opt.traceback:
-			args = ['scripts/traceback_run.py'] + args
 
 		qargs = ['{q}{}{q}'.format(a,q=('',"'")[' ' in a]) for a in args]
 		cmd_disp = ' '.join(qargs).replace('\\','/') # for mingw
@@ -594,7 +596,11 @@ class TestSuiteRunner(object):
 
 		if msg_only: return
 
-		if opt.log: self.log_fd.write(cmd_disp+'\n')
+		if opt.log:
+			try:
+				self.log_fd.write(cmd_disp+'\n')
+			except:
+				self.log_fd.write(ascii(cmd_disp)+'\n')
 
 		from test.pexpect import MMGenPexpect
 		return MMGenPexpect(args,no_output=no_output)

test/test_py_d/common.py

@@ -93,10 +93,13 @@ def iqmsg(s):
 def iqmsg_r(s):
 	if not opt.quiet: omsg_r(s)
 
-devnull_fh = open('/dev/null','w')
-def silence():
-	if not (opt.verbose or opt.exact_output):
-		g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno()
+if g.platform == 'win':
+	def silence(): pass
+else:
+	devnull_fh = open('/dev/null','w')
+	def silence():
+		if not (opt.verbose or opt.exact_output):
+			g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno()
 
 def end_silence():
 	if not (opt.verbose or opt.exact_output):

test/test_py_d/ts_ethdev.py

@@ -49,21 +49,18 @@ try:
 					subprocess.Popen(['solc','--version'],stdout=subprocess.PIPE
 						).stdout.read().decode()).group(1)
 except:
-	solc_ver = ''
-
+	solc_ver = '' # no solc on system - prompt for precompiled v0.5.3 contract files
 
 if re.match(r'\b0.5.1\b',solc_ver): # Raspbian Stretch
 	vbal1 = '1.2288337'
 	vbal2 = '99.997085083'
 	vbal3 = '1.23142165'
 	vbal4 = '127.0287837'
-elif re.match(r'\b0.5.3\b',solc_ver): # Ubuntu Bionic
+elif solc_ver == '' or re.match(r'\b0.5.3\b',solc_ver): # Ubuntu Bionic
 	vbal1 = '1.2288487'
 	vbal2 = '99.997092733'
 	vbal3 = '1.23142915'
 	vbal4 = '127.0287987'
-else:
-	vbal1 = vbal2 = vbal3 = vbal4 = None
 
 bals = {
 	'1': [  ('98831F3A:E:1','123.456')],
@@ -263,8 +260,16 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def setup(self):
 		self.spawn('',msg_only=True)
 		os.environ['MMGEN_BOGUS_WALLET_DATA'] = ''
-		if subprocess.call(['which','parity'],stdout=subprocess.PIPE) == 0:
-			lf_arg = '--log-file=' + joinpath(self.tr.data_dir,'parity.log')
+		opts = ['--ports-shift=4','--config=dev']
+		lf_arg = '--log-file=' + joinpath(self.tr.data_dir,'parity.log')
+		if g.platform == 'win':
+			dc_dir = joinpath(os.environ['LOCALAPPDATA'],'Parity','Ethereum','chains','DevelopmentChain')
+			shutil.rmtree(dc_dir,ignore_errors=True)
+			m1 = 'Please start parity on another terminal as follows:\n'
+			m2 = ['parity',lf_arg] + opts
+			m3 = '\nPress ENTER to continue: '
+			my_raw_input(m1 + ' '.join(m2) + m3)
+		elif subprocess.call(['which','parity'],stdout=subprocess.PIPE) == 0:
 			ss = 'parity.*--log-file=test/data_dir.*/parity.log' # allow for UTF8_DEBUG
 			try:
 				pid = subprocess.check_output(['pgrep','-af',ss]).split()[0]
@@ -276,7 +281,6 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			bdir = joinpath(self.tr.data_dir,'parity')
 			try: os.mkdir(bdir)
 			except: pass
-			opts = ['--ports-shift=4','--config=dev']
 			redir = None if opt.exact_output else subprocess.PIPE
 			pidfile = joinpath(self.tmpdir,parity_pid_fn)
 			subprocess.check_call(['parity',lf_arg] + opts + ['daemon',pidfile],stderr=redir,stdout=redir)
@@ -483,10 +487,17 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		return t
 
 	def token_compile(self,token_data={}):
+		odir = joinpath(self.tmpdir,token_data['symbol'].lower())
+		if self.skip_for_win():
+			m ='Copy solc v0.5.3 contract data for token {} to directory {} and hit ENTER: '
+			input(m.format(token_data['symbol'],odir))
+			return 'skip'
 		self.spawn('',msg_only=True)
 		cmd_args = ['--{}={}'.format(k,v) for k,v in list(token_data.items())]
 		imsg("Compiling solidity token contract '{}' with 'solc'".format(token_data['symbol']))
-		cmd = ['scripts/create-token.py','--coin='+g.coin,'--outdir='+self.tmpdir] + cmd_args + [dfl_addr_chk]
+		try: os.mkdir(odir)
+		except: pass
+		cmd = ['scripts/create-token.py','--coin='+g.coin,'--outdir='+odir] + cmd_args + [dfl_addr_chk]
 		imsg("Executing: {}".format(' '.join(cmd)))
 		subprocess.check_output(cmd,stderr=subprocess.STDOUT)
 		imsg("ERC20 token '{}' compiled".format(token_data['symbol']))
@@ -507,7 +518,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'):
 		self._rpc_init()
 		keyfile = joinpath(self.tmpdir,parity_key_fn)
-		fn = joinpath(self.tmpdir,key+'.bin')
+		fn = joinpath(self.tmpdir,'mm'+str(num),key+'.bin')
 		os.environ['MMGEN_BOGUS_SEND'] = ''
 		args = ['-B',
 				'--tx-fee='+tx_fee,
@@ -534,7 +545,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			"Contract '{}:{}' failed to execute. Aborting".format(num,key))
 		if key == 'Token':
 			self.write_to_tmpfile('token_addr{}'.format(num),addr+'\n')
-			imsg('\nToken #{} ({}) deployed!'.format(num,addr))
+			imsg('\nToken MM{} deployed!'.format(num))
 		return t
 
 	def token_deploy1a(self): return self.token_deploy(num=1,key='SafeMath',gas=200000)