From dcab10949ec8c20fa957a7b9b6f338bff16c68c1 Mon Sep 17 00:00:00 2001 From: MMGen Date: Mon, 25 Mar 2019 10:07:04 +0000 Subject: [PATCH] Support MSWin via MSYS2 This is a work in progress. Currently, basic operations for BTC and ETH are supported. The successor to the MinGW64 project, MSYS2 features package management via `pacman` and support for Python 3: https://sourceforge.net/projects/msys2 https://www.msys2.org --- mmgen/globalvars.py | 2 ++ mmgen/main.py | 9 +++++++-- mmgen/regtest.py | 26 +++++++++++++++--------- mmgen/term.py | 4 ++-- mmgen/util.py | 30 ++++++++++++++++++---------- scripts/traceback_run.py | 21 +++++++++---------- setup.py | 40 ++----------------------------------- test/pexpect.py | 7 +------ test/test.py | 12 ++++++++--- test/test_py_d/common.py | 11 ++++++---- test/test_py_d/ts_ethdev.py | 33 ++++++++++++++++++++---------- 11 files changed, 97 insertions(+), 98 deletions(-) diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index 1603f321..74758b3a 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -102,6 +102,7 @@ class g(object): debug_utf8 = False traceback = False test_suite = False + test_suite_popen_spawn = False for k in ('linux','win','msys'): if sys.platform[:len(k)] == k: @@ -166,6 +167,7 @@ class g(object): 'MMGEN_DEBUG_ALL', # special: there is no g.debug_all var 'MMGEN_TEST_SUITE', + 'MMGEN_TEST_SUITE_POPEN_SPAWN', 'MMGEN_BOGUS_WALLET_DATA', 'MMGEN_BOGUS_SEND', 'MMGEN_DEBUG', diff --git a/mmgen/main.py b/mmgen/main.py index a3584776..945cab27 100755 --- a/mmgen/main.py +++ b/mmgen/main.py @@ -28,8 +28,13 @@ def launch(what): import sys - try: import termios - except: # Windows + try: + import termios + platform = 'linux' + except: + platform = 'win' + + if platform == 'win': __import__('mmgen.main_' + what) else: import os,atexit diff --git a/mmgen/regtest.py b/mmgen/regtest.py index 9968922b..b2c24d5b 100755 --- a/mmgen/regtest.py +++ b/mmgen/regtest.py @@ -235,14 +235,22 @@ def setup(): def get_current_user_win(quiet=False): if test_daemon() == 'stopped': return None - p = start_cmd('grep','Using wallet',os.path.join(daemon_dir,'debug.log'),quiet=True) - try: wallet_fn = p.stdout.readlines()[-1].split()[-1].decode() - except: return None - for k in ('miner','bob','alice'): - if wallet_fn == 'wallet.dat.'+k: - if not quiet: msg('Current user is {}'.format(k.capitalize())) - return k - return None + logfile = os.path.join(daemon_dir,'debug.log') + p = start_cmd('grep','Wallet completed loading in',logfile,quiet=True) + last_line = p.stdout.readlines()[-1].decode() + + import re + m = re.search(r'\[wallet.dat.([a-z]+)\]',last_line) + if not m: + return None + + user = m.group(1) + if user in ('miner','bob','alice'): + if not quiet: + msg('Current user is {}'.format(user.capitalize())) + return user + else: + return None def get_current_user_unix(quiet=False): p = start_cmd('pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,rpc_port),quiet=True) @@ -254,7 +262,7 @@ def get_current_user_unix(quiet=False): return k return None -get_current_user = (get_current_user_win,get_current_user_unix)[g.platform=='linux'] +get_current_user = { 'win':get_current_user_win, 'linux':get_current_user_unix }[g.platform] def bob(): return user('bob',quiet=False) def alice(): return user('alice',quiet=False) diff --git a/mmgen/term.py b/mmgen/term.py index fa62da7a..33bfd704 100755 --- a/mmgen/term.py +++ b/mmgen/term.py @@ -133,12 +133,12 @@ def _get_keypress_mswin(prompt='',immed_chars='',prehold_protect=True,num_chars= def _get_keypress_mswin_raw(prompt='',immed_chars='',prehold_protect=None,num_chars=None): msg_r(prompt) ch = msvcrt.getch() - if ord(ch) == 3: raise KeyboardInterrupt + if ch == 3: raise KeyboardInterrupt return ch def _get_keypress_mswin_stub(prompt='',immed_chars='',prehold_protect=None,num_chars=None): msg_r(prompt) - return sys.stdin.read(1) + return os.read(0,1) def _get_terminal_size_linux(): try: diff --git a/mmgen/util.py b/mmgen/util.py index 5cd8779f..4bee1489 100755 --- a/mmgen/util.py +++ b/mmgen/util.py @@ -634,9 +634,12 @@ def write_data_to_file( outfile,data,desc='data', m = "{} in file '{}' has been altered by some other program! Aborting file write" die(3,m.format(desc,outfile)) - f = open_file_or_exit(outfile,('w','wb')[bool(binary)]) + # To maintain portability, always open files in binary mode + # If 'binary' option not set, encode/decode data before writing and after reading + f = open_file_or_exit(outfile,'wb') + try: - f.write(data) + f.write(data if binary else data.encode()) except: die(2,"Failed to write {} to file '{}'".format(desc,outfile)) f.close @@ -654,7 +657,6 @@ def write_data_to_file( outfile,data,desc='data', do_file(outfile,ask_write_prompt) def get_words_from_user(prompt): - # split() also strips words = my_raw_input(prompt, echo=opt.echo_passphrase).split() dmsg('Sanitized input: [{}]'.format(' '.join(words))) return words @@ -662,8 +664,8 @@ def get_words_from_user(prompt): def get_words_from_file(infile,desc,quiet=False): if not quiet: qmsg("Getting {} from file '{}'".format(desc,infile)) - f = open_file_or_exit(infile, 'r') - try: words = f.read().split() # split() also strips + f = open_file_or_exit(infile, 'rb') + try: words = f.read().decode().split() except: die(1,'{} data must be UTF-8 encoded.'.format(capfirst(desc))) f.close() dmsg('Sanitized input: [{}]'.format(' '.join(words))) @@ -687,7 +689,7 @@ def mmgen_decrypt_file_maybe(fn,desc='',quiet=False,silent=False): def get_lines_from_file(fn,desc='',trim_comments=False,quiet=False,silent=False): dec = mmgen_decrypt_file_maybe(fn,desc,quiet=quiet,silent=silent) - ret = dec.decode('utf8').splitlines() # DOS-safe + ret = dec.decode().splitlines() if trim_comments: ret = remove_comments(ret) dmsg("Got {} lines from file '{}'".format(len(ret),fn)) return ret @@ -703,12 +705,13 @@ def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,q if not opt.quiet and not silent and not quiet and desc: qmsg("Getting {} from file '{}'".format(desc,infile)) - mode = ('r','rb')[bool(binary)] - if dash and infile == '-': - data = os.fdopen(0,mode).read(g.max_input_size+1) + data = os.fdopen(0,'rb').read(g.max_input_size+1) else: - data = open_file_or_exit(infile,mode,silent=silent).read(g.max_input_size+1) + data = open_file_or_exit(infile,'rb',silent=silent).read(g.max_input_size+1) + + if not binary: + data = data.decode() if len(data) == g.max_input_size + 1: raise MaxInputSizeExceeded('Too much input data! Max input data size: {} bytes'.format(g.max_input_size)) @@ -744,11 +747,16 @@ def my_raw_input(prompt,echo=True,insert_txt='',use_readline=True): from mmgen.term import kb_hold_protect kb_hold_protect() - if echo or not sys.stdin.isatty(): + + if g.test_suite_popen_spawn: + msg(prompt) + reply = os.read(0,4096).decode() + elif echo or not sys.stdin.isatty(): reply = input(prompt) else: from getpass import getpass reply = getpass(prompt) + kb_hold_protect() try: diff --git a/scripts/traceback_run.py b/scripts/traceback_run.py index 3d0c7a78..eadaed79 100755 --- a/scripts/traceback_run.py +++ b/scripts/traceback_run.py @@ -4,7 +4,7 @@ # file, as all names will be seen by the exec'ed file. To prevent name collisions, all names # defined here should begin with 'traceback_run_' -import sys,time +import sys,os,time def traceback_run_init(): import os @@ -13,7 +13,7 @@ def traceback_run_init(): if 'TMUX' in os.environ: del os.environ['TMUX'] os.environ['MMGEN_TRACEBACK'] = '1' - of = os.path.join(os.environ['PWD'],'my.err') + of = 'my.err' try: os.unlink(of) except: pass @@ -28,10 +28,12 @@ def traceback_run_process_exception(): exc = l.pop() if exc[:11] == 'SystemExit:': l.pop() - - red = lambda s: '\033[31;1m{}\033[0m'.format(s) - yellow = lambda s: '\033[33;1m{}\033[0m'.format(s) - sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc))) + if os.getenv('MMGEN_DISABLE_COLOR'): + sys.stdout.write('{}{}'.format(''.join(l),exc)) + else: + red = lambda s: '\033[31;1m{}\033[0m'.format(s) + yellow = lambda s: '\033[33;1m{}\033[0m'.format(s) + sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc))) open(traceback_run_outfile,'w').write(''.join(l+[exc])) @@ -50,10 +52,5 @@ except Exception as e: traceback_run_process_exception() sys.exit(e.mmcode if hasattr(e,'mmcode') else e.code if hasattr(e,'code') else 1) -blue = lambda s: '\033[34;1m{}\033[0m'.format(s) +blue = lambda s: s if os.getenv('MMGEN_DISABLE_COLOR') else '\033[34;1m{}\033[0m'.format(s) sys.stdout.write(blue('Runtime: {:0.5f} secs\n'.format(time.time() - traceback_run_tstart))) - -# else: -# print('else: '+repr(sys.exc_info())) -# finally: -# print('finally: '+repr(sys.exc_info())) diff --git a/setup.py b/setup.py index c05c6c14..17c8a632 100755 --- a/setup.py +++ b/setup.py @@ -26,43 +26,8 @@ if ver[0] < min_ver[0] or ver[1] < min_ver[1]: sys.stderr.write(m.format(*ver,M=min_ver[0],m=min_ver[1])) sys.exit(1) -_gvi = subprocess.check_output(['gcc','--version']).decode().splitlines()[0] -have_mingw64 = 'x86_64' in _gvi and 'MinGW' in _gvi -have_arm = subprocess.check_output(['uname','-m']).strip() == 'aarch64' -have_msys2 = not have_mingw64 and os.getenv('MSYSTEM') == 'MINGW64' - -# Zipfile module under Windows (MinGW) can't handle UTF-8 filenames. -# Move it so that distutils will use the 'zip' utility instead. -def divert_zipfile_module(): - msg1 = 'Unable to divert zipfile module. UTF-8 filenames may be broken in the Python archive.' - def return_warn(m): - sys.stderr.write('WARNING: {}\n'.format(m)) - return False - - dirname = os.path.dirname(sys.modules['os'].__file__) - if not dirname: return return_warn(msg1) - stem = os.path.join(dirname,'zipfile') - a,b = stem+'.py',stem+'-is-broken.py' - - try: os.stat(a) - except: return - - try: - sys.stderr.write('moving {} -> {}\n'.format(a,b)) - os.rename(a,b) - except: - return return_warn(msg1) - else: - try: - os.unlink(stem+'.pyc') - os.unlink(stem+'.pyo') - except: - pass - -if have_mingw64: -# import zipfile -# sys.exit() - divert_zipfile_module() +have_arm = subprocess.check_output(['uname','-m']).strip() == b'aarch64' +have_msys2 = subprocess.check_output(['uname','-s']).strip()[:7] == b'MSYS_NT' from distutils.core import setup,Extension from distutils.command.build_ext import build_ext @@ -92,7 +57,6 @@ module1 = Extension( libraries = ['secp256k1'], library_dirs = ['/usr/local/lib',r'c:\msys\local\lib'], # mingw32 needs this, Linux can use it, but it breaks mingw64 - extra_link_args = (['-lgmp'],[])[have_mingw64], include_dirs = ['/usr/local/include',r'c:\msys\local\include'], ) diff --git a/test/pexpect.py b/test/pexpect.py index e481eb1e..c75082c6 100755 --- a/test/pexpect.py +++ b/test/pexpect.py @@ -37,12 +37,7 @@ def debug_pexpect_msg(p): msg('\n{}{}{}'.format(red('BEFORE ['),p.before,red(']'))) msg('{}{}{}'.format(red('MATCH ['),p.after,red(']'))) -if g.platform == 'linux' and not opt.pexpect_spawn: - import atexit - atexit.register(lambda: os.system('stty sane')) - NL = '\n' -else: - NL = '\r\n' +NL = '\n' class MMGenPexpect(object): diff --git a/test/test.py b/test/test.py index 716ba8b9..9809ab69 100755 --- a/test/test.py +++ b/test/test.py @@ -42,6 +42,7 @@ def create_shm_dir(data_dir,trash_dir): time.sleep(2) shutil.rmtree(tdir) os.mkdir(tdir,0o755) + shm_dir = 'test' else: tdir,pfx = '/dev/shm','mmgen-test-' try: @@ -566,6 +567,9 @@ class TestSuiteRunner(object): args = [cmd] + passthru_opts + ['--data-dir='+self.data_dir] + args + if opt.traceback: + args = ['scripts/traceback_run.py'] + args + if g.platform == 'win': args = ['python3'] + args @@ -576,8 +580,6 @@ class TestSuiteRunner(object): if opt.coverage: args = ['python3','-m','trace','--count','--coverdir='+self.coverdir,'--file='+self.accfile] + args - elif opt.traceback: - args = ['scripts/traceback_run.py'] + args qargs = ['{q}{}{q}'.format(a,q=('',"'")[' ' in a]) for a in args] cmd_disp = ' '.join(qargs).replace('\\','/') # for mingw @@ -594,7 +596,11 @@ class TestSuiteRunner(object): if msg_only: return - if opt.log: self.log_fd.write(cmd_disp+'\n') + if opt.log: + try: + self.log_fd.write(cmd_disp+'\n') + except: + self.log_fd.write(ascii(cmd_disp)+'\n') from test.pexpect import MMGenPexpect return MMGenPexpect(args,no_output=no_output) diff --git a/test/test_py_d/common.py b/test/test_py_d/common.py index 217450d3..b43f4837 100755 --- a/test/test_py_d/common.py +++ b/test/test_py_d/common.py @@ -93,10 +93,13 @@ def iqmsg(s): def iqmsg_r(s): if not opt.quiet: omsg_r(s) -devnull_fh = open('/dev/null','w') -def silence(): - if not (opt.verbose or opt.exact_output): - g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno() +if g.platform == 'win': + def silence(): pass +else: + devnull_fh = open('/dev/null','w') + def silence(): + if not (opt.verbose or opt.exact_output): + g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno() def end_silence(): if not (opt.verbose or opt.exact_output): diff --git a/test/test_py_d/ts_ethdev.py b/test/test_py_d/ts_ethdev.py index d02afc09..318c80b1 100755 --- a/test/test_py_d/ts_ethdev.py +++ b/test/test_py_d/ts_ethdev.py @@ -49,21 +49,18 @@ try: subprocess.Popen(['solc','--version'],stdout=subprocess.PIPE ).stdout.read().decode()).group(1) except: - solc_ver = '' - + solc_ver = '' # no solc on system - prompt for precompiled v0.5.3 contract files if re.match(r'\b0.5.1\b',solc_ver): # Raspbian Stretch vbal1 = '1.2288337' vbal2 = '99.997085083' vbal3 = '1.23142165' vbal4 = '127.0287837' -elif re.match(r'\b0.5.3\b',solc_ver): # Ubuntu Bionic +elif solc_ver == '' or re.match(r'\b0.5.3\b',solc_ver): # Ubuntu Bionic vbal1 = '1.2288487' vbal2 = '99.997092733' vbal3 = '1.23142915' vbal4 = '127.0287987' -else: - vbal1 = vbal2 = vbal3 = vbal4 = None bals = { '1': [ ('98831F3A:E:1','123.456')], @@ -263,8 +260,16 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): def setup(self): self.spawn('',msg_only=True) os.environ['MMGEN_BOGUS_WALLET_DATA'] = '' - if subprocess.call(['which','parity'],stdout=subprocess.PIPE) == 0: - lf_arg = '--log-file=' + joinpath(self.tr.data_dir,'parity.log') + opts = ['--ports-shift=4','--config=dev'] + lf_arg = '--log-file=' + joinpath(self.tr.data_dir,'parity.log') + if g.platform == 'win': + dc_dir = joinpath(os.environ['LOCALAPPDATA'],'Parity','Ethereum','chains','DevelopmentChain') + shutil.rmtree(dc_dir,ignore_errors=True) + m1 = 'Please start parity on another terminal as follows:\n' + m2 = ['parity',lf_arg] + opts + m3 = '\nPress ENTER to continue: ' + my_raw_input(m1 + ' '.join(m2) + m3) + elif subprocess.call(['which','parity'],stdout=subprocess.PIPE) == 0: ss = 'parity.*--log-file=test/data_dir.*/parity.log' # allow for UTF8_DEBUG try: pid = subprocess.check_output(['pgrep','-af',ss]).split()[0] @@ -276,7 +281,6 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): bdir = joinpath(self.tr.data_dir,'parity') try: os.mkdir(bdir) except: pass - opts = ['--ports-shift=4','--config=dev'] redir = None if opt.exact_output else subprocess.PIPE pidfile = joinpath(self.tmpdir,parity_pid_fn) subprocess.check_call(['parity',lf_arg] + opts + ['daemon',pidfile],stderr=redir,stdout=redir) @@ -483,10 +487,17 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): return t def token_compile(self,token_data={}): + odir = joinpath(self.tmpdir,token_data['symbol'].lower()) + if self.skip_for_win(): + m ='Copy solc v0.5.3 contract data for token {} to directory {} and hit ENTER: ' + input(m.format(token_data['symbol'],odir)) + return 'skip' self.spawn('',msg_only=True) cmd_args = ['--{}={}'.format(k,v) for k,v in list(token_data.items())] imsg("Compiling solidity token contract '{}' with 'solc'".format(token_data['symbol'])) - cmd = ['scripts/create-token.py','--coin='+g.coin,'--outdir='+self.tmpdir] + cmd_args + [dfl_addr_chk] + try: os.mkdir(odir) + except: pass + cmd = ['scripts/create-token.py','--coin='+g.coin,'--outdir='+odir] + cmd_args + [dfl_addr_chk] imsg("Executing: {}".format(' '.join(cmd))) subprocess.check_output(cmd,stderr=subprocess.STDOUT) imsg("ERC20 token '{}' compiled".format(token_data['symbol'])) @@ -507,7 +518,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'): self._rpc_init() keyfile = joinpath(self.tmpdir,parity_key_fn) - fn = joinpath(self.tmpdir,key+'.bin') + fn = joinpath(self.tmpdir,'mm'+str(num),key+'.bin') os.environ['MMGEN_BOGUS_SEND'] = '' args = ['-B', '--tx-fee='+tx_fee, @@ -534,7 +545,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): "Contract '{}:{}' failed to execute. Aborting".format(num,key)) if key == 'Token': self.write_to_tmpfile('token_addr{}'.format(num),addr+'\n') - imsg('\nToken #{} ({}) deployed!'.format(num,addr)) + imsg('\nToken MM{} deployed!'.format(num)) return t def token_deploy1a(self): return self.token_deploy(num=1,key='SafeMath',gas=200000)