initialize developer tools via builtins; add test

mmgen/data/version

@@ -1 +1 @@
-13.3.dev6
+13.3.dev7

mmgen/devinit.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet
+# Copyright (C)2013-2022 The MMGen Project <mmgen@tuta.io>
+# Licensed under the GNU General Public License, Version 3:
+#   https://www.gnu.org/licenses
+# Public project repositories:
+#   https://github.com/mmgen/mmgen
+#   https://gitlab.com/mmgen/mmgen
+
+"""
+devinit.py: Developer tools init/launch code for the MMGen suite
+"""
+
+devtools_funcs = {
+	'pfmt':              lambda *args,**kwargs: devtools_call('pfmt',*args,**kwargs),
+	'pmsg':              lambda *args,**kwargs: devtools_call('pmsg',*args,**kwargs),
+	'pdie':              lambda *args,**kwargs: devtools_call('pdie',*args,**kwargs),
+	'pexit':             lambda *args,**kwargs: devtools_call('pexit',*args,**kwargs),
+	'Pmsg':              lambda *args,**kwargs: devtools_call('Pmsg',*args,**kwargs),
+	'Pdie':              lambda *args,**kwargs: devtools_call('Pdie',*args,**kwargs),
+	'Pexit':             lambda *args,**kwargs: devtools_call('Pexit',*args,**kwargs),
+	'print_stack_trace': lambda *args,**kwargs: devtools_call('print_stack_trace',*args,**kwargs),
+	'get_diff':          lambda *args,**kwargs: devtools_call('get_diff',*args,**kwargs),
+	'print_diff':        lambda *args,**kwargs: devtools_call('print_diff',*args,**kwargs),
+	'get_ndiff':         lambda *args,**kwargs: devtools_call('get_ndiff',*args,**kwargs),
+	'print_ndiff':       lambda *args,**kwargs: devtools_call('print_ndiff',*args,**kwargs),
+}
+
+def devtools_call(funcname,*args,**kwargs):
+	import mmgen.devtools
+	return getattr(mmgen.devtools,funcname)(*args,**kwargs)
+
+def MMGenObject_call(methodname,*args,**kwargs):
+	from .devtools import MMGenObjectMethods
+	return getattr(MMGenObjectMethods,methodname)(*args,**kwargs)
+
+class MMGenObject:
+
+	pmsg  = lambda *args,**kwargs: MMGenObject_call('pmsg',*args,**kwargs)
+	pdie  = lambda *args,**kwargs: MMGenObject_call('pdie',*args,**kwargs)
+	pexit = lambda *args,**kwargs: MMGenObject_call('pexit',*args,**kwargs)
+	pfmt  = lambda *args,**kwargs: MMGenObject_call('pfmt',*args,**kwargs)
+
+	# Check that all immutables have been initialized.  Expensive, so do only when testing.
+	def immutable_attr_init_check(self):
+
+		cls = type(self)
+
+		for attrname in self.valid_attrs:
+
+			for o in (cls,cls.__bases__[0]): # assume there's only one base class
+				if attrname in o.__dict__:
+					attr = o.__dict__[attrname]
+					break
+			else:
+				from .util import die
+				die(4,f'unable to find descriptor {cls.__name__}.{attrname}')
+
+			if type(attr).__name__ == 'ImmutableAttr' and attrname not in self.__dict__:
+				from .util import die
+				die(4,f'attribute {attrname!r} of {cls.__name__} has not been initialized in constructor!')
+
+def init_dev():
+	import builtins
+	setattr(builtins,'MMGenObject',MMGenObject)
+	for funcname,func in devtools_funcs.items():
+		setattr(builtins,funcname,func)

mmgen/devtools.py

@@ -1,162 +1,180 @@
 #!/usr/bin/env python3
+#
+# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet
+# Copyright (C)2013-2022 The MMGen Project <mmgen@tuta.io>
+# Licensed under the GNU General Public License, Version 3:
+#   https://www.gnu.org/licenses
+# Public project repositories:
+#   https://github.com/mmgen/mmgen
+#   https://gitlab.com/mmgen/mmgen
 
-class MMGenObject(object):
-	'placeholder - overridden when testing'
-	def immutable_attr_init_check(self): pass
+"""
+devtools.py: Developer tools for the MMGen suite
+"""
 
-import os
-if os.getenv('MMGEN_DEBUG') or os.getenv('MMGEN_TEST_SUITE') or os.getenv('MMGEN_EXEC_WRAPPER'):
+import sys
 
-	import sys,re,traceback,json,pprint
-	from decimal import Decimal
-	from difflib import unified_diff,ndiff
+def pfmt(*args):
+	import pprint
+	return (
+		pprint.PrettyPrinter(indent=4).pformat(
+			args if len(args) > 1 else '' if not args else args[0] )
+		+ '\n' )
 
-	def pmsg(*args,out=sys.stderr):
-		d = args if len(args) > 1 else '' if not args else args[0]
-		out.write(pprint.PrettyPrinter(indent=4).pformat(d) + '\n')
-	def pdie(*args,exit_val=1,out=sys.stderr):
-		pmsg(*args,out=out)
+def pmsg(*args):
+	sys.stderr.write(pfmt(*args))
+
+def pdie(*args,exit_val=1):
+	pmsg(*args)
+	sys.exit(exit_val)
+
+def pexit(*args):
+	pdie(*args,exit_val=0)
+
+def Pmsg(*args):
+	sys.stdout.write(pfmt(*args))
+
+def Pdie(*args,exit_val=1):
+	Pmsg(*args)
+	sys.exit(exit_val)
+
+def Pexit(*args):
+	Pdie(*args,exit_val=0)
+
+def print_stack_trace(message=None,fh_list=[],nl='\n',sep='\n  ',trim=4):
+	import os
+	if not fh_list:
+		fh_list.append(open(f'devtools.trace.{os.getpid()}','w'))
+		nl = ''
+	res = get_stack_trace(message,nl,sep,trim)
+	sys.stderr.write(res)
+	fh_list[0].write(res)
+
+def get_stack_trace(message=None,nl='\n',sep='\n  ',trim=3):
+
+	import os,re,traceback
+
+	tb = [t for t in traceback.extract_stack() if t.filename[:1] != '<']
+	fs = '{}:{}: in {}:\n    {}'
+	out = [
+		fs.format(
+			re.sub(r'^\./','',os.path.relpath(t.filename)),
+			t.lineno,
+			(t.name+'()' if t.name[-1] != '>' else t.name),
+			t.line or '(none)')
+		for t in (tb[:-trim] if trim else tb) ]
+
+	return f'{nl}STACK TRACE {message or "[unnamed]"}:{sep}{sep.join(out)}\n'
+
+def print_diff(*args,**kwargs):
+	sys.stderr.write(get_diff(*args,**kwargs))
+
+def get_diff(a,b,a_fn='',b_fn='',from_json=True):
+
+	if from_json:
+		import json
+		a = json.dumps(json.loads(a),indent=4)
+		b = json.dumps(json.loads(b),indent=4)
+
+	from difflib import unified_diff
+	# chunk headers have trailing newlines, hence the rstrip()
+	return '  DIFF:\n    {}\n'.format(
+		'\n    '.join(a.rstrip('\n') for a in unified_diff(
+			a.split('\n'),
+			b.split('\n'),
+			a_fn,
+			b_fn )))
+
+def print_ndiff(*args,**kwargs):
+	sys.stderr.write(get_ndiff(*args,**kwargs))
+
+def get_ndiff(a,b):
+	from difflib import ndiff
+	return list(ndiff(
+		a.split('\n'),
+		b.split('\n') ))
+
+class MMGenObjectMethods: # mixin class for MMGenObject
+
+	# Pretty-print an MMGenObject instance, recursing into sub-objects - WIP
+	def pmsg(self):
+		sys.stdout.write('\n'+self.pfmt())
+
+	def pdie(self,exit_val=1):
+		self.pmsg()
 		sys.exit(exit_val)
-	def pexit(*args,out=sys.stderr):
-		pdie(*args,exit_val=0,out=out)
 
-	def Pmsg(*args):
-		pmsg(*args,out=sys.stdout)
-	def Pdie(*args):
-		pdie(*args,out=sys.stdout)
-	def Pexit(*args):
-		pexit(*args,out=sys.stdout)
+	def pexit(self):
+		self.pdie(exit_val=0)
 
-	def print_stack_trace(message=None,fh=[],nl='\n',sep='\n  '):
-
-		if not fh:
-			fh.append(open(f'devtools.trace.{os.getpid()}','w'))
-			nl = ''
-
-		tb = [t for t in traceback.extract_stack() if t.filename[:1] != '<'][:-1]
-		fs = '{}:{}: in {}:\n    {}'
-		out = [
-			fs.format(
-				re.sub(r'^\./','',os.path.relpath(t.filename)),
-				t.lineno,
-				(t.name+'()' if t.name[-1] != '>' else t.name),
-				t.line or '(none)')
-			for t in tb ]
-
-		text = f'{nl}STACK TRACE {message or "[unnamed]"}:{sep}{sep.join(out)}\n'
-		sys.stderr.write(text)
-		fh[0].write(text)
-
-	class MMGenObject(object):
-
-		# Pretty-print any object subclassed from MMGenObject, recursing into sub-objects - WIP
-		def pmsg(self,*args):
-			print(args[0] if len(args) == 1 else args if args else self.pfmt())
-
-		def pdie(self,*args):
-			self.pmsg(*args)
-			sys.exit(1)
-
-		def pexit(self,*args):
-			self.pmsg(*args)
-			sys.exit(0)
-
-		def pfmt(self,lvl=0,id_list=[]):
-			scalars = (str,int,float,Decimal)
-			def do_list(out,e,lvl=0,is_dict=False):
-				out.append('\n')
-				for i in e:
-					el = i if not is_dict else e[i]
-					if is_dict:
-						out.append('{s}{:<{l}}'.format(i,s=' '*(4*lvl+8),l=10,l2=8*(lvl+1)+8))
-					if hasattr(el,'pfmt'):
-						out.append('{:>{l}}{}'.format(
-							'',
-							el.pfmt( lvl=lvl+1, id_list=id_list+[id(self)] ),
-							l = (lvl+1)*8 ))
-					elif isinstance(el,scalars):
-						if isList(e):
-							out.append( '{:>{l}}{!r:16}\n'.format( '', el, l=lvl*8 ))
-						else:
-							out.append(f' {el!r}')
-					elif isList(el) or isDict(el):
-						indent = 1 if is_dict else lvl*8+4
-						out.append('{:>{l}}{:16}'.format( '', f'<{type(el).__name__}>', l=indent ))
-						if isList(el) and isinstance(el[0],scalars):
-							out.append('\n')
-						do_list(out,el,lvl=lvl+1,is_dict=isDict(el))
-					else:
-						out.append('{:>{l}}{:16} {!r}\n'.format( '', f'<{type(el).__name__}>', el, l=(lvl*8)+8 ))
-					out.append('\n')
-
-				if not e:
-					out.append(f'{e!r}\n')
-
-			def isDict(obj):
-				return isinstance(obj,dict)
-			def isList(obj):
-				return isinstance(obj,list)
-			def isScalar(obj):
-				return isinstance(obj,scalars)
-
-			out = [f'<{type(self).__name__}>{" "+repr(self) if isScalar(self) else ""}\n']
-
-			if id(self) in id_list:
-				return out[-1].rstrip() + ' [RECURSION]\n'
-			if isList(self) or isDict(self):
-				do_list(out,self,lvl=lvl,is_dict=isDict(self))
-
-			for k in self.__dict__:
-				e = getattr(self,k)
-				if isList(e) or isDict(e):
-					out.append('{:>{l}}{:<10} {:16}'.format( '', k, f'<{type(e).__name__}>', l=(lvl*8)+4 ))
-					do_list(out,e,lvl=lvl,is_dict=isDict(e))
-				elif hasattr(e,'pfmt') and type(e) != type:
-					out.append('{:>{l}}{:10} {}'.format(
+	def pfmt(self,lvl=0,id_list=[]):
+		from decimal import Decimal
+		scalars = (str,int,float,Decimal)
+		def do_list(out,e,lvl=0,is_dict=False):
+			out.append('\n')
+			for i in e:
+				el = i if not is_dict else e[i]
+				if is_dict:
+#					out.append('{s}{:<{l}}'.format(i,s=' '*(4*lvl+8),l=10,l2=8*(lvl+1)+8))
+					out.append('{s1}{i}{s2}'.format(
+						i  = i,
+						s1 = ' ' * (4*lvl+8),
+						s2 = ' ' * 10 ))
+				if hasattr(el,'pfmt'):
+					out.append('{:>{l}}{}'.format(
 						'',
-						k,
-						e.pfmt( lvl=lvl+1, id_list=id_list+[id(self)] ),
-						l = (lvl*8)+4 ))
+						el.pfmt( lvl=lvl+1, id_list=id_list+[id(self)] ),
+						l = (lvl+1)*8 ))
+				elif isinstance(el,scalars):
+					if isList(e):
+						out.append( '{:>{l}}{!r:16}\n'.format( '', el, l=lvl*8 ))
+					else:
+						out.append(f' {el!r}')
+				elif isList(el) or isDict(el):
+					indent = 1 if is_dict else lvl*8+4
+					out.append('{:>{l}}{:16}'.format( '', f'<{type(el).__name__}>', l=indent ))
+					if isList(el) and isinstance(el[0],scalars):
+						out.append('\n')
+					do_list(out,el,lvl=lvl+1,is_dict=isDict(el))
 				else:
-					out.append('{:>{l}}{:<10} {:16} {}\n'.format(
-						'',
-						k,
-						f'<{type(e).__name__}>',
-						repr(e),
-						l=(lvl*8)+4 ))
+					out.append('{:>{l}}{:16} {!r}\n'.format( '', f'<{type(el).__name__}>', el, l=(lvl*8)+8 ))
+				out.append('\n')
 
-			import re
-			return re.sub('\n+','\n',''.join(out))
+			if not e:
+				out.append(f'{e!r}\n')
 
-		# Check that all immutables have been initialized.  Expensive, so do only when testing.
-		def immutable_attr_init_check(self):
-			from .globalvars import g
-			if g.test_suite:
-				from .util import die
-				cls = type(self)
-				for attrname in sorted({a for a in self.valid_attrs if a[0] != '_'}):
-					for o in (cls,cls.__bases__[0]): # assume there's only one base class
-						if attrname in o.__dict__:
-							attr = o.__dict__[attrname]
-							break
-					else:
-						die(4,f'unable to find descriptor {cls.__name__}.{attrname}')
-					if type(attr).__name__ == 'ImmutableAttr':
-						if attrname not in self.__dict__:
-							die(4,
-						f'attribute {attrname!r} of {cls.__name__} has not been initialized in constructor!')
+		def isDict(obj):
+			return isinstance(obj,dict)
+		def isList(obj):
+			return isinstance(obj,list)
+		def isScalar(obj):
+			return isinstance(obj,scalars)
 
-	def print_diff(a,b,from_file='',to_file='',from_json=True):
-		if from_json:
-			a = json.dumps(json.loads(a),indent=4).split('\n') if a else []
-			b = json.dumps(json.loads(b),indent=4).split('\n') if b else []
-		else:
-			a = a.split('\n')
-			b = b.split('\n')
-		sys.stderr.write('  DIFF:\n    {}\n'.format(
-			'\n    '.join(unified_diff(a,b,from_file,to_file)) ))
+		out = [f'<{type(self).__name__}>{" "+repr(self) if isScalar(self) else ""}\n']
 
-	def get_ndiff(a,b):
-		a = a.split('\n')
-		b = b.split('\n')
-		return list(ndiff(a,b))
+		if id(self) in id_list:
+			return out[-1].rstrip() + ' [RECURSION]\n'
+		if isList(self) or isDict(self):
+			do_list(out,self,lvl=lvl,is_dict=isDict(self))
+
+		for k in self.__dict__:
+			e = getattr(self,k)
+			if isList(e) or isDict(e):
+				out.append('{:>{l}}{:<10} {:16}'.format( '', k, f'<{type(e).__name__}>', l=(lvl*8)+4 ))
+				do_list(out,e,lvl=lvl,is_dict=isDict(e))
+			elif hasattr(e,'pfmt') and type(e) != type:
+				out.append('{:>{l}}{:10} {}'.format(
+					'',
+					k,
+					e.pfmt( lvl=lvl+1, id_list=id_list+[id(self)] ),
+					l = (lvl*8)+4 ))
+			else:
+				out.append('{:>{l}}{:<10} {:16} {}\n'.format(
+					'',
+					k,
+					f'<{type(e).__name__}>',
+					repr(e),
+					l=(lvl*8)+4 ))
+
+		import re
+		return re.sub('\n+','\n',''.join(out))

mmgen/globalvars.py

@@ -22,8 +22,6 @@ globalvars.py:  Constants and configuration options for the MMGen suite
 
 import sys,os
 from collections import namedtuple
-from .devtools import *
-
 from .base_obj import Lockable
 
 def die(exit_val,s=''):

mmgen/objmethods.py

@@ -22,9 +22,15 @@ objmethods.py: Mixin classes for MMGen data objects
 
 import unicodedata
 from .globalvars import g
-from .devtools import *
 import mmgen.color as color_mod
 
+if 'MMGenObject' in __builtins__: # added to builtins by devinit.init_dev()
+	MMGenObject = __builtins__['MMGenObject']
+else:
+	class MMGenObject:
+		'placeholder - overridden when testing'
+		def immutable_attr_init_check(self): pass
+
 def truncate_str(s,width): # width = screen width
 	wide_count = 0
 	for i in range(len(s)):

mmgen/proto/btc/regtest.py

@@ -25,6 +25,7 @@ from subprocess import run,PIPE
 from ...common import *
 from ...protocol import init_proto
 from ...rpc import rpc_init,json_encoder
+from ...objmethods import MMGenObject
 
 def create_data_dir(data_dir):
 	try: os.stat(os.path.join(data_dir,'regtest'))

mmgen/protocol.py

@@ -22,8 +22,8 @@ protocol.py: Coin protocol base classes and initializer
 
 from collections import namedtuple
 
-from .devtools import *
 from .globalvars import g
+from .objmethods import MMGenObject
 
 decoded_wif = namedtuple('decoded_wif',['sec','pubkey_type','compressed'])
 decoded_addr = namedtuple('decoded_addr',['bytes','ver_bytes','fmt'])

mmgen/rpc.py

@@ -26,7 +26,7 @@ from collections import namedtuple
 
 from .common import *
 from .base_obj import AsyncInit
-from .objmethods import Hilite,InitErrors
+from .objmethods import Hilite,InitErrors,MMGenObject
 
 auth_data = namedtuple('rpc_auth_data',['user','passwd'])
 

scripts/exec_wrapper.py

@@ -111,6 +111,9 @@ exec_wrapper_init() # sets sys.path[0], runs overlay_setup()
 exec_wrapper_tstart = time.time()
 exec_wrapper_tracemalloc_setup()
 
+from mmgen.devinit import init_dev # import mmgen mods only after overlay setup!
+init_dev()
+
 try:
 	sys.argv.pop(0)
 	exec_wrapper_execed_file = sys.argv[0]

test/objtest.py

@@ -26,6 +26,9 @@ from include.tests_header import repo_root
 from test.overlay import overlay_setup
 sys.path.insert(0,overlay_setup(repo_root))
 
+from mmgen.devinit import init_dev
+init_dev()
+
 os.environ['MMGEN_TEST_SUITE'] = '1'
 
 # Import these _after_ local path's been added to sys.path

test/objtest_py_d/ot_btc_mainnet.py

@@ -7,6 +7,8 @@
 test.objtest_py_d.ot_btc_mainnet: BTC mainnet test vectors for MMGen data objects
 """
 
+from decimal import Decimal
+
 from mmgen.obj import *
 from mmgen.addrlist import AddrIdxList
 from mmgen.seedsplit import *

test/objtest_py_d/ot_eth_mainnet.py

@@ -7,6 +7,8 @@
 test.objtest_py_d.ot_eth_mainnet: ETH mainnet test vectors for MMGen data objects
 """
 
+from decimal import Decimal
+
 from mmgen.obj import *
 from .ot_common import *
 

test/objtest_py_d/ot_ltc_mainnet.py

@@ -7,6 +7,8 @@
 test.objtest_py_d.ot_ltc_mainnet: LTC mainnet test vectors for MMGen data objects
 """
 
+from decimal import Decimal
+
 from mmgen.obj import *
 from .ot_common import *
 

test/test_py_d/ts_ethdev.py

@@ -20,7 +20,7 @@
 ts_ethdev.py: Ethdev tests for the test.py test suite
 """
 
-import sys,os,re,shutil,asyncio
+import sys,os,re,shutil,asyncio,json
 from decimal import Decimal
 from collections import namedtuple
 from subprocess import run,PIPE,DEVNULL
@@ -504,7 +504,6 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			if cp.returncode:
 				die(1,cp.stderr.decode())
 
-		import json
 		d.stop(quiet=True)
 		d.remove_datadir()
 

test/unit_tests.py

@@ -24,6 +24,10 @@ import sys,os,time,importlib,platform
 
 from include.tests_header import repo_root
 from include.common import end_msg
+
+from mmgen.devinit import init_dev
+init_dev()
+
 from mmgen.common import *
 
 opts_data = {

test/unit_tests_d/ut_devtools.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+test.unit_tests_d.ut_devtools: devtools unit tests for the MMGen suite
+"""
+
+import os,json
+from mmgen.devtools import *
+from . import unit_tests_base
+
+textA = """
+def main():
+	a = 1
+	b = 2
+	c = 3
+""".lstrip()
+
+textB = """
+def main():
+	a = 1
+	b = 0
+	c = 3
+""".lstrip()
+
+jsonA = open('test/ref/ethereum/tracking-wallet-v1.json').read()
+dataB = json.loads(jsonA)
+dataB['coin'] = 'ETC'
+jsonB = json.dumps(dataB)
+
+text_data = (
+	(textA, textB, 'a/main.py',   'b/main.py',   False, 'text: one line difference'),
+	('',    textB, 'a/main.py',   'b/main.py',   False, 'text: first file empty'),
+	(textA, textA, 'a/main.py',   'b/main.py',   False, 'text: identical files'),
+	('',    '',    'a/empty.txt', 'b/empty.txt', False, 'text: empty files'),
+)
+
+json_data = (
+	(jsonA, jsonB, 'a/data.json', 'b/data.json', True,  'json: one difference'),
+	('{}',  jsonB, 'a/data.json', 'b/data.json', True,  'json: first file empty'),
+	(jsonA, jsonA, 'a/data.json', 'b/data.json', True,  'json: identical files'),
+	('{}',  '{}',  'a/data.json', 'b/data.json', True,  'json: empty files'),
+)
+
+def print_hdr(hdr):
+	print('{a} {b} {c}'.format(
+		a = '-' * ((78 - len(hdr))//2),
+		b = hdr,
+		c = '-' * ((78 - len(hdr))//2 + (len(hdr) % 2)) ))
+
+# TODO: add data checks
+class unit_tests(unit_tests_base):
+
+	def _pre_subtest(self,name,subname,ut):
+		self._silence()
+
+	def _post_subtest(self,name,subname,ut):
+		print('-' * 80 + '\n')
+		self._end_silence()
+
+	def diff(self,name,ut):
+		for data in text_data + json_data:
+			print_hdr(data[-1])
+			print_diff(*data[:-1])
+		return True
+
+	def ndiff(self,name,ut):
+		for data in text_data:
+			print_hdr(data[-1])
+			print('\n'.join(get_ndiff(*data[:2])))
+		return True
+
+	def stack_trace(self,name,ut):
+		print_hdr('stack trace')
+		print_stack_trace('Test',fh_list=[open(os.devnull,'w')],trim=0)
+		return True
+
+	def obj_pmsg(self,name,ut):
+		from mmgen.protocol import init_proto
+		from mmgen.seed import Seed
+		from mmgen.addrlist import AddrList
+		print_hdr('MMGenObject.pmsg()')
+		AddrList(
+			proto       = init_proto('btc'),
+			seed        = Seed(seed_bin=bytes.fromhex('bead'*16)),
+			addr_idxs   = '1',
+			mmtype      = 'B',
+			skip_chksum = True ).pmsg()
+		return True

test/unit_tests_d/ut_obj.py

@@ -3,6 +3,8 @@
 test.unit_tests_d.ut_obj: data object unit tests for the MMGen suite
 """
 
+from decimal import Decimal
+
 from mmgen.common import *
 
 class unit_tests: