gentest.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566
  1. #!/usr/bin/env python3
  2. #
  3. # mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution
  4. # Copyright (C)2013-2023 The MMGen Project <mmgen@tuta.io>
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. """
  19. test/gentest.py: Cryptocoin key/address generation tests for the MMGen suite
  20. """
  21. import sys,os,time
  22. import include.test_init
  23. # Import these _after_ local path's been added to sys.path
  24. import mmgen.opts as opts
  25. from mmgen.cfg import gc,Config
  26. from mmgen.color import green,red,purple
  27. from mmgen.util import msg,ymsg,capfirst,is_int,die
  28. results_file = 'gentest.out.json'
  29. rounds = 100
  30. opts_data = {
  31. 'text': {
  32. 'desc': 'Test key/address generation of the MMGen suite in various ways',
  33. 'usage':'[options] <spec> <rounds | dump file>',
  34. 'options': """
  35. -h, --help Print this help message
  36. --, --longhelp Print help message for long options (common options)
  37. -a, --all-coins Test all coins supported by specified external tool
  38. -k, --use-internal-keccak-module Force use of the internal keccak module
  39. -q, --quiet Produce quieter output
  40. -s, --save-results Save output of external tool in Compare test to
  41. {rf!r}
  42. -t, --type=t Specify address type (e.g. 'compressed','segwit',
  43. 'zcash_z','bech32')
  44. -v, --verbose Produce more verbose output
  45. """,
  46. 'notes': """
  47. TEST TYPES:
  48. Compare: {prog} A:B <rounds> (compare address generators A and B)
  49. Speed: {prog} A <rounds> (test speed of generator A)
  50. Dump: {prog} A <dump file> (compare generator A to wallet dump)
  51. where:
  52. A and B are keygen backend numbers ('1' being the default); or
  53. B is the name of an external tool (see below) or 'ext'.
  54. If B is 'ext', the external tool will be chosen automatically.
  55. For the Compare test, A may be 'all' to test all backends for the current
  56. coin/address type combination.
  57. EXAMPLES:
  58. Compare addresses generated by 'libsecp256k1' and 'python-ecdsa' backends,
  59. with 100 random rounds plus private-key edge cases:
  60. $ {prog} 1:2 100
  61. Compare Segwit addresses from default 'libsecp256k1' backend to 'pycoin'
  62. library for all supported coins, 100 rounds + edge cases:
  63. $ {prog} --all-coins --type=segwit 1:pycoin 100
  64. Compare addresses from 'python-ecdsa' backend to output of 'keyconv' tool
  65. for all supported coins, 100 rounds + edge cases:
  66. $ {prog} --all-coins --type=compressed 2:keyconv 100
  67. Compare bech32 addrs from 'libsecp256k1' backend to Bitcoin Core wallet
  68. dump:
  69. $ {prog} --type=bech32 1 bech32wallet.dump
  70. Compare addresses from Monero 'ed25519ll' backend to output of default
  71. external tool, 10 rounds + edge cases:
  72. $ {prog} --coin=xmr 3:ext 10
  73. Test the speed of default Monero 'nacl' backend, 10,000 rounds:
  74. $ test/gentest.py --coin=xmr 1 10000
  75. Same for Zcash:
  76. $ test/gentest.py --coin=zec --type=zcash_z 1 10000
  77. Test all configured Monero backends against the 'monero-python' library, 3 rounds
  78. + edge cases:
  79. $ test/gentest.py --coin=xmr all:monero-python 3
  80. Test 'nacl' and 'ed25519ll_djbec' backends against each other, 10,000 rounds
  81. + edge cases:
  82. $ test/gentest.py --coin=xmr 1:2 10000
  83. SUPPORTED EXTERNAL TOOLS:
  84. + ethkey (for ETH,ETC)
  85. https://github.com/openethereum/openethereum
  86. (build with 'cargo build -p ethkey-cli --release')
  87. + zcash-mini (for Zcash-Z addresses and view keys)
  88. https://github.com/FiloSottile/zcash-mini
  89. + monero-python (for Monero addresses and view keys)
  90. https://github.com/monero-ecosystem/monero-python
  91. + pycoin (for supported coins)
  92. https://github.com/richardkiss/pycoin
  93. + keyconv (for supported coins)
  94. https://github.com/exploitagency/vanitygen-plus
  95. ('keyconv' does not generate Segwit addresses)
  96. """
  97. },
  98. 'code': {
  99. 'options': lambda s: s.format(
  100. rf=results_file,
  101. ),
  102. 'notes': lambda s: s.format(
  103. prog='test/gentest.py',
  104. pnm=gc.proj_name,
  105. snum=rounds )
  106. }
  107. }
  108. def get_cmd_output(cmd,input=None):
  109. return run(cmd,input=input,stdout=PIPE,stderr=DEVNULL).stdout.decode().splitlines()
  110. saved_results = {}
  111. class GenTool(object):
  112. def __init__(self,proto,addr_type):
  113. self.proto = proto
  114. self.addr_type = addr_type
  115. self.data = {}
  116. def __del__(self):
  117. if cfg.save_results:
  118. key = f'{self.proto.coin}-{self.proto.network}-{self.addr_type.name}-{self.desc}'.lower()
  119. saved_results[key] = {k.hex():v._asdict() for k,v in self.data.items()}
  120. def run_tool(self,sec,cache_data):
  121. vcoin = 'BTC' if self.proto.coin == 'BCH' else self.proto.coin
  122. key = sec.orig_bytes
  123. if key in self.data:
  124. return self.data[key]
  125. else:
  126. ret = self.run(sec,vcoin)
  127. if cache_data:
  128. self.data[key] = sd( **{'reduced':sec.hex()}, **ret._asdict() )
  129. return ret
  130. class GenToolEthkey(GenTool):
  131. desc = 'ethkey'
  132. def __init__(self,*args,**kwargs):
  133. self.cmdname = get_ethkey()
  134. return super().__init__(*args,**kwargs)
  135. def run(self,sec,vcoin):
  136. o = get_cmd_output([self.cmdname,'info',sec.hex()])
  137. return gtr(
  138. o[0].split()[1],
  139. o[-1].split()[1],
  140. None )
  141. class GenToolKeyconv(GenTool):
  142. desc = 'keyconv'
  143. def run(self,sec,vcoin):
  144. o = get_cmd_output(['keyconv','-C',vcoin,sec.wif])
  145. return gtr(
  146. o[1].split()[1],
  147. o[0].split()[1],
  148. None )
  149. class GenToolZcash_mini(GenTool):
  150. desc = 'zcash-mini'
  151. def run(self,sec,vcoin):
  152. o = get_cmd_output(['zcash-mini','-key','-simple'],input=(sec.wif+'\n').encode())
  153. return gtr( o[1], o[0], o[-1] )
  154. class GenToolPycoin(GenTool):
  155. """
  156. pycoin/networks/all.py pycoin/networks/legacy_networks.py
  157. """
  158. desc = 'pycoin'
  159. def __init__(self,*args,**kwargs):
  160. super().__init__(*args,**kwargs)
  161. try:
  162. from pycoin.networks.registry import network_for_netcode
  163. except:
  164. raise ImportError('Unable to import pycoin.networks.registry. Is pycoin installed on your system?')
  165. self.nfnc = network_for_netcode
  166. def run(self,sec,vcoin):
  167. if self.proto.testnet:
  168. vcoin = cinfo.external_tests['testnet']['pycoin'][vcoin]
  169. network = self.nfnc(vcoin)
  170. key = network.keys.private(
  171. secret_exponent = int(sec.hex(),16),
  172. is_compressed = self.addr_type.name != 'legacy' )
  173. if key is None:
  174. die(1,f'can’t parse {sec.hex()}')
  175. if self.addr_type.name in ('segwit','bech32'):
  176. hash160_c = key.hash160(is_compressed=True)
  177. if self.addr_type.name == 'segwit':
  178. p2sh_script = network.contract.for_p2pkh_wit(hash160_c)
  179. addr = network.address.for_p2s(p2sh_script)
  180. else:
  181. addr = network.address.for_p2pkh_wit(hash160_c)
  182. else:
  183. addr = key.address()
  184. return gtr( key.wif(), addr, None )
  185. class GenToolMonero_python(GenTool):
  186. desc = 'monero-python'
  187. def __init__(self,*args,**kwargs):
  188. super().__init__(*args,**kwargs)
  189. from mmgen.util import load_cryptodomex
  190. load_cryptodomex()
  191. try:
  192. from monero.seed import Seed
  193. except:
  194. raise ImportError('Unable to import monero-python. Is monero-python installed on your system?')
  195. self.Seed = Seed
  196. def run(self,sec,vcoin):
  197. seed = self.Seed( sec.orig_bytes.hex() )
  198. sk = seed.secret_spend_key()
  199. vk = seed.secret_view_key()
  200. addr = seed.public_address()
  201. return gtr( sk, addr, vk )
  202. def find_or_check_tool(proto,addr_type,toolname):
  203. ext_progs = list(cinfo.external_tests[proto.network])
  204. if toolname not in ext_progs + ['ext']:
  205. die(1,f'{toolname!r}: unsupported tool for network {proto.network}')
  206. if cfg.all_coins and toolname == 'ext':
  207. die(1,"'--all-coins' must be combined with a specific external testing tool")
  208. else:
  209. tool = cinfo.get_test_support(
  210. proto.coin,
  211. addr_type.name,
  212. proto.network,
  213. verbose = not cfg.quiet,
  214. toolname = toolname if toolname != 'ext' else None )
  215. if tool and toolname in ext_progs and toolname != tool:
  216. sys.exit(3)
  217. if tool == None:
  218. return None
  219. return tool
  220. def test_equal(desc,a_val,b_val,in_bytes,sec,wif,a_desc,b_desc):
  221. if a_val != b_val:
  222. fs = """
  223. {i:{w}}: {}
  224. {s:{w}}: {}
  225. {W:{w}}: {}
  226. {a:{w}}: {}
  227. {b:{w}}: {}
  228. """
  229. die(3,
  230. red('\nERROR: {} do not match!').format(desc)
  231. + fs.format(
  232. in_bytes.hex(), sec, wif, a_val, b_val,
  233. i='input', s='sec key', W='WIF key', a=a_desc, b=b_desc,
  234. w=max(len(e) for e in (a_desc,b_desc)) + 1
  235. ).rstrip())
  236. def do_ab_test(proto,scfg,addr_type,gen1,kg2,ag,tool,cache_data):
  237. def do_ab_inner(n,trounds,in_bytes):
  238. global last_t
  239. if cfg.verbose or time.time() - last_t >= 0.1:
  240. qmsg_r(f'\rRound {i+1}/{trounds} ')
  241. last_t = time.time()
  242. sec = PrivKey(proto,in_bytes,compressed=addr_type.compressed,pubkey_type=addr_type.pubkey_type)
  243. data = kg1.gen_data(sec)
  244. addr1 = ag.to_addr(data)
  245. tinfo = ( in_bytes, sec, sec.wif, type(kg1).__name__, type(kg2).__name__ if kg2 else tool.desc )
  246. def do_msg():
  247. if cfg.verbose:
  248. msg( fs.format( b=in_bytes.hex(), r=sec.hex(), k=sec.wif, v=vk2, a=addr1 ))
  249. if tool:
  250. def run_tool():
  251. o = tool.run_tool(sec,cache_data)
  252. test_equal( 'WIF keys', sec.wif, o.wif, *tinfo )
  253. test_equal( 'addresses', addr1, o.addr, *tinfo )
  254. if o.viewkey:
  255. test_equal( 'view keys', ag.to_viewkey(data), o.viewkey, *tinfo )
  256. return o.viewkey
  257. vk2 = run_tool()
  258. do_msg()
  259. else:
  260. test_equal( 'addresses', addr1, ag.to_addr(kg2.gen_data(sec)), *tinfo )
  261. vk2 = None
  262. do_msg()
  263. qmsg_r(f'\rRound {n+1}/{trounds} ')
  264. def get_randbytes():
  265. if tool and len(tool.data) > len(edgecase_sks):
  266. for privbytes in tuple(tool.data)[len(edgecase_sks):]:
  267. yield privbytes
  268. else:
  269. for i in range(scfg.rounds):
  270. yield getrand(32)
  271. kg1 = KeyGenerator( cfg, proto, addr_type.pubkey_type, gen1 )
  272. if type(kg1) == type(kg2):
  273. die(4,'Key generators are the same!')
  274. e = cinfo.get_entry(proto.coin,proto.network)
  275. qmsg(green("Comparing address generators '{A}' and '{B}' for {N} {c} ({n}), addrtype {a!r}".format(
  276. A = type(kg1).__name__.replace('_','-'),
  277. B = type(kg2).__name__.replace('_','-') if kg2 else tool.desc,
  278. N = proto.network,
  279. c = proto.coin,
  280. n = e.name if e else '---',
  281. a = addr_type.name )))
  282. global last_t
  283. last_t = time.time()
  284. fs = (
  285. '\ninput: {b}' +
  286. '\nreduced: {r}' +
  287. '\n{:9} {{k}}'.format(addr_type.wif_label+':') +
  288. ('\nviewkey: {v}' if 'viewkey' in addr_type.extra_attrs else '') +
  289. '\naddr: {a}\n' )
  290. ge = CoinProtocol.Secp256k1.secp256k1_ge
  291. # test some important private key edge cases:
  292. edgecase_sks = (
  293. bytes([0x00]*31 + [0x01]), # min
  294. bytes([0xff]*32), # max
  295. bytes([0x0f] + [0xff]*31), # produces same key as above for zcash-z
  296. int.to_bytes(ge + 1, 32, 'big'), # bitcoin will reduce
  297. int.to_bytes(ge - 1, 32, 'big'), # bitcoin will not reduce
  298. bytes([0x00]*31 + [0xff]), # monero will reduce
  299. bytes([0xff]*31 + [0x0f]), # monero will not reduce
  300. bytes.fromhex('deadbeef'*8),
  301. )
  302. qmsg(purple('edge cases:'))
  303. for i,privbytes in enumerate(edgecase_sks):
  304. do_ab_inner(i,len(edgecase_sks),privbytes)
  305. qmsg(green('\rOK ' if cfg.verbose else 'OK'))
  306. qmsg(purple('random input:'))
  307. for i,privbytes in enumerate(get_randbytes()):
  308. do_ab_inner(i,scfg.rounds,privbytes)
  309. qmsg(green('\rOK ' if cfg.verbose else 'OK'))
  310. def init_tool(proto,addr_type,toolname):
  311. return globals()['GenTool'+capfirst(toolname.replace('-','_'))](proto,addr_type)
  312. def ab_test(proto,scfg):
  313. addr_type = MMGenAddrType( proto=proto, id_str=cfg.type or proto.dfl_mmtype )
  314. if scfg.gen2:
  315. assert scfg.gen1 != 'all', "'all' must be used only with external tool"
  316. kg2 = KeyGenerator( cfg, proto, addr_type.pubkey_type, scfg.gen2 )
  317. tool = None
  318. else:
  319. toolname = find_or_check_tool( proto, addr_type, scfg.tool )
  320. if toolname == None:
  321. ymsg(f'Warning: skipping tool {scfg.tool!r} for {proto.coin} {addr_type.name}')
  322. return
  323. tool = init_tool( proto, addr_type, toolname )
  324. kg2 = None
  325. ag = AddrGenerator( cfg, proto, addr_type )
  326. if scfg.all_backends: # check all backends against external tool
  327. for n in range(len(get_backends(addr_type.pubkey_type))):
  328. do_ab_test( proto, scfg, addr_type, gen1=n+1, kg2=kg2, ag=ag, tool=tool, cache_data=scfg.rounds < 1000 and not n )
  329. else: # check specific backend against external tool or another backend
  330. do_ab_test( proto, scfg, addr_type, gen1=scfg.gen1, kg2=kg2, ag=ag, tool=tool, cache_data=False )
  331. def speed_test(proto,kg,ag,rounds):
  332. qmsg(green('Testing speed of address generator {!r} for coin {}'.format(
  333. type(kg).__name__,
  334. proto.coin )))
  335. from struct import pack,unpack
  336. seed = getrand(28)
  337. qmsg('Incrementing key with each round')
  338. qmsg('Starting key: {}'.format( (seed + pack('I',0)).hex() ))
  339. import time
  340. start = last_t = time.time()
  341. for i in range(rounds):
  342. if time.time() - last_t >= 0.1:
  343. qmsg_r(f'\rRound {i+1}/{rounds} ')
  344. last_t = time.time()
  345. sec = PrivKey( proto, seed+pack('I', i), compressed=ag.compressed, pubkey_type=ag.pubkey_type )
  346. addr = ag.to_addr(kg.gen_data(sec))
  347. vmsg(f'\nkey: {sec.wif}\naddr: {addr}\n')
  348. qmsg(
  349. f'\rRound {i+1}/{rounds} ' +
  350. f'\n{rounds} addresses generated' +
  351. ('' if cfg.test_suite_deterministic else f' in {time.time()-start:.2f} seconds')
  352. )
  353. def dump_test(proto,kg,ag,filename):
  354. with open(filename) as fp:
  355. dump = [[*(e.split()[0] for e in line.split('addr='))] for line in fp.readlines() if 'addr=' in line]
  356. if not dump:
  357. die(1,f'File {filename!r} appears not to be a wallet dump')
  358. qmsg(green(
  359. "A: generator pair '{}:{}'\nB: wallet dump {!r}".format(
  360. type(kg).__name__,
  361. type(ag).__name__,
  362. filename)))
  363. for count,(b_wif,b_addr) in enumerate(dump,1):
  364. qmsg_r(f'\rKey {count}/{len(dump)} ')
  365. try:
  366. b_sec = PrivKey(proto,wif=b_wif)
  367. except:
  368. die(2,f'\nInvalid {proto.network} WIF address in dump file: {b_wif}')
  369. a_addr = ag.to_addr(kg.gen_data(b_sec))
  370. vmsg(f'\nwif: {b_wif}\naddr: {b_addr}\n')
  371. tinfo = (b_sec,b_sec.hex(),b_wif,type(kg).__name__,filename)
  372. test_equal('addresses',a_addr,b_addr,*tinfo)
  373. qmsg(green(('\n','')[bool(cfg.verbose)] + 'OK'))
  374. def get_protos(proto,addr_type,toolname):
  375. init_genonly_altcoins(testnet=proto.testnet)
  376. for coin in cinfo.external_tests[proto.network][toolname]:
  377. if coin.lower() not in CoinProtocol.coins:
  378. continue
  379. ret = init_proto( cfg, coin, testnet=proto.testnet )
  380. if addr_type not in ret.mmtypes:
  381. continue
  382. yield ret
  383. def parse_args():
  384. if len(cfg._args) != 2:
  385. cfg._opts.usage()
  386. arg1,arg2 = cfg._args
  387. gen1,gen2,rounds = (0,0,0)
  388. tool,all_backends,dumpfile = (None,None,None)
  389. if is_int(arg1) and is_int(arg2):
  390. test = 'speed'
  391. gen1 = arg1
  392. rounds = arg2
  393. elif is_int(arg1) and os.access(arg2,os.R_OK):
  394. test = 'dump'
  395. gen1 = arg1
  396. dumpfile = arg2
  397. else:
  398. test = 'ab'
  399. rounds = arg2
  400. if not is_int(arg2):
  401. die(1,'Second argument must be dump filename or integer rounds specification')
  402. try:
  403. a,b = arg1.split(':')
  404. except:
  405. die(1,'First argument must be a generator backend number or two colon-separated arguments')
  406. if is_int(a):
  407. gen1 = a
  408. else:
  409. if a == 'all':
  410. all_backends = True
  411. else:
  412. die(1,"First part of first argument must be a generator backend number or 'all'")
  413. if is_int(b):
  414. if cfg.all_coins:
  415. die(1,'--all-coins must be used with external tool only')
  416. gen2 = b
  417. else:
  418. tool = b
  419. ext_progs = list(cinfo.external_tests[cfg._proto.network]) + ['ext']
  420. if b not in ext_progs:
  421. die(1,f'Second part of first argument must be a generator backend number or one of {ext_progs}')
  422. return namedtuple('parsed_args',['test','gen1','gen2','rounds','tool','all_backends','dumpfile'])(
  423. test,
  424. int(gen1) or None,
  425. int(gen2) or None,
  426. int(rounds) or None,
  427. tool,
  428. all_backends,
  429. dumpfile )
  430. def main():
  431. scfg = parse_args()
  432. addr_type = MMGenAddrType( proto=proto, id_str=cfg.type or proto.dfl_mmtype )
  433. if scfg.test == 'ab':
  434. protos = get_protos(proto,addr_type,scfg.tool) if cfg.all_coins else [proto]
  435. for p in protos:
  436. ab_test( p, scfg )
  437. else:
  438. kg = KeyGenerator( cfg, proto, addr_type.pubkey_type, scfg.gen1 )
  439. ag = AddrGenerator( cfg, proto, addr_type )
  440. if scfg.test == 'speed':
  441. speed_test( proto, kg, ag, scfg.rounds )
  442. elif scfg.test == 'dump':
  443. dump_test( proto, kg, ag, scfg.dumpfile )
  444. if saved_results:
  445. import json
  446. with open(results_file,'w') as fp:
  447. fp.write(json.dumps( saved_results, indent=4 ))
  448. from subprocess import run,PIPE,DEVNULL
  449. from collections import namedtuple
  450. from mmgen.protocol import init_proto,CoinProtocol
  451. from mmgen.altcoin import init_genonly_altcoins
  452. from mmgen.altcoin import CoinInfo as cinfo
  453. from mmgen.key import PrivKey
  454. from mmgen.addr import MMGenAddrType
  455. from mmgen.addrgen import KeyGenerator,AddrGenerator
  456. from mmgen.keygen import get_backends
  457. from test.include.common import getrand,get_ethkey,set_globals
  458. gtr = namedtuple('gen_tool_result',['wif','addr','viewkey'])
  459. sd = namedtuple('saved_data_item',['reduced','wif','addr','viewkey'])
  460. sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
  461. cfg = Config(opts_data=opts_data)
  462. set_globals(cfg)
  463. qmsg = cfg._util.qmsg
  464. qmsg_r = cfg._util.qmsg_r
  465. vmsg = cfg._util.vmsg
  466. proto = cfg._proto
  467. main()