gentest.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547
  1. #!/usr/bin/env python3
  2. #
  3. # mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution
  4. # Copyright (C)2013-2022 The MMGen Project <mmgen@tuta.io>
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. """
  19. test/gentest.py: Cryptocoin key/address generation tests for the MMGen suite
  20. """
  21. import sys,os
  22. from include.tests_header import repo_root
  23. from test.overlay import overlay_setup
  24. sys.path.insert(0,overlay_setup(repo_root))
  25. # Import these _after_ local path's been added to sys.path
  26. from mmgen.common import *
  27. from test.include.common import getrand
  28. results_file = 'gentest.out.json'
  29. rounds = 100
  30. opts_data = {
  31. 'text': {
  32. 'desc': 'Test key/address generation of the MMGen suite in various ways',
  33. 'usage':'[options] <spec> <rounds | dump file>',
  34. 'options': """
  35. -h, --help Print this help message
  36. --, --longhelp Print help message for long options (common options)
  37. -a, --all-coins Test all coins supported by specified external tool
  38. -k, --use-internal-keccak-module Force use of the internal keccak module
  39. -q, --quiet Produce quieter output
  40. -s, --save-results Save output of external tool in Compare test to
  41. {rf!r}
  42. -t, --type=t Specify address type (e.g. 'compressed','segwit',
  43. 'zcash_z','bech32')
  44. -v, --verbose Produce more verbose output
  45. """,
  46. 'notes': """
  47. TEST TYPES:
  48. Compare: {prog} A:B <rounds> (compare address generators A and B)
  49. Speed: {prog} A <rounds> (test speed of generator A)
  50. Dump: {prog} A <dump file> (compare generator A to wallet dump)
  51. where:
  52. A and B are keygen backend numbers ('1' being the default); or
  53. B is the name of an external tool (see below) or 'ext'.
  54. If B is 'ext', the external tool will be chosen automatically.
  55. For the Compare test, A may be 'all' to test all backends for the current
  56. coin/address type combination.
  57. EXAMPLES:
  58. Compare addresses generated by 'libsecp256k1' and 'python-ecdsa' backends,
  59. with 100 random rounds plus private-key edge cases:
  60. $ {prog} 1:2 100
  61. Compare Segwit addresses from default 'libsecp256k1' backend to 'pycoin'
  62. library for all supported coins, 100 rounds + edge cases:
  63. $ {prog} --all-coins --type=segwit 1:pycoin 100
  64. Compare addresses from 'python-ecdsa' backend to output of 'keyconv' tool
  65. for all supported coins, 100 rounds + edge cases:
  66. $ {prog} --all-coins --type=compressed 2:keyconv 100
  67. Compare bech32 addrs from 'libsecp256k1' backend to Bitcoin Core wallet
  68. dump:
  69. $ {prog} --type=bech32 1 bech32wallet.dump
  70. Compare addresses from Monero 'ed25519ll' backend to output of default
  71. external tool, 10 rounds + edge cases:
  72. $ {prog} --coin=xmr 3:ext 10
  73. Test the speed of default Monero 'nacl' backend, 10,000 rounds:
  74. $ test/gentest.py --coin=xmr 1 10000
  75. Same for Zcash:
  76. $ test/gentest.py --coin=zec --type=zcash_z 1 10000
  77. Test all configured Monero backends against 'moneropy' library, 3 rounds
  78. + edge cases:
  79. $ test/gentest.py --coin=xmr all:moneropy 3
  80. Test 'nacl' and 'ed25519ll_djbec' backends against each other, 10,000 rounds
  81. + edge cases:
  82. $ test/gentest.py --coin=xmr 1:2 10000
  83. SUPPORTED EXTERNAL TOOLS:
  84. + ethkey (for ETH,ETC)
  85. https://github.com/openethereum/openethereum
  86. (build with 'cargo build -p ethkey-cli --release')
  87. + zcash-mini (for Zcash-Z addresses and view keys)
  88. https://github.com/FiloSottile/zcash-mini
  89. + moneropy (for Monero addresses and view keys)
  90. https://github.com/bigreddmachine/MoneroPy
  91. + pycoin (for supported coins)
  92. https://github.com/richardkiss/pycoin
  93. + keyconv (for supported coins)
  94. https://github.com/exploitagency/vanitygen-plus
  95. ('keyconv' does not generate Segwit addresses)
  96. """
  97. },
  98. 'code': {
  99. 'options': lambda s: s.format(
  100. rf=results_file,
  101. ),
  102. 'notes': lambda s: s.format(
  103. prog='test/gentest.py',
  104. pnm=g.proj_name,
  105. snum=rounds )
  106. }
  107. }
  108. gtr = namedtuple('gen_tool_result',['wif','addr','viewkey'])
  109. sd = namedtuple('saved_data_item',['reduced','wif','addr','viewkey'])
  110. def get_cmd_output(cmd,input=None):
  111. return run(cmd,input=input,stdout=PIPE,stderr=DEVNULL).stdout.decode().splitlines()
  112. saved_results = {}
  113. class GenTool(object):
  114. def __init__(self,proto,addr_type):
  115. self.proto = proto
  116. self.addr_type = addr_type
  117. self.data = {}
  118. def __del__(self):
  119. if opt.save_results:
  120. key = f'{self.proto.coin}-{self.proto.network}-{self.addr_type.name}-{self.desc}'.lower()
  121. saved_results[key] = {k.hex():v._asdict() for k,v in self.data.items()}
  122. def run_tool(self,sec,cache_data):
  123. vcoin = 'BTC' if self.proto.coin == 'BCH' else self.proto.coin
  124. key = sec.orig_bytes
  125. if key in self.data:
  126. return self.data[key]
  127. else:
  128. ret = self.run(sec,vcoin)
  129. if cache_data:
  130. self.data[key] = sd( **{'reduced':sec.hex()}, **ret._asdict() )
  131. return ret
  132. class GenToolEthkey(GenTool):
  133. desc = 'ethkey'
  134. def run(self,sec,vcoin):
  135. o = get_cmd_output(['ethkey','info',sec.hex()])
  136. return gtr(
  137. o[0].split()[1],
  138. o[-1].split()[1],
  139. None )
  140. class GenToolKeyconv(GenTool):
  141. desc = 'keyconv'
  142. def run(self,sec,vcoin):
  143. o = get_cmd_output(['keyconv','-C',vcoin,sec.wif])
  144. return gtr(
  145. o[1].split()[1],
  146. o[0].split()[1],
  147. None )
  148. class GenToolZcash_mini(GenTool):
  149. desc = 'zcash-mini'
  150. def run(self,sec,vcoin):
  151. o = get_cmd_output(['zcash-mini','-key','-simple'],input=(sec.wif+'\n').encode())
  152. return gtr( o[1], o[0], o[-1] )
  153. class GenToolPycoin(GenTool):
  154. """
  155. pycoin/networks/all.py pycoin/networks/legacy_networks.py
  156. """
  157. desc = 'pycoin'
  158. def __init__(self,*args,**kwargs):
  159. super().__init__(*args,**kwargs)
  160. try:
  161. from pycoin.networks.registry import network_for_netcode
  162. except:
  163. raise ImportError('Unable to import pycoin.networks.registry. Is pycoin installed on your system?')
  164. self.nfnc = network_for_netcode
  165. def run(self,sec,vcoin):
  166. if self.proto.testnet:
  167. vcoin = cinfo.external_tests['testnet']['pycoin'][vcoin]
  168. network = self.nfnc(vcoin)
  169. key = network.keys.private(
  170. secret_exponent = int(sec.hex(),16),
  171. is_compressed = self.addr_type.name != 'legacy' )
  172. if key is None:
  173. die(1,f'can’t parse {sec.hex()}')
  174. if self.addr_type.name in ('segwit','bech32'):
  175. hash160_c = key.hash160(is_compressed=True)
  176. if self.addr_type.name == 'segwit':
  177. p2sh_script = network.contract.for_p2pkh_wit(hash160_c)
  178. addr = network.address.for_p2s(p2sh_script)
  179. else:
  180. addr = network.address.for_p2pkh_wit(hash160_c)
  181. else:
  182. addr = key.address()
  183. return gtr( key.wif(), addr, None )
  184. class GenToolMoneropy(GenTool):
  185. desc = 'moneropy'
  186. def __init__(self,*args,**kwargs):
  187. super().__init__(*args,**kwargs)
  188. try:
  189. import moneropy.account
  190. except:
  191. raise ImportError('Unable to import moneropy. Is moneropy installed on your system?')
  192. self.mpa = moneropy.account
  193. def run(self,sec,vcoin):
  194. sk,vk,addr = self.mpa.account_from_spend_key(sec.hex()) # VERY slow!
  195. return gtr( sk, addr, vk )
  196. def find_or_check_tool(proto,addr_type,toolname):
  197. ext_progs = list(cinfo.external_tests[proto.network])
  198. if toolname not in ext_progs + ['ext']:
  199. die(1,f'{toolname!r}: unsupported tool for network {proto.network}')
  200. if opt.all_coins and toolname == 'ext':
  201. die(1,"'--all-coins' must be combined with a specific external testing tool")
  202. else:
  203. tool = cinfo.get_test_support(
  204. proto.coin,
  205. addr_type.name,
  206. proto.network,
  207. verbose = not opt.quiet,
  208. toolname = toolname if toolname != 'ext' else None )
  209. if tool and toolname in ext_progs and toolname != tool:
  210. sys.exit(3)
  211. if tool == None:
  212. return None
  213. return tool
  214. def test_equal(desc,a_val,b_val,in_bytes,sec,wif,a_desc,b_desc):
  215. if a_val != b_val:
  216. fs = """
  217. {i:{w}}: {}
  218. {s:{w}}: {}
  219. {W:{w}}: {}
  220. {a:{w}}: {}
  221. {b:{w}}: {}
  222. """
  223. die(3,
  224. red('\nERROR: {} do not match!').format(desc)
  225. + fs.format(
  226. in_bytes.hex(), sec, wif, a_val, b_val,
  227. i='input', s='sec key', W='WIF key', a=a_desc, b=b_desc,
  228. w=max(len(e) for e in (a_desc,b_desc)) + 1
  229. ).rstrip())
  230. def do_ab_test(proto,cfg,addr_type,gen1,kg2,ag,tool,cache_data):
  231. def do_ab_inner(n,trounds,in_bytes):
  232. global last_t
  233. if opt.verbose or time.time() - last_t >= 0.1:
  234. qmsg_r(f'\rRound {i+1}/{trounds} ')
  235. last_t = time.time()
  236. sec = PrivKey(proto,in_bytes,compressed=addr_type.compressed,pubkey_type=addr_type.pubkey_type)
  237. data = kg1.gen_data(sec)
  238. addr1 = ag.to_addr(data)
  239. tinfo = ( in_bytes, sec, sec.wif, type(kg1).__name__, type(kg2).__name__ if kg2 else tool.desc )
  240. def do_msg():
  241. if opt.verbose:
  242. msg( fs.format( b=in_bytes.hex(), r=sec.hex(), k=sec.wif, v=vk2, a=addr1 ))
  243. if tool:
  244. def run_tool():
  245. o = tool.run_tool(sec,cache_data)
  246. test_equal( 'WIF keys', sec.wif, o.wif, *tinfo )
  247. test_equal( 'addresses', addr1, o.addr, *tinfo )
  248. if o.viewkey:
  249. test_equal( 'view keys', ag.to_viewkey(data), o.viewkey, *tinfo )
  250. return o.viewkey
  251. vk2 = run_tool()
  252. do_msg()
  253. else:
  254. test_equal( 'addresses', addr1, ag.to_addr(kg2.gen_data(sec)), *tinfo )
  255. vk2 = None
  256. do_msg()
  257. qmsg_r(f'\rRound {n+1}/{trounds} ')
  258. def get_randbytes():
  259. if tool and len(tool.data) > len(edgecase_sks):
  260. for privbytes in tuple(tool.data)[len(edgecase_sks):]:
  261. yield privbytes
  262. else:
  263. for i in range(cfg.rounds):
  264. yield getrand(32)
  265. kg1 = KeyGenerator( proto, addr_type.pubkey_type, gen1 )
  266. if type(kg1) == type(kg2):
  267. die(4,'Key generators are the same!')
  268. e = cinfo.get_entry(proto.coin,proto.network)
  269. qmsg(green("Comparing address generators '{A}' and '{B}' for {N} {c} ({n}), addrtype {a!r}".format(
  270. A = type(kg1).__name__.replace('_','-'),
  271. B = type(kg2).__name__.replace('_','-') if kg2 else tool.desc,
  272. N = proto.network,
  273. c = proto.coin,
  274. n = e.name if e else '---',
  275. a = addr_type.name )))
  276. global last_t
  277. last_t = time.time()
  278. fs = (
  279. '\ninput: {b}' +
  280. '\nreduced: {r}' +
  281. '\n{:9} {{k}}'.format(addr_type.wif_label+':') +
  282. ('\nviewkey: {v}' if 'viewkey' in addr_type.extra_attrs else '') +
  283. '\naddr: {a}\n' )
  284. ge = CoinProtocol.Secp256k1.secp256k1_ge
  285. # test some important private key edge cases:
  286. edgecase_sks = (
  287. bytes([0x00]*31 + [0x01]), # min
  288. bytes([0xff]*32), # max
  289. bytes([0x0f] + [0xff]*31), # produces same key as above for zcash-z
  290. int.to_bytes(ge + 1, 32, 'big'), # bitcoin will reduce
  291. int.to_bytes(ge - 1, 32, 'big'), # bitcoin will not reduce
  292. bytes([0x00]*31 + [0xff]), # monero will reduce
  293. bytes([0xff]*31 + [0x0f]), # monero will not reduce
  294. bytes.fromhex('deadbeef'*8),
  295. )
  296. qmsg(purple('edge cases:'))
  297. for i,privbytes in enumerate(edgecase_sks):
  298. do_ab_inner(i,len(edgecase_sks),privbytes)
  299. qmsg(green('\rOK ' if opt.verbose else 'OK'))
  300. qmsg(purple('random input:'))
  301. for i,privbytes in enumerate(get_randbytes()):
  302. do_ab_inner(i,cfg.rounds,privbytes)
  303. qmsg(green('\rOK ' if opt.verbose else 'OK'))
  304. def init_tool(proto,addr_type,toolname):
  305. return globals()['GenTool'+capfirst(toolname.replace('-','_'))](proto,addr_type)
  306. def ab_test(proto,cfg):
  307. addr_type = MMGenAddrType( proto=proto, id_str=opt.type or proto.dfl_mmtype )
  308. if cfg.gen2:
  309. assert cfg.gen1 != 'all', "'all' must be used only with external tool"
  310. kg2 = KeyGenerator( proto, addr_type.pubkey_type, cfg.gen2 )
  311. tool = None
  312. else:
  313. toolname = find_or_check_tool( proto, addr_type, cfg.tool )
  314. if toolname == None:
  315. ymsg(f'Warning: skipping tool {cfg.tool!r} for {proto.coin} {addr_type.name}')
  316. return
  317. tool = init_tool( proto, addr_type, toolname )
  318. kg2 = None
  319. ag = AddrGenerator( proto, addr_type )
  320. if cfg.all_backends: # check all backends against external tool
  321. for n in range(len(get_backends(addr_type.pubkey_type))):
  322. do_ab_test( proto, cfg, addr_type, gen1=n+1, kg2=kg2, ag=ag, tool=tool, cache_data=not n )
  323. else: # check specific backend against external tool or another backend
  324. do_ab_test( proto, cfg, addr_type, gen1=cfg.gen1, kg2=kg2, ag=ag, tool=tool, cache_data=False )
  325. def speed_test(proto,kg,ag,rounds):
  326. qmsg(green('Testing speed of address generator {!r} for coin {}'.format(
  327. type(kg).__name__,
  328. proto.coin )))
  329. from struct import pack,unpack
  330. seed = getrand(28)
  331. qmsg('Incrementing key with each round')
  332. qmsg('Starting key: {}'.format( (seed + pack('I',0)).hex() ))
  333. import time
  334. start = last_t = time.time()
  335. for i in range(rounds):
  336. if time.time() - last_t >= 0.1:
  337. qmsg_r(f'\rRound {i+1}/{rounds} ')
  338. last_t = time.time()
  339. sec = PrivKey( proto, seed+pack('I', i), compressed=ag.compressed, pubkey_type=ag.pubkey_type )
  340. addr = ag.to_addr(kg.gen_data(sec))
  341. vmsg(f'\nkey: {sec.wif}\naddr: {addr}\n')
  342. qmsg(
  343. f'\rRound {i+1}/{rounds} ' +
  344. f'\n{rounds} addresses generated' +
  345. ('' if g.test_suite_deterministic else f' in {time.time()-start:.2f} seconds')
  346. )
  347. def dump_test(proto,kg,ag,filename):
  348. with open(filename) as fp:
  349. dump = [[*(e.split()[0] for e in line.split('addr='))] for line in fp.readlines() if 'addr=' in line]
  350. if not dump:
  351. die(1,f'File {filename!r} appears not to be a wallet dump')
  352. qmsg(green(
  353. "A: generator pair '{}:{}'\nB: wallet dump {!r}".format(
  354. type(kg).__name__,
  355. type(ag).__name__,
  356. filename)))
  357. for count,(b_wif,b_addr) in enumerate(dump,1):
  358. qmsg_r(f'\rKey {count}/{len(dump)} ')
  359. try:
  360. b_sec = PrivKey(proto,wif=b_wif)
  361. except:
  362. die(2,f'\nInvalid {proto.network} WIF address in dump file: {b_wif}')
  363. a_addr = ag.to_addr(kg.gen_data(b_sec))
  364. vmsg(f'\nwif: {b_wif}\naddr: {b_addr}\n')
  365. tinfo = (b_sec,b_sec.hex(),b_wif,type(kg).__name__,filename)
  366. test_equal('addresses',a_addr,b_addr,*tinfo)
  367. qmsg(green(('\n','')[bool(opt.verbose)] + 'OK'))
  368. def get_protos(proto,addr_type,toolname):
  369. init_genonly_altcoins(testnet=proto.testnet)
  370. for coin in cinfo.external_tests[proto.network][toolname]:
  371. if coin.lower() not in CoinProtocol.coins:
  372. continue
  373. ret = init_proto(coin,testnet=proto.testnet)
  374. if addr_type not in ret.mmtypes:
  375. continue
  376. yield ret
  377. def parse_args():
  378. if len(cmd_args) != 2:
  379. opts.usage()
  380. arg1,arg2 = cmd_args
  381. cfg = namedtuple('parsed_args',['test','gen1','gen2','rounds','tool','all_backends','dumpfile'])
  382. gen1,gen2,rounds = (0,0,0)
  383. tool,all_backends,dumpfile = (None,None,None)
  384. if is_int(arg1) and is_int(arg2):
  385. test = 'speed'
  386. gen1 = arg1
  387. rounds = arg2
  388. elif is_int(arg1) and os.access(arg2,os.R_OK):
  389. test = 'dump'
  390. gen1 = arg1
  391. dumpfile = arg2
  392. else:
  393. test = 'ab'
  394. rounds = arg2
  395. if not is_int(arg2):
  396. die(1,'Second argument must be dump filename or integer rounds specification')
  397. try:
  398. a,b = arg1.split(':')
  399. except:
  400. die(1,'First argument must be a generator backend number or two colon-separated arguments')
  401. if is_int(a):
  402. gen1 = a
  403. else:
  404. if a == 'all':
  405. all_backends = True
  406. else:
  407. die(1,"First part of first argument must be a generator backend number or 'all'")
  408. if is_int(b):
  409. if opt.all_coins:
  410. die(1,'--all-coins must be used with external tool only')
  411. gen2 = b
  412. else:
  413. tool = b
  414. proto = init_proto_from_opts()
  415. ext_progs = list(cinfo.external_tests[proto.network]) + ['ext']
  416. if b not in ext_progs:
  417. die(1,f'Second part of first argument must be a generator backend number or one of {ext_progs}')
  418. return cfg(
  419. test,
  420. int(gen1) or None,
  421. int(gen2) or None,
  422. int(rounds) or None,
  423. tool,
  424. all_backends,
  425. dumpfile )
  426. def main():
  427. cfg = parse_args()
  428. proto = init_proto_from_opts()
  429. addr_type = MMGenAddrType( proto=proto, id_str=opt.type or proto.dfl_mmtype )
  430. if cfg.test == 'ab':
  431. protos = get_protos(proto,addr_type,cfg.tool) if opt.all_coins else [proto]
  432. for proto in protos:
  433. ab_test( proto, cfg )
  434. else:
  435. kg = KeyGenerator( proto, addr_type.pubkey_type, cfg.gen1 )
  436. ag = AddrGenerator( proto, addr_type )
  437. if cfg.test == 'speed':
  438. speed_test( proto, kg, ag, cfg.rounds )
  439. elif cfg.test == 'dump':
  440. dump_test( proto, kg, ag, cfg.dumpfile )
  441. if saved_results:
  442. import json
  443. with open(results_file,'w') as fp:
  444. fp.write(json.dumps( saved_results, indent=4 ))
  445. from subprocess import run,PIPE,DEVNULL
  446. from collections import namedtuple
  447. from mmgen.protocol import init_proto,init_proto_from_opts,CoinProtocol
  448. from mmgen.altcoin import init_genonly_altcoins,CoinInfo as cinfo
  449. from mmgen.key import PrivKey
  450. from mmgen.addr import KeyGenerator,AddrGenerator,MMGenAddrType
  451. from mmgen.keygen import get_backends
  452. sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
  453. cmd_args = opts.init(opts_data)
  454. main()