gentest.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584
  1. #!/usr/bin/env python3
  2. #
  3. # mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution
  4. # Copyright (C)2013-2023 The MMGen Project <mmgen@tuta.io>
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. """
  19. test/gentest.py: Cryptocoin key/address generation tests for the MMGen suite
  20. """
  21. import sys,os,time
  22. import include.test_init
  23. # Import these _after_ local path's been added to sys.path
  24. import mmgen.opts as opts
  25. from mmgen.cfg import gc,Config
  26. from mmgen.color import green,red,purple
  27. from mmgen.util import msg,ymsg,capfirst,is_int,die
  28. results_file = 'gentest.out.json'
  29. rounds = 100
  30. opts_data = {
  31. 'text': {
  32. 'desc': 'Test key/address generation of the MMGen suite in various ways',
  33. 'usage':'[options] <spec> <rounds | dump file>',
  34. 'options': """
  35. -h, --help Print this help message
  36. --, --longhelp Print help message for long options (common options)
  37. -a, --all-coins Test all coins supported by specified external tool
  38. -k, --use-internal-keccak-module Force use of the internal keccak module
  39. -q, --quiet Produce quieter output
  40. -s, --save-results Save output of external tool in Compare test to
  41. {rf!r}
  42. -t, --type=t Specify address type (e.g. 'compressed','segwit',
  43. 'zcash_z','bech32')
  44. -v, --verbose Produce more verbose output
  45. """,
  46. 'notes': """
  47. TEST TYPES:
  48. Compare: {prog} A:B <rounds> (compare address generators A and B)
  49. Speed: {prog} A <rounds> (test speed of generator A)
  50. Dump: {prog} A <dump file> (compare generator A to wallet dump)
  51. where:
  52. A and B are keygen backend numbers ('1' being the default); or
  53. B is the name of an external tool (see below) or 'ext'.
  54. If B is 'ext', the external tool will be chosen automatically.
  55. For the Compare test, A may be 'all' to test all backends for the current
  56. coin/address type combination.
  57. EXAMPLES:
  58. Compare addresses generated by 'libsecp256k1' and 'python-ecdsa' backends,
  59. with 100 random rounds plus private-key edge cases:
  60. $ {prog} 1:2 100
  61. Compare Segwit addresses from default 'libsecp256k1' backend to 'pycoin'
  62. library for all supported coins, 100 rounds + edge cases:
  63. $ {prog} --all-coins --type=segwit 1:pycoin 100
  64. Compare addresses from 'python-ecdsa' backend to output of 'keyconv' tool
  65. for all supported coins, 100 rounds + edge cases:
  66. $ {prog} --all-coins --type=compressed 2:keyconv 100
  67. Compare bech32 addrs from 'libsecp256k1' backend to Bitcoin Core wallet
  68. dump:
  69. $ {prog} --type=bech32 1 bech32wallet.dump
  70. Compare addresses from Monero 'ed25519ll' backend to output of default
  71. external tool, 10 rounds + edge cases:
  72. $ {prog} --coin=xmr 3:ext 10
  73. Test the speed of default Monero 'nacl' backend, 10,000 rounds:
  74. $ test/gentest.py --coin=xmr 1 10000
  75. Same for Zcash:
  76. $ test/gentest.py --coin=zec --type=zcash_z 1 10000
  77. Test all configured Monero backends against the 'monero-python' library, 3 rounds
  78. + edge cases:
  79. $ test/gentest.py --coin=xmr all:monero-python 3
  80. Test 'nacl' and 'ed25519ll_djbec' backends against each other, 10,000 rounds
  81. + edge cases:
  82. $ test/gentest.py --coin=xmr 1:2 10000
  83. SUPPORTED EXTERNAL TOOLS:
  84. + ethkey (for ETH,ETC)
  85. https://github.com/openethereum/openethereum
  86. (build with 'cargo build -p ethkey-cli --release')
  87. + zcash-mini (for Zcash-Z addresses and view keys)
  88. https://github.com/FiloSottile/zcash-mini
  89. + monero-python (for Monero addresses and view keys)
  90. https://github.com/monero-ecosystem/monero-python
  91. + pycoin (for supported coins)
  92. https://github.com/richardkiss/pycoin
  93. + keyconv (for supported coins)
  94. https://github.com/exploitagency/vanitygen-plus
  95. ('keyconv' does not generate Segwit addresses)
  96. """
  97. },
  98. 'code': {
  99. 'options': lambda s: s.format(
  100. rf=results_file,
  101. ),
  102. 'notes': lambda s: s.format(
  103. prog='test/gentest.py',
  104. pnm=gc.proj_name,
  105. snum=rounds )
  106. }
  107. }
  108. def get_cmd_output(cmd,input=None):
  109. return run(cmd,input=input,stdout=PIPE,stderr=DEVNULL).stdout.decode().splitlines()
  110. saved_results = {}
  111. class GenTool:
  112. def __init__(self,proto,addr_type):
  113. self.proto = proto
  114. self.addr_type = addr_type
  115. self.data = {}
  116. def __del__(self):
  117. if cfg.save_results:
  118. key = f'{self.proto.coin}-{self.proto.network}-{self.addr_type.name}-{self.desc}'.lower()
  119. saved_results[key] = {k.hex():v._asdict() for k,v in self.data.items()}
  120. def run_tool(self,sec,cache_data):
  121. vcoin = 'BTC' if self.proto.coin == 'BCH' else self.proto.coin
  122. key = sec.orig_bytes
  123. if key in self.data:
  124. return self.data[key]
  125. else:
  126. ret = self.run(sec,vcoin)
  127. if cache_data:
  128. self.data[key] = sd( **{'reduced':sec.hex()}, **ret._asdict() )
  129. return ret
  130. class GenToolEthkey(GenTool):
  131. desc = 'ethkey'
  132. def __init__(self,*args,**kwargs):
  133. self.cmdname = get_ethkey()
  134. return super().__init__(*args,**kwargs)
  135. def run(self,sec,vcoin):
  136. o = get_cmd_output([self.cmdname,'info',sec.hex()])
  137. return gtr(
  138. o[0].split()[1],
  139. o[-1].split()[1],
  140. None )
  141. class GenToolKeyconv(GenTool):
  142. desc = 'keyconv'
  143. def run(self,sec,vcoin):
  144. o = get_cmd_output(['keyconv','-C',vcoin,sec.wif])
  145. return gtr(
  146. o[1].split()[1],
  147. o[0].split()[1],
  148. None )
  149. class GenToolZcash_mini(GenTool):
  150. desc = 'zcash-mini'
  151. def run(self,sec,vcoin):
  152. o = get_cmd_output(['zcash-mini','-key','-simple'],input=(sec.wif+'\n').encode())
  153. return gtr( o[1], o[0], o[-1] )
  154. class GenToolPycoin(GenTool):
  155. """
  156. pycoin/networks/all.py pycoin/networks/legacy_networks.py
  157. """
  158. desc = 'pycoin'
  159. def __init__(self,*args,**kwargs):
  160. super().__init__(*args,**kwargs)
  161. try:
  162. from pycoin.networks.registry import network_for_netcode
  163. except:
  164. raise ImportError(
  165. 'Unable to import pycoin.networks.registry. Is pycoin installed on your system?')
  166. self.nfnc = network_for_netcode
  167. def run(self,sec,vcoin):
  168. if self.proto.testnet:
  169. vcoin = cinfo.external_tests['testnet']['pycoin'][vcoin]
  170. network = self.nfnc(vcoin)
  171. key = network.keys.private(
  172. secret_exponent = int(sec.hex(),16),
  173. is_compressed = self.addr_type.name != 'legacy' )
  174. if key is None:
  175. die(1,f'can’t parse {sec.hex()}')
  176. if self.addr_type.name in ('segwit','bech32'):
  177. hash160_c = key.hash160(is_compressed=True)
  178. if self.addr_type.name == 'segwit':
  179. p2sh_script = network.contract.for_p2pkh_wit(hash160_c)
  180. addr = network.address.for_p2s(p2sh_script)
  181. else:
  182. addr = network.address.for_p2pkh_wit(hash160_c)
  183. else:
  184. addr = key.address()
  185. return gtr( key.wif(), addr, None )
  186. class GenToolMonero_python(GenTool):
  187. desc = 'monero-python'
  188. def __init__(self,*args,**kwargs):
  189. super().__init__(*args,**kwargs)
  190. from mmgen.util import load_cryptodomex
  191. load_cryptodomex()
  192. try:
  193. from monero.seed import Seed
  194. except:
  195. raise ImportError(
  196. 'Unable to import monero-python. Is monero-python installed on your system?')
  197. self.Seed = Seed
  198. def run(self,sec,vcoin):
  199. seed = self.Seed( sec.orig_bytes.hex() )
  200. sk = seed.secret_spend_key()
  201. vk = seed.secret_view_key()
  202. addr = seed.public_address()
  203. return gtr( sk, addr, vk )
  204. def find_or_check_tool(proto,addr_type,toolname):
  205. ext_progs = list(cinfo.external_tests[proto.network])
  206. if toolname not in ext_progs + ['ext']:
  207. die(1,f'{toolname!r}: unsupported tool for network {proto.network}')
  208. if cfg.all_coins and toolname == 'ext':
  209. die(1,"'--all-coins' must be combined with a specific external testing tool")
  210. else:
  211. tool = cinfo.get_test_support(
  212. proto.coin,
  213. addr_type.name,
  214. proto.network,
  215. verbose = not cfg.quiet,
  216. toolname = toolname if toolname != 'ext' else None )
  217. if tool and toolname in ext_progs and toolname != tool:
  218. sys.exit(3)
  219. if tool == None:
  220. return None
  221. return tool
  222. def test_equal(desc,a_val,b_val,in_bytes,sec,wif,a_desc,b_desc):
  223. if a_val != b_val:
  224. fs = """
  225. {i:{w}}: {}
  226. {s:{w}}: {}
  227. {W:{w}}: {}
  228. {a:{w}}: {}
  229. {b:{w}}: {}
  230. """
  231. die(3,
  232. red('\nERROR: {} do not match!').format(desc)
  233. + fs.format(
  234. in_bytes.hex(), sec, wif, a_val, b_val,
  235. i='input', s='sec key', W='WIF key', a=a_desc, b=b_desc,
  236. w=max(len(e) for e in (a_desc,b_desc)) + 1
  237. ).rstrip())
  238. def do_ab_test(proto,scfg,addr_type,gen1,kg2,ag,tool,cache_data):
  239. def do_ab_inner(n,trounds,in_bytes):
  240. global last_t
  241. if cfg.verbose or time.time() - last_t >= 0.1:
  242. qmsg_r(f'\rRound {i+1}/{trounds} ')
  243. last_t = time.time()
  244. sec = PrivKey(proto,in_bytes,compressed=addr_type.compressed,pubkey_type=addr_type.pubkey_type)
  245. data = kg1.gen_data(sec)
  246. addr1 = ag.to_addr(data)
  247. tinfo = ( in_bytes, sec, sec.wif, type(kg1).__name__, type(kg2).__name__ if kg2 else tool.desc )
  248. def do_msg():
  249. if cfg.verbose:
  250. msg( fs.format( b=in_bytes.hex(), r=sec.hex(), k=sec.wif, v=vk2, a=addr1 ))
  251. if tool:
  252. def run_tool():
  253. o = tool.run_tool(sec,cache_data)
  254. test_equal( 'WIF keys', sec.wif, o.wif, *tinfo )
  255. test_equal( 'addresses', addr1, o.addr, *tinfo )
  256. if o.viewkey:
  257. test_equal( 'view keys', ag.to_viewkey(data), o.viewkey, *tinfo )
  258. return o.viewkey
  259. vk2 = run_tool()
  260. do_msg()
  261. else:
  262. test_equal( 'addresses', addr1, ag.to_addr(kg2.gen_data(sec)), *tinfo )
  263. vk2 = None
  264. do_msg()
  265. qmsg_r(f'\rRound {n+1}/{trounds} ')
  266. def get_randbytes():
  267. if tool and len(tool.data) > len(edgecase_sks):
  268. for privbytes in tuple(tool.data)[len(edgecase_sks):]:
  269. yield privbytes
  270. else:
  271. for i in range(scfg.rounds):
  272. yield getrand(32)
  273. kg1 = KeyGenerator( cfg, proto, addr_type.pubkey_type, gen1 )
  274. if type(kg1) == type(kg2):
  275. die(4,'Key generators are the same!')
  276. e = cinfo.get_entry(proto.coin,proto.network)
  277. qmsg(green("Comparing address generators '{A}' and '{B}' for {N} {c} ({n}), addrtype {a!r}".format(
  278. A = type(kg1).__name__.replace('_','-'),
  279. B = type(kg2).__name__.replace('_','-') if kg2 else tool.desc,
  280. N = proto.network,
  281. c = proto.coin,
  282. n = e.name if e else '---',
  283. a = addr_type.name )))
  284. global last_t
  285. last_t = time.time()
  286. fs = (
  287. '\ninput: {b}' +
  288. '\nreduced: {r}' +
  289. '\n{:9} {{k}}'.format(addr_type.wif_label+':') +
  290. ('\nviewkey: {v}' if 'viewkey' in addr_type.extra_attrs else '') +
  291. '\naddr: {a}\n' )
  292. ge = CoinProtocol.Secp256k1.secp256k1_ge
  293. # test some important private key edge cases:
  294. edgecase_sks = (
  295. bytes([0x00]*31 + [0x01]), # min
  296. bytes([0xff]*32), # max
  297. bytes([0x0f] + [0xff]*31), # produces same key as above for zcash-z
  298. int.to_bytes(ge + 1, 32, 'big'), # bitcoin will reduce
  299. int.to_bytes(ge - 1, 32, 'big'), # bitcoin will not reduce
  300. bytes([0x00]*31 + [0xff]), # monero will reduce
  301. bytes([0xff]*31 + [0x0f]), # monero will not reduce
  302. bytes.fromhex('deadbeef'*8),
  303. )
  304. qmsg(purple('edge cases:'))
  305. for i,privbytes in enumerate(edgecase_sks):
  306. do_ab_inner(i,len(edgecase_sks),privbytes)
  307. qmsg(green('\rOK ' if cfg.verbose else 'OK'))
  308. qmsg(purple('random input:'))
  309. for i,privbytes in enumerate(get_randbytes()):
  310. do_ab_inner(i,scfg.rounds,privbytes)
  311. qmsg(green('\rOK ' if cfg.verbose else 'OK'))
  312. def init_tool(proto,addr_type,toolname):
  313. return globals()['GenTool'+capfirst(toolname.replace('-','_'))](proto,addr_type)
  314. def ab_test(proto,scfg):
  315. addr_type = MMGenAddrType( proto=proto, id_str=cfg.type or proto.dfl_mmtype )
  316. if scfg.gen2:
  317. assert scfg.gen1 != 'all', "'all' must be used only with external tool"
  318. kg2 = KeyGenerator( cfg, proto, addr_type.pubkey_type, scfg.gen2 )
  319. tool = None
  320. else:
  321. toolname = find_or_check_tool( proto, addr_type, scfg.tool )
  322. if toolname == None:
  323. ymsg(f'Warning: skipping tool {scfg.tool!r} for {proto.coin} {addr_type.name}')
  324. return
  325. tool = init_tool( proto, addr_type, toolname )
  326. kg2 = None
  327. ag = AddrGenerator( cfg, proto, addr_type )
  328. if scfg.all_backends: # check all backends against external tool
  329. for n in range(len(get_backends(addr_type.pubkey_type))):
  330. do_ab_test(
  331. proto,
  332. scfg,
  333. addr_type,
  334. gen1 = n+1,
  335. kg2 = kg2,
  336. ag = ag,
  337. tool = tool,
  338. cache_data = scfg.rounds < 1000 and not n)
  339. else: # check specific backend against external tool or another backend
  340. do_ab_test(
  341. proto,
  342. scfg,
  343. addr_type,
  344. gen1 = scfg.gen1,
  345. kg2 = kg2,
  346. ag = ag,
  347. tool = tool,
  348. cache_data = False)
  349. def speed_test(proto,kg,ag,rounds):
  350. qmsg(green('Testing speed of address generator {!r} for coin {}'.format(
  351. type(kg).__name__,
  352. proto.coin )))
  353. from struct import pack,unpack
  354. seed = getrand(28)
  355. qmsg('Incrementing key with each round')
  356. qmsg('Starting key: {}'.format( (seed + pack('I',0)).hex() ))
  357. import time
  358. start = last_t = time.time()
  359. for i in range(rounds):
  360. if time.time() - last_t >= 0.1:
  361. qmsg_r(f'\rRound {i+1}/{rounds} ')
  362. last_t = time.time()
  363. sec = PrivKey( proto, seed+pack('I', i), compressed=ag.compressed, pubkey_type=ag.pubkey_type )
  364. addr = ag.to_addr(kg.gen_data(sec))
  365. vmsg(f'\nkey: {sec.wif}\naddr: {addr}\n')
  366. qmsg(
  367. f'\rRound {i+1}/{rounds} ' +
  368. f'\n{rounds} addresses generated' +
  369. ('' if cfg.test_suite_deterministic else f' in {time.time()-start:.2f} seconds')
  370. )
  371. def dump_test(proto,kg,ag,filename):
  372. with open(filename) as fp:
  373. dump = [[*(e.split()[0] for e in line.split('addr='))] for line in fp.readlines() if 'addr=' in line]
  374. if not dump:
  375. die(1,f'File {filename!r} appears not to be a wallet dump')
  376. qmsg(green(
  377. "A: generator pair '{}:{}'\nB: wallet dump {!r}".format(
  378. type(kg).__name__,
  379. type(ag).__name__,
  380. filename)))
  381. for count,(b_wif,b_addr) in enumerate(dump,1):
  382. qmsg_r(f'\rKey {count}/{len(dump)} ')
  383. try:
  384. b_sec = PrivKey(proto,wif=b_wif)
  385. except:
  386. die(2,f'\nInvalid {proto.network} WIF address in dump file: {b_wif}')
  387. a_addr = ag.to_addr(kg.gen_data(b_sec))
  388. vmsg(f'\nwif: {b_wif}\naddr: {b_addr}\n')
  389. tinfo = (b_sec,b_sec.hex(),b_wif,type(kg).__name__,filename)
  390. test_equal('addresses',a_addr,b_addr,*tinfo)
  391. qmsg(green(('\n','')[bool(cfg.verbose)] + 'OK'))
  392. def get_protos(proto,addr_type,toolname):
  393. init_genonly_altcoins(testnet=proto.testnet)
  394. for coin in cinfo.external_tests[proto.network][toolname]:
  395. if coin.lower() not in CoinProtocol.coins:
  396. continue
  397. ret = init_proto( cfg, coin, testnet=proto.testnet )
  398. if addr_type not in ret.mmtypes:
  399. continue
  400. yield ret
  401. def parse_args():
  402. if len(cfg._args) != 2:
  403. cfg._opts.usage()
  404. arg1,arg2 = cfg._args
  405. gen1,gen2,rounds = (0,0,0)
  406. tool,all_backends,dumpfile = (None,None,None)
  407. if is_int(arg1) and is_int(arg2):
  408. test = 'speed'
  409. gen1 = arg1
  410. rounds = arg2
  411. elif is_int(arg1) and os.access(arg2,os.R_OK):
  412. test = 'dump'
  413. gen1 = arg1
  414. dumpfile = arg2
  415. else:
  416. test = 'ab'
  417. rounds = arg2
  418. if not is_int(arg2):
  419. die(1,'Second argument must be dump filename or integer rounds specification')
  420. try:
  421. a,b = arg1.split(':')
  422. except:
  423. die(1,'First argument must be a generator backend number or two colon-separated arguments')
  424. if is_int(a):
  425. gen1 = a
  426. else:
  427. if a == 'all':
  428. all_backends = True
  429. else:
  430. die(1,"First part of first argument must be a generator backend number or 'all'")
  431. if is_int(b):
  432. if cfg.all_coins:
  433. die(1,'--all-coins must be used with external tool only')
  434. gen2 = b
  435. else:
  436. tool = b
  437. ext_progs = list(cinfo.external_tests[cfg._proto.network]) + ['ext']
  438. if b not in ext_progs:
  439. die(1,f'Second part of first argument must be a generator backend number or one of {ext_progs}')
  440. return namedtuple('parsed_args',['test','gen1','gen2','rounds','tool','all_backends','dumpfile'])(
  441. test,
  442. int(gen1) or None,
  443. int(gen2) or None,
  444. int(rounds) or None,
  445. tool,
  446. all_backends,
  447. dumpfile )
  448. def main():
  449. scfg = parse_args()
  450. addr_type = MMGenAddrType( proto=proto, id_str=cfg.type or proto.dfl_mmtype )
  451. if scfg.test == 'ab':
  452. protos = get_protos(proto,addr_type,scfg.tool) if cfg.all_coins else [proto]
  453. for p in protos:
  454. ab_test( p, scfg )
  455. else:
  456. kg = KeyGenerator( cfg, proto, addr_type.pubkey_type, scfg.gen1 )
  457. ag = AddrGenerator( cfg, proto, addr_type )
  458. if scfg.test == 'speed':
  459. speed_test( proto, kg, ag, scfg.rounds )
  460. elif scfg.test == 'dump':
  461. dump_test( proto, kg, ag, scfg.dumpfile )
  462. if saved_results:
  463. import json
  464. with open(results_file,'w') as fp:
  465. fp.write(json.dumps( saved_results, indent=4 ))
  466. from subprocess import run,PIPE,DEVNULL
  467. from collections import namedtuple
  468. from mmgen.protocol import init_proto,CoinProtocol
  469. from mmgen.altcoin import init_genonly_altcoins
  470. from mmgen.altcoin import CoinInfo as cinfo
  471. from mmgen.key import PrivKey
  472. from mmgen.addr import MMGenAddrType
  473. from mmgen.addrgen import KeyGenerator,AddrGenerator
  474. from mmgen.keygen import get_backends
  475. from test.include.common import getrand,get_ethkey,set_globals
  476. gtr = namedtuple('gen_tool_result',['wif','addr','viewkey'])
  477. sd = namedtuple('saved_data_item',['reduced','wif','addr','viewkey'])
  478. sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
  479. cfg = Config(opts_data=opts_data)
  480. set_globals(cfg)
  481. qmsg = cfg._util.qmsg
  482. qmsg_r = cfg._util.qmsg_r
  483. vmsg = cfg._util.vmsg
  484. proto = cfg._proto
  485. main()