From 8c45829682cada2a5812465b3647380db427af76 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 2 Oct 2021 17:54:10 +0000 Subject: [PATCH] add remove_dups() function, use lists instead of sets to preserve order --- mmgen/addr.py | 6 ++++-- mmgen/main_autosign.py | 5 ++--- mmgen/opts.py | 13 ++++++++----- mmgen/tx.py | 26 ++++++++++++++++++-------- mmgen/txsign.py | 7 +++++-- mmgen/util.py | 14 ++++++++++++++ 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/mmgen/addr.py b/mmgen/addr.py index a82754be..8bfcea8b 100755 --- a/mmgen/addr.py +++ b/mmgen/addr.py @@ -426,10 +426,12 @@ Removed {{}} duplicate WIF key{{}} from keylist (also in {pnm} key-address file do_chksum = False elif addrlist: # data from flat address list self.al_id = None - adata = AddrListData([AddrListEntry(proto=proto,addr=a) for a in set(addrlist)]) + addrlist = remove_dups(addrlist,edesc='address',desc='address list') + adata = AddrListData([AddrListEntry(proto=proto,addr=a) for a in addrlist]) elif keylist: # data from flat key list self.al_id = None - adata = AddrListData([AddrListEntry(proto=proto,sec=PrivKey(proto=proto,wif=k)) for k in set(keylist)]) + keylist = remove_dups(keylist,edesc='key',desc='key list',hide=True) + adata = AddrListData([AddrListEntry(proto=proto,sec=PrivKey(proto=proto,wif=k)) for k in keylist]) elif seed or addr_idxs: die(3,'Must specify both seed and addr indexes') elif al_id or adata: diff --git a/mmgen/main_autosign.py b/mmgen/main_autosign.py index 3d7b1283..ff3969d2 100755 --- a/mmgen/main_autosign.py +++ b/mmgen/main_autosign.py @@ -217,9 +217,8 @@ async def sign_tx_file(txfile): return False async def sign(): - dirlist = os.listdir(tx_dir) - raw,signed = [set(f[:-6] for f in dirlist if f.endswith(ext)) for ext in ('.rawtx','.sigtx')] - unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw - signed] + raw,signed = [tuple(f[:-6] for f in os.listdir(tx_dir) if f.endswith(ext)) for ext in ('.rawtx','.sigtx')] + unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw if f not in signed] if unsigned: signed_txs,fails = [],[] diff --git a/mmgen/opts.py b/mmgen/opts.py index 7a46cb52..64735b1a 100755 --- a/mmgen/opts.py +++ b/mmgen/opts.py @@ -164,12 +164,15 @@ def show_common_opts_diff(): def do_fmt(set_data): return fmt_list(['--'+s.replace('_','-') for s in set_data],fmt='col',indent=' ') - a = set(g.common_opts) - b = set(common_opts_data_to_list()) + a = g.common_opts + b = list(common_opts_data_to_list()) + a_minus_b = [e for e in a if e not in b] + b_minus_a = [e for e in b if e not in a] + a_and_b = [e for e in a if e in b] - msg(f'g.common_opts - common_opts_data:\n {do_fmt(a-b) if a-b else "None"}\n') - msg(f'common_opts_data - g.common_opts (these do not set global var):\n{do_fmt(b-a)}\n') - msg(f'common_opts_data ^ g.common_opts (these set global var):\n{do_fmt(b.intersection(a))}\n') + msg(f'g.common_opts - common_opts_data:\n {do_fmt(a_minus_b) if a_minus_b else "None"}\n') + msg(f'common_opts_data - g.common_opts (these do not set global var):\n{do_fmt(b_minus_a)}\n') + msg(f'common_opts_data ^ g.common_opts (these set global var):\n{do_fmt(a_and_b)}\n') sys.exit(0) diff --git a/mmgen/tx.py b/mmgen/tx.py index 618dfc56..db66bf1a 100755 --- a/mmgen/tx.py +++ b/mmgen/tx.py @@ -495,7 +495,10 @@ class MMGenTX: return False def get_non_mmaddrs(self,desc): - return {i.addr for i in getattr(self,desc) if not i.mmid} + return remove_dups( + (i.addr for i in getattr(self,desc) if not i.mmid), + edesc = 'non-MMGen address', + quiet = True ) def check_non_mmgen_inputs(self,caller,non_mmaddrs=None): non_mmaddrs = non_mmaddrs or self.get_non_mmaddrs('inputs') @@ -714,8 +717,16 @@ class MMGenTX: async def get_outputs_from_cmdline(self,cmd_args): from .addr import AddrList,AddrData,TwAddrData - addrfiles = [a for a in cmd_args if get_extension(a) == AddrList.ext] - cmd_args = set(cmd_args) - set(addrfiles) + addrfiles = remove_dups( + tuple(a for a in cmd_args if get_extension(a) == AddrList.ext), + desc = 'command line', + edesc = 'argument', + ) + cmd_args = remove_dups( + tuple(a for a in cmd_args if a not in addrfiles), + desc = 'command line', + edesc = 'argument', + ) ad_f = AddrData(self.proto) for a in addrfiles: @@ -1208,11 +1219,10 @@ class MMGenTX: if hasattr(e,attr): delattr(e,attr) - def get_input_sids(self): - return set(e.mmid.sid for e in self.inputs if e.mmid) - - def get_output_sids(self): - return set(e.mmid.sid for e in self.outputs if e.mmid) + def get_sids(self,desc): + return remove_dups( + (e.mmid.sid for e in getattr(self,desc) if e.mmid), + quiet = True ) async def sign(self,tx_num_str,keys): # return signed object or False; don't exit or raise exception diff --git a/mmgen/txsign.py b/mmgen/txsign.py index 25e7ebbd..1306ac19 100755 --- a/mmgen/txsign.py +++ b/mmgen/txsign.py @@ -57,7 +57,7 @@ def get_seed_for_seed_id(sid,infiles,saved_seeds): def generate_kals_for_mmgen_addrs(need_keys,infiles,saved_seeds,proto): mmids = [e.mmid for e in need_keys] - sids = {i.sid for i in mmids} + sids = remove_dups((i.sid for i in mmids),quiet=True) vmsg(f"Need seed{suf(sids)}: {' '.join(sids)}") def gen_kals(): for sid in sids: @@ -169,7 +169,10 @@ async def txsign(tx,seed_files,kl,kal,tx_num_str=''): tx.delete_attrs('inputs','have_wif') tx.delete_attrs('outputs','have_wif') - extra_sids = set(saved_seeds) - tx.get_input_sids() - tx.get_output_sids() + extra_sids = remove_dups( + (s for s in saved_seeds if s not in tx.get_sids('inputs') + tx.get_sids('outputs')), + quiet = True ) + if extra_sids: msg(f"Unused Seed ID{suf(extra_sids)}: {' '.join(extra_sids)}") diff --git a/mmgen/util.py b/mmgen/util.py index 6e3182a4..14646096 100755 --- a/mmgen/util.py +++ b/mmgen/util.py @@ -130,6 +130,20 @@ def list_gen(*data): yield i[0] return list(gen()) +def remove_dups(iterable,edesc='element',desc='list',quiet=False,hide=False): + """ + Remove duplicate occurrences of iterable elements, preserving first occurrence + If iterable is a generator, return a list, else type(iterable) + """ + ret = [] + for e in iterable: + if e in ret: + if not quiet: + ymsg(f'Warning: removing duplicate {edesc} {"(hidden)" if hide else e} in {desc}') + else: + ret.append(e) + return ret if type(iterable).__name__ == 'generator' else type(iterable)(ret) + def exit_if_mswin(feature): if g.platform == 'win': m = capfirst(feature) + ' not supported on the MSWin / MSYS2 platform'