add remove_dups() function, use lists instead of sets to preserve order

mmgen/addr.py

@@ -426,10 +426,12 @@ Removed {{}} duplicate WIF key{{}} from keylist (also in {pnm} key-address file
 			do_chksum = False
 		elif addrlist:           # data from flat address list
 			self.al_id = None
-			adata = AddrListData([AddrListEntry(proto=proto,addr=a) for a in set(addrlist)])
+			addrlist = remove_dups(addrlist,edesc='address',desc='address list')
+			adata = AddrListData([AddrListEntry(proto=proto,addr=a) for a in addrlist])
 		elif keylist:            # data from flat key list
 			self.al_id = None
-			adata = AddrListData([AddrListEntry(proto=proto,sec=PrivKey(proto=proto,wif=k)) for k in set(keylist)])
+			keylist = remove_dups(keylist,edesc='key',desc='key list',hide=True)
+			adata = AddrListData([AddrListEntry(proto=proto,sec=PrivKey(proto=proto,wif=k)) for k in keylist])
 		elif seed or addr_idxs:
 			die(3,'Must specify both seed and addr indexes')
 		elif al_id or adata:

mmgen/main_autosign.py

@@ -217,9 +217,8 @@ async def sign_tx_file(txfile):
 		return False
 
 async def sign():
-	dirlist  = os.listdir(tx_dir)
-	raw,signed = [set(f[:-6] for f in dirlist if f.endswith(ext)) for ext in ('.rawtx','.sigtx')]
-	unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw - signed]
+	raw,signed = [tuple(f[:-6] for f in os.listdir(tx_dir) if f.endswith(ext)) for ext in ('.rawtx','.sigtx')]
+	unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw if f not in signed]
 
 	if unsigned:
 		signed_txs,fails = [],[]

mmgen/opts.py

@@ -164,12 +164,15 @@ def show_common_opts_diff():
 	def do_fmt(set_data):
 		return fmt_list(['--'+s.replace('_','-') for s in set_data],fmt='col',indent='   ')
 
-	a = set(g.common_opts)
-	b = set(common_opts_data_to_list())
-
-	msg(f'g.common_opts - common_opts_data:\n   {do_fmt(a-b) if a-b else "None"}\n')
-	msg(f'common_opts_data - g.common_opts (these do not set global var):\n{do_fmt(b-a)}\n')
-	msg(f'common_opts_data ^ g.common_opts (these set global var):\n{do_fmt(b.intersection(a))}\n')
+	a = g.common_opts
+	b = list(common_opts_data_to_list())
+	a_minus_b = [e for e in a if e not in b]
+	b_minus_a = [e for e in b if e not in a]
+	a_and_b   = [e for e in a if e in b]
+
+	msg(f'g.common_opts - common_opts_data:\n   {do_fmt(a_minus_b) if a_minus_b else "None"}\n')
+	msg(f'common_opts_data - g.common_opts (these do not set global var):\n{do_fmt(b_minus_a)}\n')
+	msg(f'common_opts_data ^ g.common_opts (these set global var):\n{do_fmt(a_and_b)}\n')
 
 	sys.exit(0)
 

mmgen/tx.py

@@ -495,7 +495,10 @@ class MMGenTX:
 				return False
 
 		def get_non_mmaddrs(self,desc):
-			return {i.addr for i in getattr(self,desc) if not i.mmid}
+			return remove_dups(
+				(i.addr for i in getattr(self,desc) if not i.mmid),
+				edesc = 'non-MMGen address',
+				quiet = True )
 
 		def check_non_mmgen_inputs(self,caller,non_mmaddrs=None):
 			non_mmaddrs = non_mmaddrs or self.get_non_mmaddrs('inputs')
@@ -714,8 +717,16 @@ class MMGenTX:
 
 		async def get_outputs_from_cmdline(self,cmd_args):
 			from .addr import AddrList,AddrData,TwAddrData
-			addrfiles = [a for a in cmd_args if get_extension(a) == AddrList.ext]
-			cmd_args = set(cmd_args) - set(addrfiles)
+			addrfiles = remove_dups(
+				tuple(a for a in cmd_args if get_extension(a) == AddrList.ext),
+				desc = 'command line',
+				edesc = 'argument',
+			)
+			cmd_args  = remove_dups(
+				tuple(a for a in cmd_args if a not in addrfiles),
+				desc = 'command line',
+				edesc = 'argument',
+			)
 
 			ad_f = AddrData(self.proto)
 			for a in addrfiles:
@@ -1208,11 +1219,10 @@ class MMGenTX:
 				if hasattr(e,attr):
 					delattr(e,attr)
 
-		def get_input_sids(self):
-			return set(e.mmid.sid for e in self.inputs if e.mmid)
-
-		def get_output_sids(self):
-			return set(e.mmid.sid for e in self.outputs if e.mmid)
+		def get_sids(self,desc):
+			return remove_dups(
+				(e.mmid.sid for e in getattr(self,desc) if e.mmid),
+				quiet = True )
 
 		async def sign(self,tx_num_str,keys): # return signed object or False; don't exit or raise exception
 

mmgen/txsign.py

@@ -57,7 +57,7 @@ def get_seed_for_seed_id(sid,infiles,saved_seeds):
 
 def generate_kals_for_mmgen_addrs(need_keys,infiles,saved_seeds,proto):
 	mmids = [e.mmid for e in need_keys]
-	sids = {i.sid for i in mmids}
+	sids = remove_dups((i.sid for i in mmids),quiet=True)
 	vmsg(f"Need seed{suf(sids)}: {' '.join(sids)}")
 	def gen_kals():
 		for sid in sids:
@@ -169,7 +169,10 @@ async def txsign(tx,seed_files,kl,kal,tx_num_str=''):
 	tx.delete_attrs('inputs','have_wif')
 	tx.delete_attrs('outputs','have_wif')
 
-	extra_sids = set(saved_seeds) - tx.get_input_sids() - tx.get_output_sids()
+	extra_sids = remove_dups(
+		(s for s in saved_seeds if s not in tx.get_sids('inputs') + tx.get_sids('outputs')),
+		quiet = True )
+
 	if extra_sids:
 		msg(f"Unused Seed ID{suf(extra_sids)}: {' '.join(extra_sids)}")
 

mmgen/util.py

@@ -130,6 +130,20 @@ def list_gen(*data):
 				yield i[0]
 	return list(gen())
 
+def remove_dups(iterable,edesc='element',desc='list',quiet=False,hide=False):
+	"""
+	Remove duplicate occurrences of iterable elements, preserving first occurrence
+	If iterable is a generator, return a list, else type(iterable)
+	"""
+	ret = []
+	for e in iterable:
+		if e in ret:
+			if not quiet:
+				ymsg(f'Warning: removing duplicate {edesc} {"(hidden)" if hide else e} in {desc}')
+		else:
+			ret.append(e)
+	return ret if type(iterable).__name__ == 'generator' else type(iterable)(ret)
+
 def exit_if_mswin(feature):
 	if g.platform == 'win':
 		m = capfirst(feature) + ' not supported on the MSWin / MSYS2 platform'