Support UTF-8 wallet labels; new UTF-8 restrictions

- this means .mmdat wallet file itself is now UTF-8 (formerly ASCII-only)
- control and combining characters are now forbidden in all labels

mmgen/main_wallet.py

@@ -125,7 +125,7 @@ if invoked_as in ('conv','passchg'):
 ss_in = None if invoked_as == 'gen' else SeedSource(sf,passchg=(invoked_as=='passchg'))
 if invoked_as == 'chk':
 	lbl = ss_in.ssdata.label.hl() if hasattr(ss_in.ssdata,'label') else 'NONE'
-	vmsg('Wallet label: {}'.format(lbl))
+	vmsg(u'Wallet label: {}'.format(lbl))
 	# TODO: display creation date
 	sys.exit(0)
 

mmgen/obj.py

@@ -670,7 +670,12 @@ class MMGenLabel(unicode,Hilite,InitErrors):
 			s = s.strip()
 			if type(s) != unicode:
 				s = s.decode('utf8')
-			from mmgen.util import capfirst
+			for ch in s:
+				# Allow:    (L)etter,(N)umber,(P)unctuation,(S)ymbol,(Z)space
+				# Disallow: (C)ontrol,(M)combining
+				# Combining characters create width formatting issues, so disallow them for now
+				assert unicodedata.category(ch)[0] not in 'CM','{!r}: {} characters not allowed'.format(
+					ch,('control','combining')[unicodedata.category(ch)[0]=='M'])
 			assert len(s) <= cls.max_len, 'too long (>{} symbols)'.format(cls.max_len)
 			assert len(s) >= cls.min_len, 'too short (<{} symbols)'.format(cls.min_len)
 			assert not cls.allowed or set(list(s)).issubset(set(cls.allowed)),\
@@ -684,7 +689,6 @@ class MMGenLabel(unicode,Hilite,InitErrors):
 
 class MMGenWalletLabel(MMGenLabel):
 	max_len = 48
-	allowed = map(unichr,range(32,127))
 	desc = 'wallet label'
 
 class TwComment(MMGenLabel):

mmgen/seed.py

@@ -582,8 +582,8 @@ class Wallet (SeedSourceEnc):
 	ext = 'mmdat'
 
 	def _get_label_from_user(self,old_lbl=''):
-		d = "to reuse the label '{}'".format(old_lbl.hl()) if old_lbl else 'for no label'
-		p = 'Enter a wallet label, or hit ENTER {}: '.format(d)
+		d = u"to reuse the label '{}'".format(old_lbl.hl()) if old_lbl else 'for no label'
+		p = u'Enter a wallet label, or hit ENTER {}: '.format(d)
 		while True:
 			msg_r(p)
 			ret = my_raw_input('')
@@ -603,19 +603,19 @@ class Wallet (SeedSourceEnc):
 		if hasattr(self,'ss_in') and hasattr(self.ss_in.ssdata,'label'):
 			old_lbl = self.ss_in.ssdata.label
 			if opt.keep_label:
-				qmsg("Reusing label '{}' at user request".format(old_lbl.hl()))
+				qmsg(u"Reusing label '{}' at user request".format(old_lbl.hl()))
 				self.ssdata.label = old_lbl
 			elif opt.label:
-				qmsg("Using label '{}' requested on command line".format(opt.label.hl()))
+				qmsg(u"Using label '{}' requested on command line".format(opt.label.hl()))
 				lbl = self.ssdata.label = opt.label
 			else: # Prompt, using old value as default
 				lbl = self._get_label_from_user(old_lbl)
 
 			if (not opt.keep_label) and self.op == 'pwchg_new':
-				m = ("changed to '{}'".format(lbl),'unchanged')[lbl==old_lbl]
-				qmsg('Label {}'.format(m))
+				m = (u"changed to '{}'".format(lbl),'unchanged')[lbl==old_lbl]
+				qmsg(u'Label {}'.format(m))
 		elif opt.label:
-			qmsg("Using label '{}' requested on command line".format(opt.label.hl()))
+			qmsg(u"Using label '{}' requested on command line".format(opt.label.hl()))
 			self.ssdata.label = opt.label
 		else:
 			self._get_label_from_user()
@@ -640,8 +640,8 @@ class Wallet (SeedSourceEnc):
 			'{} {}'.format(make_chksum_6(slt_fmt),split_into_cols(4,slt_fmt)),
 			'{} {}'.format(make_chksum_6(es_fmt), split_into_cols(4,es_fmt))
 		)
-		chksum = make_chksum_6(' '.join(lines))
-		self.fmt_data = '{}\n'.format('\n'.join((chksum,)+lines))
+		chksum = make_chksum_6(' '.join(lines).encode('utf8'))
+		self.fmt_data = '\n'.join((chksum,)+lines) + '\n'
 
 	def _deformat(self):
 

scripts/compute-file-chksum.py

@@ -19,7 +19,7 @@ cmd_args = opts.init(opts_data)
 
 lines = get_lines_from_file(cmd_args[0])
 start = (1,0)[bool(opt.include_first_line)]
-a = make_chksum_6(' '.join(lines[start:]))
+a = make_chksum_6(' '.join(lines[start:]).encode('utf8'))
 if start == 1:
 	b = lines[0]
 	msg(("Checksum in file ({}) doesn't match computed value!".format(b),'Checksum in file OK')[a==b])

test/mmgen_pexpect.py

@@ -176,7 +176,7 @@ class MMGenPexpect(object):
 		p = "'w' for conditions and warranty info, or 'c' to continue: "
 		my_expect(self.p,p,'c')
 
-	def label(self,label='Test Label'):
+	def label(self,label=u'Test Label (UTF-8) α'):
 		p = 'Enter a wallet label, or hit ENTER for no label: '
 		my_expect(self.p,p,label+'\n')
 

test/objtest.py

@@ -97,6 +97,9 @@ def run_test(test,arg,input_data):
 
 r32,r24,r16,r17,r18 = os.urandom(32),os.urandom(24),os.urandom(16),os.urandom(17),os.urandom(18)
 tw_pfx = g.proto.base_coin.lower()+':'
+utf8_text           = u'[α-$ample UTF-8 text-ω]' * 10   # 230 chars, unicode types L,N,P,S,Z
+utf8_text_combining = u'[α-$ámple UTF-8 téxt-ω]' * 10   # L,N,P,S,Z,M
+utf8_text_control   = u'[α-$ample\nUTF-8\ntext-ω]' * 10 # L,N,P,S,Z,C
 
 from collections import OrderedDict
 tests = OrderedDict([
@@ -120,7 +123,7 @@ tests = OrderedDict([
 		'good': (('80999999.12345678',Decimal('80999999.12345678')),)
 		}),
 	('CoinAddr', {
-		'bad':  (1,'x','я'),
+		'bad':  (1,'x',u'я'),
 		'good': {
 			'btc': (('1MjjELEy6EJwk8fSNfpS8b5teFRo4X5fZr','32GiSWo9zJQgkCmjAaLRrbPwXhKry2jHhj'),
 					('n2FgXPKwuFkCXF946EnoxWJDWF2VwQ6q8J','2MspvWFjBbkv2wzQGqhxJUYPCk3Y2jMaxLN')),
@@ -130,13 +133,13 @@ tests = OrderedDict([
 	}),
 	('SeedID', {
 		'bad':  (
-			{'sid':'я'},
+			{'sid':u'я'},
 			{'sid':'F00F00'},
 			{'sid':'xF00F00x'},
 			{'sid':1},
 			{'sid':'F00BAA123'},
 			{'sid':'f00baa12'},
-			'я',r32,'abc'),
+			u'я',r32,'abc'),
 		'good': (({'sid':'F00BAA12'},'F00BAA12'),(Seed(r16),Seed(r16).sid))
 	}),
 	('MMGenID', {
@@ -144,36 +147,32 @@ tests = OrderedDict([
 		'good': (('F00BAA12:99','F00BAA12:L:99'),'F00BAA12:L:99','F00BAA12:S:99')
 	}),
 	('TwMMGenID', {
-		'bad':  ('x','я','я:я',1,'f00f00f','a:b','x:L:3','F00BAA12:0','F00BAA12:Z:99',tw_pfx,tw_pfx+'я'),
+		'bad':  ('x',u'я',u'я:я',1,'f00f00f','a:b','x:L:3','F00BAA12:0','F00BAA12:Z:99',tw_pfx,tw_pfx+u'я'),
 		'good': (('F00BAA12:99','F00BAA12:L:99'),'F00BAA12:L:99','F00BAA12:S:9999999',tw_pfx+'x')
 	}),
-	('TwComment', {
-		'bad':  ('я',"comment too long for tracking wallet",),
-		'good': ('OK comment',)
-	}),
 	('TwLabel', {
-		'bad':  ('x x','x я','я:я',1,'f00f00f','a:b','x:L:3','F00BAA12:0 x',
-				'F00BAA12:Z:99','F00BAA12:L:99 я',tw_pfx+' x',tw_pfx+'я x'),
+		'bad':  ('x x',u'x я',u'я:я',1,'f00f00f','a:b','x:L:3','F00BAA12:0 x',
+				'F00BAA12:Z:99',tw_pfx+' x',tw_pfx+u'я x'),
 		'good': (
 			('F00BAA12:99 a comment','F00BAA12:L:99 a comment'),
-			'F00BAA12:L:99 comment',
+			u'F00BAA12:L:99 comment (UTF-8) α',
 			'F00BAA12:S:9999999 comment',
 			tw_pfx+'x comment')
 	}),
 	('HexStr', {
-		'bad':  (1,[],'\0','\1','я','g','gg','FF','f00'),
+		'bad':  (1,[],'\0','\1',u'я','g','gg','FF','f00'),
 		'good': ('deadbeef','f00baa12')
 	}),
 	('MMGenTxID', {
-		'bad':  (1,[],'\0','\1','я','g','gg','FF','f00','F00F0012'),
+		'bad':  (1,[],'\0','\1',u'я','g','gg','FF','f00','F00F0012'),
 		'good': ('DEADBE','F00BAA')
 	}),
 	('CoinTxID',{
-		'bad':  (1,[],'\0','\1','я','g','gg','FF','f00','F00F0012',hexlify(r16),hexlify(r32)+'ee'),
+		'bad':  (1,[],'\0','\1',u'я','g','gg','FF','f00','F00F0012',hexlify(r16),hexlify(r32)+'ee'),
 		'good': (hexlify(r32),)
 	}),
 	('WifKey', {
-		'bad':  (1,[],'\0','\1','я','g','gg','FF','f00',hexlify(r16),'2MspvWFjBbkv2wzQGqhxJUYPCk3Y2jMaxLN'),
+		'bad':  (1,[],'\0','\1',u'я','g','gg','FF','f00',hexlify(r16),'2MspvWFjBbkv2wzQGqhxJUYPCk3Y2jMaxLN'),
 		'good': {
 			'btc': (('5KXEpVzjWreTcQoG5hX357s1969MUKNLuSfcszF6yu84kpsNZKb',
 					'KwWr9rDh8KK5TtDa3HLChEvQXNYcUXpwhRFUPc5uSNnMtqNKLFhk'),
@@ -223,16 +222,16 @@ tests = OrderedDict([
 		)
 	}),
 	('MMGenWalletLabel', {
-		'bad': ('яqwerty','This text is too long to fit in an MMGen wallet label'),
-		'good':  ('a good label',)
+		'bad': (utf8_text[:49],utf8_text_combining[:48],utf8_text_control[:48]),
+		'good':  (utf8_text[:48],)
 	}),
 	('TwComment', {
-		'bad': (u'яqwerty','This text is too long for a TW comment'),
-		'good':  ('a good comment',)
+		'bad': (utf8_text[:41],utf8_text_combining[:40],utf8_text_control[:40]),
+		'good':  (utf8_text[:40],)
 	}),
 	('MMGenTXLabel',{
-		'bad': ('This text is too long for a transaction comment. '*2,),
-		'good':  (u'UTF-8 is OK: я','a good comment',)
+		'bad': (utf8_text[:73],utf8_text_combining[:72],utf8_text_control[:72]),
+		'good':  (utf8_text[:72],)
 	}),
 	('MMGenPWIDString', { # forbidden = list(u' :/\\')
 		'bad': ('foo/','foo:','foo:\\'),
@@ -241,15 +240,17 @@ tests = OrderedDict([
 	('MMGenAddrType', {
 		'bad': ('U','z','xx',1,'dogecoin'),
 		'good':  (
-		{'s':'segwit','ret':'S'},
-		{'s':'S','ret':'S'},
 		{'s':'legacy','ret':'L'},
 		{'s':'L','ret':'L'},
 		{'s':'compressed','ret':'C'},
-		{'s':'C','ret':'C'}
+		{'s':'C','ret':'C'},
+		{'s':'segwit','ret':'S'},
+		{'s':'S','ret':'S'},
+		{'s':'bech32','ret':'B'},
+		{'s':'B','ret':'B'},
 	)}),
 	('MMGenPasswordType', {
-		'bad': ('U','z','я',1,'passw0rd'),
+		'bad': ('U','z',u'я',1,'passw0rd'),
 		'good':  (
 		{'s':'password','ret':'P'},
 		{'s':'P','ret':'P'},

test/test.py

@@ -612,6 +612,8 @@ cmd_group['main'] = OrderedDict([
 	['walletgen',       (1,'wallet generation',        [[['del_dw_run'],15]],1)],
 #	['walletchk',       (1,'wallet check',             [[['mmdat'],1]])],
 	['passchg',         (5,'password, label and hash preset change',[[['mmdat',pwfile],1]],1)],
+	['passchg_keeplabel',(5,'password, label and hash preset change (keep label)',[[['mmdat',pwfile],1]],1)],
+	['passchg_usrlabel',(5,'password, label and hash preset change (interactive label)',[[['mmdat',pwfile],1]],1)],
 	['walletchk_newpass',(5,'wallet check with new pw, label and hash preset',[[['mmdat',pwfile],5]],1)],
 	['addrgen',         (1,'address generation',       [[['mmdat',pwfile],1]],1)],
 	['addrimport',      (1,'address import',           [[['addrs'],1]],1)],
@@ -1464,7 +1466,7 @@ class MMGenTestSuite(object):
 	def brainwalletgen_ref(self,name):
 		sl_arg = '-l{}'.format(cfg['seed_len'])
 		hp_arg = '-p{}'.format(ref_wallet_hash_preset)
-		label = "test.py ref. wallet (pw '{}', seed len {})".format(ref_wallet_brainpass,cfg['seed_len'])
+		label = u"test.py ref. wallet (pw '{}', seed len {}) α".format(ref_wallet_brainpass,cfg['seed_len'])
 		bf = 'ref.mmbrain'
 		args = ['-d',cfg['tmpdir'],hp_arg,sl_arg,'-ib','-L',label]
 		write_to_tmpfile(cfg,bf,ref_wallet_brainpass)
@@ -1479,20 +1481,24 @@ class MMGenTestSuite(object):
 
 	def refwalletgen(self,name): self.brainwalletgen_ref(name)
 
-	def passchg(self,name,wf,pf):
+	def passchg(self,name,wf,pf,label_action='cmdline'):
 		silence()
 		write_to_tmpfile(cfg,pwfile,get_data_from_file(pf))
 		end_silence()
-		t = MMGenExpect(name,'mmgen-passchg', [usr_rand_arg] +
-				['-d',cfg['tmpdir'],'-p','2','-L','Changed label'] + ([],[wf])[bool(wf)])
+		add_args = {'cmdline': ['-d',cfg['tmpdir'],'-L',u'Changed label (UTF-8) α'],
+					'keep':    ['-d',trash_dir,'--keep-label'],
+					'user':    ['-d',trash_dir]
+					}[label_action]
+		t = MMGenExpect(name,'mmgen-passchg', add_args + [usr_rand_arg, '-p2'] + ([],[wf])[bool(wf)])
 		t.license()
 		t.passphrase('MMGen wallet',cfgs['1']['wpasswd'],pwtype='old')
 		t.expect_getend('Hash preset changed to ')
 		t.passphrase('MMGen wallet',cfg['wpasswd'],pwtype='new') # reuse passphrase?
 		t.expect('Repeat passphrase: ',cfg['wpasswd']+'\n')
 		t.usr_rand(usr_rand_chars)
-#		t.expect('Enter a wallet label.*: ','Changed Label\n',regex=True)
-		t.expect_getend('Label changed to ')
+		if label_action == 'user':
+			t.expect('Enter a wallet label.*: ',u'Interactive Label (UTF-8) α\n',regex=True)
+		t.expect_getend(('Label changed to ','Reusing label ')[label_action=='keep'])
 #		t.expect_getend('Key ID changed: ')
 		if not wf:
 			t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
@@ -1505,6 +1511,12 @@ class MMGenTestSuite(object):
 			t.written_to_file('MMGen wallet')
 		t.ok()
 
+	def passchg_keeplabel(self,name,wf,pf):
+		return self.passchg(name,wf,pf,label_action='keep')
+
+	def passchg_usrlabel(self,name,wf,pf):
+		return self.passchg(name,wf,pf,label_action='user')
+
 	def passchg_dfl_wallet(self,name,pf):
 		return self.passchg(name=name,wf=None,pf=pf)