[qt] full support for unicode with utf-8 encoding (closes #142378)

We were decoding strings for changeset description and file data only. But we also have to decode all all meta-information from Hg repo (a.k.a. usernames, bookmarks, tags, branches, filenames).

String are decoded just at rendering times. Data are kept binary string for other operation (eg: comparing filenames).

We use the hgviewlib.util.tounicode function everywhere now.

We also missed using utf-8 for Scintilla.

Fix initiated by:Юрий Мандрик.

Note

We try to decode using utf8, iso-8859-15 and cp1252 (in this order) using the first that successfully decode the string. If all fail we use utf8 with replace strategy.

test case used:

hg init cyrillic
cd cyrillic
hg branch 'ЖЗИЙФ'
echo 'ЖЗИЙФ' > ЖЗИЙФ
hg add ЖЗИЙФ
hg ci -m 'ЖЗИЙФ' -u 'ЖЗИЙФ'
hg bookmark 'ЖЗИЙФЖЗИЙФ'
hg tag 'ЖЗИЙФ'
hg mv 'ЖЗИЙФ' 'ЖЗИЙФЖЗИЙФ'
hg ci -m 'move'
echo 'hello' > ЖЗИЙФЖЗИЙФ
hg ci -m 'back to ascii'
hg rm ЖЗИЙФЖЗИЙФ
hg ci -m 'remove'
authorAlain Leufroy <alain.leufroy@logilab.fr>
changeseta802dac29c7e
branchstable
phasepublic
hiddenno
parent revision#cf93defad8a8 [launcher] load hgviewlib manually if standalone (closes #117624)
child revision#8aa4cdf51939 [setup] use ``gmake`` instead of ``make`` on freebsd
files modified by this revision
hgviewlib/hggraph.py
hgviewlib/qt4/hgfiledialog.py
hgviewlib/qt4/hgfileview.py
hgviewlib/qt4/hgmanifestdialog.py
hgviewlib/qt4/hgrepomodel.py
hgviewlib/qt4/hgrepoview.py
hgviewlib/util.py
# HG changeset patch
# User Alain Leufroy <alain.leufroy@logilab.fr>
# Date 1369737267 -7200
# Tue May 28 12:34:27 2013 +0200
# Branch stable
# Node ID a802dac29c7e1b8d8b1f72d1c6739921c26bbf06
# Parent cf93defad8a826e80ae683c84368cc3869731552
[qt] full support for unicode with utf-8 encoding (closes #142378)

We were decoding strings for changeset description and file data only. But we
also have to decode all all meta-information from Hg repo (a.k.a. usernames,
bookmarks, tags, branches, filenames).

String are decoded just at rendering times. Data are kept binary string for
other operation (eg: comparing filenames).

We use the ``hgviewlib.util.tounicode`` function everywhere now.

We also missed using utf-8 for Scintilla.

:Fix initiated by: Юрий Мандрик.

.. note:: We try to decode using utf8, iso-8859-15 and cp1252 (in this order)
using the first that successfully decode the string. If all fail we
use utf8 with ``replace`` strategy.

test case used::

hg init cyrillic
cd cyrillic
hg branch 'ЖЗИЙФ'
echo 'ЖЗИЙФ' > ЖЗИЙФ
hg add ЖЗИЙФ
hg ci -m 'ЖЗИЙФ' -u 'ЖЗИЙФ'
hg bookmark 'ЖЗИЙФЖЗИЙФ'
hg tag 'ЖЗИЙФ'
hg mv 'ЖЗИЙФ' 'ЖЗИЙФЖЗИЙФ'
hg ci -m 'move'
echo 'hello' > ЖЗИЙФЖЗИЙФ
hg ci -m 'back to ascii'
hg rm ЖЗИЙФЖЗИЙФ
hg ci -m 'remove'

diff --git a/hgviewlib/hggraph.py b/hgviewlib/hggraph.py
@@ -20,11 +20,11 @@
1 
2  import os
3  import re
4  from cStringIO import StringIO
5  import difflib
6 -from itertools import chain, count
7 +from itertools import chain, count, imap
8  from time import strftime, localtime
9  from functools import partial
10 
11  from mercurial.node import nullrev
12  from mercurial import patch, util, match, error, hg
@@ -38,10 +38,11 @@
13  DATE_FMT = '%F %R'
14  # match the end of the diff header, assuming that the following line
15  # looks like "@@ -25,6 +23,5 @@"
16  DIFFHEADERMATCHER = re.compile('^@@.+@@$', re.MULTILINE)
17 
18 +
19  def diff(repo, ctx1, ctx2=None, files=None):
20      """
21      Compute the diff of ``files`` between the 2 contexts ``ctx1`` and ``ctx2``.
22 
23      :Note: context may be a changectx or a filectx.
@@ -69,17 +70,11 @@
24                     opts=diffopts)
25          diffdata = out.getvalue()
26      except:
27          diffdata = '\n'.join(patch.diff(repo, ctx2.node(), ctx1.node(),
28                                          match=matchfn, opts=diffopts))
29 -    # XXX how to deal diff encodings?
30 -    try:
31 -        diffdata = unicode(diffdata, "utf-8")
32 -    except UnicodeError:
33 -        # XXX use a default encoding from config?
34 -        diffdata = unicode(diffdata, "iso-8859-15", 'ignore')
35 -    return diffdata
36 +    return tounicode(diffdata)
37 
38 
39  def __get_parents(repo, rev, branch=None):
40      """
41      Return non-null parents of `rev`. If branch is given, only return
@@ -100,21 +95,21 @@
42      if ctx.rev() is not None:
43          msg = tounicode(ctx.description())
44          if msg:
45              msg = msg.splitlines()[0]
46      else:
47 -        msg = "WORKING DIRECTORY (locally modified)"
48 +        msg = u"WORKING DIRECTORY (locally modified)"
49      return msg
50 
51  def gettags(model, ctx, gnode):
52      if ctx.rev() is None:
53 -        return ""
54 +        return u""
55      mqtags = ['qbase', 'qtip', 'qparent']
56      tags = ctx.tags()
57      if model.hide_mq_tags:
58          tags = [t for t in tags if t not in mqtags]
59 -    return ",".join(tags)
60 +    return u",".join(imap(tounicode, tags))
61 
62  def getdate(model, ctx, gnode):
63      if not ctx.date():
64          return ""
65      return strftime(DATE_FMT, localtime(int(ctx.date()[0])))
@@ -500,14 +495,14 @@
66              fctx = ctx.filectx(filename)
67              filesize = fctx.size() # compute size here to lookup data securely
68          except (LookupError, OSError):
69              fctx = None # may happen for renamed/removed files or mq patch ?
70          if isbfile(filename):
71 -            data = "[bfile]\n"
72 +            data = u"[bfile]\n"
73              if fctx:
74                  data = fctx.data()
75 -                data += "footprint: %s\n" % data
76 +                data += u"footprint: %s\n" % tounicode(data)
77              return "+", data
78          if flag not in ('-', '?'):
79              if fctx is None:# or fctx.node() is None:
80                  return '', None
81              if self.maxfilesize >= 0 and filesize > self.maxfilesize:
@@ -516,18 +511,18 @@
82                      sym = ('', 'K', 'M', 'G', 'T', 'E')[div] # more, really ???
83                      val = int(filesize / (2 ** (div * 10)))
84                  except AttributeError: # py<2.7
85                      val = filesize
86                      sym = ''
87 -                data = "File too big ! (~%i%so)" % (val, sym)
88 +                data = u"File too big ! (~%i%so)" % (val, sym)
89                  return flag, data
90              if flag == "+" or mode == 'file':
91                  flag = '+'
92                  # return the whole file
93                  data = fctx.data()
94                  if util.binary(data):
95 -                    data = "binary file"
96 +                    data = u"binary file"
97                  else: # tries to convert to unicode
98                      data = tounicode(data)
99              elif flag == "=" or isinstance(mode, int):
100                  flag = "="
101                  if isinstance(mode, int):
@@ -540,11 +535,11 @@
102                      datastart = match.start()
103                  else:
104                      datastart = 0
105                  data = data[datastart:]
106              elif flag == '':
107 -                data = ''
108 +                data = u''
109              else: # file renamed
110                  oldname, node = flag
111                  newdata = fctx.data().splitlines()
112                  olddata = self.repo.filectx(oldname, fileid=node)
113                  olddata = olddata.data().splitlines()
diff --git a/hgviewlib/qt4/hgfiledialog.py b/hgviewlib/qt4/hgfiledialog.py
@@ -245,10 +245,11 @@
114          lay.setSpacing(0)
115          lay.setContentsMargins(0, 0, 0, 0)
116          for side, idx  in (('left', 0), ('right', 3)):
117              sci = Qsci.QsciScintilla(self.frame)
118              sci.setFont(self._font)
119 +            sci.setUtf8(True)
120              sci.verticalScrollBar().setFocusPolicy(Qt.StrongFocus)
121              sci.setFocusProxy(sci.verticalScrollBar())
122              sci.verticalScrollBar().installEventFilter(self)
123              sci.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
124              sci.setFrameShape(QtGui.QFrame.NoFrame)
@@ -280,10 +281,11 @@
125              blk = BlockList(self.frame)
126              blk.linkScrollBar(sci.verticalScrollBar())
127              self.diffblock.linkScrollBar(sci.verticalScrollBar(), side)
128              lay.insertWidget(idx, blk)
129              self.block[side] = blk
130 +
131          lay.insertWidget(2, self.diffblock)
132 
133          for side in sides:
134              table = getattr(self, 'tableView_revisions_%s' % side)
135              table.setTabKeyNavigation(False)
@@ -354,11 +356,11 @@
136              side = 'right'
137          else:
138              side = 'left'
139          path = self.filerevmodel.graph.nodesdict[rev].extra[0]
140          fc = self.repo.changectx(rev).filectx(path)
141 -        self.filedata[side] = fc.data().splitlines()
142 +        self.filedata[side] = tounicode(fc.data()).splitlines()
143          self.update_diff(keeppos=otherside[side])
144 
145      def goto(self, rev):
146          index = self.filerevmodel.indexFromRev(rev)
147          if index is not None:
diff --git a/hgviewlib/qt4/hgfileview.py b/hgviewlib/qt4/hgfileview.py
@@ -30,11 +30,11 @@
148  connect = QtCore.QObject.connect
149  SIGNAL = QtCore.SIGNAL
150  nullvariant = QtCore.QVariant()
151 
152  from hgviewlib.decorators import timeit
153 -from hgviewlib.util import exec_flag_changed, isbfile, bfilepath
154 +from hgviewlib.util import exec_flag_changed, isbfile, bfilepath, tounicode
155  from hgviewlib.config import HgConfig
156 
157  from hgviewlib.qt4 import icon as geticon
158  from hgviewlib.qt4.hgfiledialog import FileViewer, FileDiffViewer
159  from hgviewlib.qt4.hgmanifestdialog import ManifestViewer
@@ -93,10 +93,11 @@
160 
161  class HgQsci(qsci):
162 
163      def __init__(self, *args, **kwargs):
164          super(HgQsci, self).__init__(*args, **kwargs)
165 +        super(HgQsci, self).setUtf8(True)
166          self.createActions()
167 
168      def _action_defs(self):
169          return [
170              ("diffmode", self.tr("Diff mode"), 'diffmode' ,
@@ -349,32 +350,32 @@
171          if flag == "+":
172              nlines = data.count('\n')
173              self.sci.setMarginWidth(1, str(nlines)+'0')
174          self.sci.setLexer(lexer)
175          self._cur_lexer = lexer
176 -        if data not in ('file too big', 'binary file'):
177 +        if data not in (u'file too big', u'binary file'):
178              self.filedata = data
179          else:
180              self.filedata = None
181 
182          flag = exec_flag_changed(filectx)
183          if flag:
184 -            self.execflaglabel.setText("<b>exec mode has been <font color='red'>%s</font></b>" % flag)
185 +            self.execflaglabel.setText(u"<b>exec mode has been <font color='red'>%s</font></b>" % flag)
186              self.execflaglabel.show()
187          else:
188              self.execflaglabel.hide()
189 
190 -        labeltxt = ''
191 +        labeltxt = u''
192          if isbfile(self._realfilename):
193 -            labeltxt += '[bfile tracked] '
194 -        labeltxt += "<b>%s</b>" % self._filename
195 +            labeltxt += u'[bfile tracked] '
196 +        labeltxt += u"<b>%s</b>" % tounicode(self._filename)
197 
198          if self._p_rev is not None:
199 -            labeltxt += ' (diff from rev %s)' % self._p_rev
200 +            labeltxt += u' (diff from rev %s)' % self._p_rev
201          renamed = filectx.renamed()
202          if renamed:
203 -            labeltxt += ' <i>(renamed from %s)</i>' % bfilepath(renamed[0])
204 +            labeltxt += u' <i>(renamed from %s)</i>' % tounicode(bfilepath(renamed[0]))
205          self.filenamelabel.setText(labeltxt)
206 
207          self.sci.setText(data)
208          if self._find_text:
209              self.highlightSearchString(self._find_text)
diff --git a/hgviewlib/qt4/hgmanifestdialog.py b/hgviewlib/qt4/hgmanifestdialog.py
@@ -83,10 +83,11 @@
210          lay.addWidget(sci)
211          sci.setMarginLineNumbers(1, True)
212          sci.setMarginWidth(1, '000')
213          sci.setReadOnly(True)
214          sci.setFont(self._font)
215 +        sci.setUtf8(True)
216 
217          sci.SendScintilla(sci.SCI_SETSELEOLFILLED, True)
218          self.textView = sci
219 
220      def fileSelected(self, index, *args):
diff --git a/hgviewlib/qt4/hgrepomodel.py b/hgviewlib/qt4/hgrepomodel.py
@@ -63,21 +63,21 @@
221      return QtCore.QDateTime.fromTime_t(int(date)).toString(QtCore.Qt.LocaleDate)
222 
223 
224  # XXX maybe it's time to make these methods of the model...
225  # in following lambdas, ctx is a hg changectx
226 -_columnmap = {'ID': lambda model, ctx, gnode: ctx.rev() is not None and str(ctx.rev()) or "",
227 +_columnmap = {'ID': lambda model, ctx, gnode: ctx.rev() is not None and tounicode(ctx.rev()) or u"",
228                'Log': getlog,
229                'Author': lambda model, ctx, gnode: tounicode(ctx.user()),
230                'Date': lambda model, ctx, gnode: cvrt_date(ctx.date()),
231                'Tags': gettags,
232 -              'Branch': lambda model, ctx, gnode: ctx.branch(),
233 -              'Filename': lambda model, ctx, gnode: gnode.extra[0],
234 -              'Phase': lambda model, ctx, gnode: ctx.phasestr(),
235 +              'Branch': lambda model, ctx, gnode: tounicode(ctx.branch()),
236 +              'Filename': lambda model, ctx, gnode: tounicode(gnode.extra[0]),
237 +              'Phase': lambda model, ctx, gnode: tounicode(ctx.phasestr()),
238                }
239 
240 -_tooltips = {'ID': lambda model, ctx, gnode: ctx.rev() is not None and ctx.hex() or "Working Directory",
241 +_tooltips = {'ID': lambda model, ctx, gnode: ctx.rev() is not None and tounicode(ctx.hex()) or u"Working Directory",
242               }
243 
244  def auth_width(model, repo):
245      auths = model._aliases.values()
246      if not auths:
@@ -209,32 +209,32 @@
247              if column == 'Author': #author
248                  user = _columnmap[column](self, ctx, gnode) if ctx.node() else u'' 
249                  return QtCore.QVariant(self.user_name(user))
250              elif column == 'Log':
251                  msg = _columnmap[column](self, ctx, gnode)
252 -                bookmarks = ctx.bookmarks()
253 +                bookmarks = [tounicode(bookmark) for bookmark in ctx.bookmarks()]
254                  if bookmarks:
255 -                    msg = '<%s> ~ %s' % (','.join(bookmarks), msg)
256 +                    msg = '<%s> ~ %s' % (u','.join(bookmarks), msg)
257                  return QtCore.QVariant(msg)
258              return QtCore.QVariant(_columnmap[column](self, ctx, gnode))
259          elif role == QtCore.Qt.ToolTipRole:
260 -            msg = "<b>Branch:</b> %s<br>\n" % ctx.branch()
261 -            msg += "<b>Phase:</b> %s<br>\n" % ctx.phasestr()
262 +            msg = u"<b>Branch:</b> %s<br>\n" % _columnmap['Branch'](self, ctx, gnode)
263 +            msg += u"<b>Phase:</b> %s<br>\n" % _columnmap['Phase'](self, ctx, gnode)
264              if gnode.rev in self.wd_revs:
265 -                msg += " <i>Working Directory position"
266 -                states = 'modified added removed deleted'.split()
267 +                msg += u" <i>Working Directory position"
268 +                states = u'modified added removed deleted'.split()
269                  status = self.wd_status[self.wd_revs.index(gnode.rev)]
270                  status = [state for st, state in zip(status, states) if st]
271                  if status:
272                      msg += ' (%s)' % (', '.join(status))
273 -                msg += "</i><br>\n"
274 +                msg += u"</i><br>\n"
275              msg += _tooltips.get(column, _columnmap[column])(self, ctx, gnode)
276              return QtCore.QVariant(msg)
277          elif role == QtCore.Qt.ForegroundRole:
278              color = None
279              if column == 'Author': #author
280 -                user = ctx.user() if ctx.node() else ''
281 +                user = tounicode(ctx.user()) if ctx.node() else u''
282                  color = QtGui.QColor(self.user_color(user))
283                  if ctx.obsolete():
284                      color = color.lighter()
285              elif column == 'Branch': #branch
286                  color = QtGui.QColor(self.namedbranch_color(ctx.branch()))
@@ -543,10 +543,11 @@
287              for f in [x for x in lst if self._filterFile(x, ctxfiles)]:
288                  desc = f
289                  bfile = isbfile(f)
290                  if bfile:
291                      desc = desc.replace('.hgbfiles'+os.sep, '')
292 +                desc = tounicode(desc)
293                  _files.append({'path': f, 'flag': flag, 'desc': desc, 'bfile': bfile,
294                                 'parent': parent, 'fromside': fromside,
295                                 'infiles': f in ctxfiles})
296                  # renamed/copied files are handled by background
297                  # filling process since it can be a bit long
@@ -604,11 +605,11 @@
298          for row, desc in files:
299              filename = desc['path']
300              if desc['flag'] == '=' and self._displaydiff:
301                  try:
302                      diff = revdiff(self.repo, self.current_ctx, None, files=[filename])
303 -                    tot = self.current_ctx.filectx(filename).data().count('\n')
304 +                    tot = tounicode(self.current_ctx.filectx(filename).data()).count('\n')
305                      add = len(replus.findall(diff))
306                      rem = len(reminus.findall(diff))
307                  except (LookupError, TypeError): # unknown revision and mq support
308                      tot, add, rem = 0, 0, 0
309 
@@ -623,17 +624,17 @@
310                      removed = self.repo.status(desc['parent'].node(),
311                                                 self.current_ctx.node())[2]
312                      oldname, node = m
313                      if oldname in removed:
314                          # removed.remove(oldname) XXX
315 -                        desc['renamedfrom'] = (oldname, node)
316 +                        desc['renamedfrom'] = (tounicode(oldname), node)
317                          desc['flag'] = '='
318 -                        desc['desc'] += '\n (was %s)' % oldname
319 +                        desc['desc'] += u'\n (was %s)' % tounicode(oldname)
320                      else:
321 -                        desc['copiedfrom'] = (oldname, node)
322 +                        desc['copiedfrom'] = (tounicode(oldname), node)
323                          desc['flag'] = '='
324 -                        desc['desc'] += '\n (copy of %s)' % oldname
325 +                        desc['desc'] += u'\n (copy of %s)' % tounicode(oldname)
326                      yield row, 0
327              yield None
328 
329      def data(self, index, role):
330          if not index.isValid() or index.row()>len(self) or not self.current_ctx:
@@ -680,11 +681,11 @@
331                      msg += "&nbsp;<b>removed lines:&nbsp;</b> %s" % rem
332                      return QtCore.QVariant(msg)
333 
334          elif column == 0:
335              if role in (QtCore.Qt.DisplayRole, QtCore.Qt.ToolTipRole):
336 -                return QtCore.QVariant(current_file_desc['desc'])
337 +                return QtCore.QVariant(tounicode(current_file_desc['desc']))
338              elif role == QtCore.Qt.DecorationRole:
339                  if self._fulllist and ismerge(self.current_ctx):
340                      icn = None
341                      if current_file_desc['infiles']:
342                          icn = geticon('leftright')
@@ -781,11 +782,11 @@
343 
344          if role != QtCore.Qt.DisplayRole:
345              return QtCore.QVariant()
346 
347          item = index.internalPointer()
348 -        return QtCore.QVariant(item.data(index.column()))
349 +        return QtCore.QVariant(tounicode(item.data(index.column())))
350 
351      def flags(self, index):
352          if not index.isValid():
353              return QtCore.Qt.ItemIsEnabled
354          return QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable
diff --git a/hgviewlib/qt4/hgrepoview.py b/hgviewlib/qt4/hgrepoview.py
@@ -152,11 +152,11 @@
355              return
356          painter = QtGui.QPainter(self)
357          icn.paint(painter, self.width() - 18, (self.height() - 18) / 2, 16, 16)
358 
359      def on_text_edited(self):
360 -        current_text = unicode(self.text()).strip()
361 +        current_text = tounicode(self.text()).strip()
362          if  current_text == self.previous_text:
363              return
364          self.previous_text = current_text
365          self.emit(SIGNAL('text_edited_no_blank'), current_text)
366 
@@ -195,11 +195,11 @@
367      def createContent(self):
368          QuickBar.createContent(self)
369          # completer
370          self.compl_model = CompleterModel(['tip'])
371          self.completer = QtGui.QCompleter(self.compl_model, self)
372 -        cb = lambda text: self.search(unicode(text))
373 +        cb = lambda text: self.search(tounicode(text))
374          self.completer.activated[str].connect(cb)
375          # entry
376          self.entry = QueryLineEdit(self)
377          self.entry.setCompleter(self.completer)
378          self.entry.setStatusTip("Enter a 'revset' to query a set of revisions")
@@ -308,11 +308,11 @@
379          if rows and revexp:
380              self.compl_model.add_to_string_list(revexp)
381 
382      def on_failed(self, err):
383          self.entry.status = 'failed'
384 -        self.show_message(unicode(err))
385 +        self.show_message(tounicode(err))
386          self._actions['next'].setEnabled(False)
387          self._actions['prev'].setEnabled(False)
388 
389 
390  class HgRepoView(QtGui.QTableView):
@@ -535,11 +535,11 @@
391              if model._columns[c] in model._stretchs:
392                  tot_stretch += model._stretchs[model._columns[c]]
393                  continue
394              w = model.maxWidthValueForColumn(c)
395              if w is not None:
396 -                w = fontm.width(unicode(w) + 'w')
397 +                w = fontm.width(tounicode(w) + 'w')
398                  self.setColumnWidth(c, w)
399              else:
400                  w = self.sizeHintForColumn(c)
401                  self.setColumnWidth(c, w)
402              col1_width -= self.columnWidth(c)
@@ -696,11 +696,11 @@
403      def visual_bell(self):
404          self.hide()
405          QtCore.QTimer.singleShot(0.01, self.show)
406 
407 
408 -TROUBLE_EXPLANATIONS = defaultdict(lambda:'unknown trouble')
409 +TROUBLE_EXPLANATIONS = defaultdict(lambda:u'unknown trouble')
410  TROUBLE_EXPLANATIONS['unstable']  = "Based on obsolete ancestor"
411  TROUBLE_EXPLANATIONS['bumped']    = "Hopeless successors of a public changeset"
412  TROUBLE_EXPLANATIONS['divergent'] = "Another changeset are also a successors "\
413                                      "of one of your precursor"
414  # temporary compat with older evolve version
@@ -784,11 +784,11 @@
415          self.setTextCursor(cursor)
416          self.setExtraSelections([])
417 
418      def searchString(self, text):
419          self.selectNone()
420 -        if text in unicode(self.toPlainText()):
421 +        if text in tounicode(self.toPlainText()):
422              clist = []
423              while self.find(text):
424                  eselect = self.ExtraSelection()
425                  eselect.cursor = self.textCursor()
426                  eselect.format.setBackground(QtGui.QColor('#ffffbb'))
@@ -806,35 +806,35 @@
427 
428      def refreshDisplay(self):
429          ctx = self.ctx
430          rev = ctx.rev()
431          cfg = HgConfig(ctx._repo.ui)
432 -        buf = "<table width=100%>\n"
433 +        buf = u"<table width=100%>\n"
434          if self.mqpatch:
435 -            buf += '<tr bgcolor=%s>' % cfg.getMQFGColor()
436 -            buf += '<td colspan=4 width=100%><b>Patch queue:</b>&nbsp;'
437 +            buf += u'<tr bgcolor=%s>' % cfg.getMQFGColor()
438 +            buf += u'<td colspan=4 width=100%><b>Patch queue:</b>&nbsp;'
439              for p in self.mqseries:
440                  if p in self.mqunapplied:
441 -                    p = "<i>%s</i>" % p
442 +                    p = u"<i>%s</i>" % tounicode(p)
443                  elif p == self.mqpatch:
444 -                    p = "<b>%s</b>" % p
445 -                buf += '&nbsp;%s&nbsp;' % (p)
446 -            buf += '</td></tr>\n'
447 +                    p = u"<b>%s</b>" % touniode(p)
448 +                buf += u'&nbsp;%s&nbsp;' % tounicode(p)
449 +            buf += u'</td></tr>\n'
450 
451 -        buf += '<tr>'
452 +        buf += u'<tr>'
453          if rev is None:
454 -            buf += "<td><b>Working Directory</b></td>\n"
455 +            buf += u"<td><b>Working Directory</b></td>\n"
456          else:
457 -            buf += '<td title="Revision"><b>'\
458 -                   '<span class="rev_number">%s</span>:'\
459 -                   '<span class="rev_hash">%s</span>'\
460 -                   '</b></td>\n' % (ctx.rev(), short_hex(ctx.node()))
461 +            buf += u'<td title="Revision"><b>'\
462 +                   u'<span class="rev_number">%s</span>:'\
463 +                   u'<span class="rev_hash">%s</span>'\
464 +                   u'</b></td>\n' % (ctx.rev(), short_hex(ctx.node()))
465 
466          user = tounicode(ctx.user()) if ctx.node() else u''
467          buf += '<td title="Author">%s</td>\n' % user
468 -        buf += '<td title="Branch name">%s</td>\n' % ctx.branch()
469 -        buf += '<td title="Phase name">%s</td>\n' % ctx.phasestr()
470 +        buf += '<td title="Branch name">%s</td>\n' % tounicode(ctx.branch())
471 +        buf += '<td title="Phase name">%s</td>\n' % tounicode(ctx.phasestr())
472          buf += '</tr>'
473          buf += "</table>\n"
474 
475          buf += "<table width=100%>\n"
476          parents = [p for p in ctx.parents() if p]
@@ -853,35 +853,35 @@
477              buf += self._html_ctx_info(prec, 'Precursor',
478                  'Previous version obsolete by this changeset')
479          for suc in first_known_successors(ctx, self.excluded):
480              buf += self._html_ctx_info(suc, 'Successors',
481                  'Updated version that make this changeset obsolete')
482 -        bookmarks = ', '.join(ctx.bookmarks())
483 +        bookmarks = ', '.join(tounicode(bookmark) for bookmark in ctx.bookmarks())
484          if bookmarks:
485              buf += '<tr><td width=50 class="label"><b>Bookmarks:</b></td>'\
486                     '<td colspan=5>&nbsp;'\
487                     '<span class="short_desc">%s</span></td></tr>'\
488                     '\n' % bookmarks
489          troubles = ctx.troubles()
490          if troubles:
491 -            span = '<span title="%s"  style="color: red;">%s</span>'
492 -            content = ', '.join([span % (TROUBLE_EXPLANATIONS[troub], troub)
493 -                                for troub in troubles])
494 +            span = u'<span title="%s"  style="color: red;">%s</span>'
495 +            content = u', '.join([span % (TROUBLE_EXPLANATIONS[troub], troub)
496 +                                 for troub in troubles])
497              buf += '<tr><td width=50 class="label"><b>Troubles:</b></td>'\
498                     '<td colspan=5>&nbsp;'\
499                     '<span class="short_desc" >%s</span></td></tr>'\
500                     '\n' % ''.join(content)
501 -        buf += "</table>\n"
502 +        buf += u"</table>\n"
503          desc = tounicode(ctx.description())
504          if self.rst_action is not None  and self.rst_action.isChecked():
505              replace = cfg.getFancyReplace()
506              if replace:
507                  desc = replace(desc)
508              desc = rst2html(desc)
509          else:
510              desc = raw2html(desc)
511 -        buf += '<div class="diff_desc">%s</div>\n' % desc
512 +        buf += u'<div class="diff_desc">%s</div>\n' % desc
513          self.setHtml(buf)
514 
515      def contextMenuEvent(self, event):
516          _context_menu = self.createStandardContextMenu()
517          _context_menu.addAction(self.rst_action)
diff --git a/hgviewlib/util.py b/hgviewlib/util.py
@@ -13,20 +13,25 @@
518  from mercurial import ui, hg
519 
520  from hgviewlib.hgpatches.scmutil import match
521  from hgviewlib.hgpatches import precursorsmarkers, successorsmarkers
522 
523 -def tounicode(string):
524 +def tounicode(text):
525 +    """
526 +    Tries to convert ``text`` into a unicode string.
527 +    If ``text`` is already a unicode object return it.
528      """
529 -    Tries to convert s into a unicode string
530 -    """
531 +    if isinstance(text, unicode):
532 +        return text
533 +    else:
534 +        text = str(text)
535      for encoding in ('utf-8', 'iso-8859-15', 'cp1252'):
536          try:
537 -            return unicode(string, encoding)
538 -        except UnicodeDecodeError:
539 +            return text.decode(encoding)
540 +        except (UnicodeError, UnicodeDecodeError):
541              pass
542 -    return unicode(string, 'utf-8', 'replace')
543 +    return text.decode('utf-8', 'replace')
544 
545  def isexec(filectx):
546      """
547      Return True is the file at filectx revision is executable
548      """