Package translate :: Package storage :: Module cpo

[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.cpo

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2002-2007 Zuza Software Foundation
      5  #
      6  # This file is part of translate.
      7  #
      8  # translate is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # translate is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with translate; if not, write to the Free Software
     20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     21
     22  """Classes that hold units of .po files (pounit) or entire files (pofile).
     23
     24  Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
     25  many other projects.
     26
     27  This uses libgettextpo from the gettext package. Any version before 0.17 will
     28  at least cause some subtle bugs or may not work at all. Developers might want
     29  to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
     30  package for the public API of the library.
     31  """
     32
     33  from ctypes import c_size_t, c_int, c_uint, c_char_p, c_long, CFUNCTYPE, POINTER
     34  from ctypes import Structure, cdll
     35  import ctypes.util
     36  import os
     37  import re
     38  import sys
     39  import tempfile
     40
     41  from translate.lang import data
     42  from translate.misc.multistring import multistring
     43  from translate.storage import base, pocommon
     44  from translate.storage import pypo
     45  from translate.storage.pocommon import encodingToUse
     46
     47  lsep = " "
     48  """Seperator for #: entries"""
     49
     50  STRING = c_char_p
     51
     52
     53  # Structures
     54 -class po_message(Structure):
     55      _fields_ = []
     56
     57  # Function prototypes
     58  xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
     59  xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
     60
     61
     62  # Structures (error handler)
     63 -class po_xerror_handler(Structure):
     64      _fields_ = [('xerror', xerror_prototype),
     65                  ('xerror2', xerror2_prototype)]
     66
     67
     68 -class po_error_handler(Structure):
     69      _fields_ = [
     70      ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
     71      ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
     72      ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
     73      ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
     74  ]
     75
     76
     77  # Callback functions for po_xerror_handler
     78 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
     79      print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
     80      if severity >= 1:
     81          raise ValueError(message_text)
     82
     83
     84 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
     85      print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
     86      if severity >= 1:
     87          raise ValueError(message_text1)
     88
     89
     90  # Load libgettextpo
     91  gpo = None
     92  # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on
     93  # windows. Therefore we test both.
     94  names = ['gettextpo', 'libgettextpo']
     95  for name in names:
     96      lib_location = ctypes.util.find_library(name)
     97      if lib_location:
     98          gpo = cdll.LoadLibrary(lib_location)
     99          if gpo:
    100              break
    101  else:
    102      # Now we are getting desperate, so let's guess a unix type DLL that might
    103      # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD
    104      try:
    105          gpo = cdll.LoadLibrary('libgettextpo.so')
    106      except OSError, e:
    107          raise ImportError("gettext PO library not found")
    108
    109  # Setup return and paramater types
    110  # File access
    111  gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
    112  gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
    113  gpo.po_file_write_v2.retype = c_int
    114
    115  # Header
    116  gpo.po_file_domain_header.restype = STRING
    117  gpo.po_header_field.restype = STRING
    118  gpo.po_header_field.argtypes = [STRING, STRING]
    119
    120  # Locations (filepos)
    121  gpo.po_filepos_file.restype = STRING
    122  gpo.po_message_filepos.restype = c_int
    123  gpo.po_message_filepos.argtypes = [c_int, c_int]
    124  gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_size_t]
    125
    126  # Message (get methods)
    127  gpo.po_message_comments.restype = STRING
    128  gpo.po_message_extracted_comments.restype = STRING
    129  gpo.po_message_prev_msgctxt.restype = STRING
    130  gpo.po_message_prev_msgid.restype = STRING
    131  gpo.po_message_prev_msgid_plural.restype = STRING
    132  gpo.po_message_is_format.restype = c_int
    133  gpo.po_message_is_format.argtypes = [c_int, STRING]
    134  gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
    135  gpo.po_message_msgctxt.restype = STRING
    136  gpo.po_message_msgid.restype = STRING
    137  gpo.po_message_msgid_plural.restype = STRING
    138  gpo.po_message_msgstr.restype = STRING
    139  gpo.po_message_msgstr_plural.restype = STRING
    140
    141  # Message (set methods)
    142  gpo.po_message_set_comments.argtypes = [c_int, STRING]
    143  gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
    144  gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
    145  gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
    146
    147  # Setup the po_xerror_handler
    148  xerror_handler = po_xerror_handler()
    149  xerror_handler.xerror = xerror_prototype(xerror_cb)
    150  xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
    151
    152
    153 -def escapeforpo(text):
    154      return pypo.escapeforpo(text)
    155
    156
    157 -def quoteforpo(text):
    158      return pypo.quoteforpo(text)
    159
    160
    161 -def unquotefrompo(postr):
    162      return pypo.unquotefrompo(postr)
    163
    164
    165 -def get_libgettextpo_version():
    166      """Returns the libgettextpo version
    167
    168         @rtype: three-value tuple
    169         @return: libgettextpo version in the following format::
    170             (major version, minor version, subminor version)
    171      """
    172      libversion = c_long.in_dll(gpo, 'libgettextpo_version')
    173      major = libversion.value >> 16
    174      minor = libversion.value >> 8
    175      subminor = libversion.value - (major << 16) - (minor << 8)
    176      return major, minor, subminor
    177
    178
    179 -class pounit(pocommon.pounit):
    180
    181 -    def __init__(self, source=None, encoding='utf-8', gpo_message=None):
    182          self._rich_source = None
    183          self._rich_target = None
    184          self._encoding = encoding or 'utf-8'
    185          if not gpo_message:
    186              self._gpo_message = gpo.po_message_create()
    187          if source or source == "":
    188              self.source = source
    189              self.target = ""
    190          elif gpo_message:
    191              self._gpo_message = gpo_message
    192          self.infer_state()
    193
    194 -    def infer_state(self):
    195          #FIXME: do obsolete
    196          if gpo.po_message_is_obsolete(self._gpo_message):
    197              self.set_state_n(self.STATE[self.S_OBSOLETE][0])
    198          elif gpo.po_message_is_fuzzy(self._gpo_message):
    199              self.set_state_n(self.STATE[self.S_FUZZY][0])
    200          elif self.gettarget():
    201              self.set_state_n(self.STATE[self.S_TRANSLATED][0])
    202          else:
    203              self.set_state_n(self.STATE[self.S_UNTRANSLATED][0])
    204
    205 -    def setmsgid_plural(self, msgid_plural):
    206          if isinstance(msgid_plural, list):
    207              msgid_plural = "".join(msgid_plural)
    208          gpo.po_message_set_msgid_plural(self._gpo_message, msgid_plural)
    209      msgid_plural = property(None, setmsgid_plural)
    210
    211 -    def getsource(self):
    212
    213          def remove_msgid_comments(text):
    214              if not text:
    215                  return text
    216              if text.startswith("_:"):
    217                  remainder = re.search(r"_: .*\n(.*)", text)
    218                  if remainder:
    219                      return remainder.group(1)
    220                  else:
    221                      return u""
    222              else:
    223                  return text
    224          singular = remove_msgid_comments((gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding))
    225          if singular:
    226              if self.hasplural():
    227                  multi = multistring(singular, self._encoding)
    228                  pluralform = (gpo.po_message_msgid_plural(self._gpo_message) or "").decode(self._encoding)
    229                  multi.strings.append(pluralform)
    230                  return multi
    231              else:
    232                  return singular
    233          else:
    234              return u""
    235
    236 -    def setsource(self, source):
    237          if isinstance(source, multistring):
    238              source = source.strings
    239          if isinstance(source, unicode):
    240              source = source.encode(self._encoding)
    241          if isinstance(source, list):
    242              gpo.po_message_set_msgid(self._gpo_message, source[0].encode(self._encoding))
    243              if len(source) > 1:
    244                  gpo.po_message_set_msgid_plural(self._gpo_message, source[1].encode(self._encoding))
    245          else:
    246              gpo.po_message_set_msgid(self._gpo_message, source)
    247              gpo.po_message_set_msgid_plural(self._gpo_message, None)
    248      source = property(getsource, setsource)
    249
    250 -    def gettarget(self):
    251          if self.hasplural():
    252              plurals = []
    253              nplural = 0
    254              plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
    255              while plural:
    256                  plurals.append(plural.decode(self._encoding))
    257                  nplural += 1
    258                  plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
    259              if plurals:
    260                  multi = multistring(plurals, encoding=self._encoding)
    261              else:
    262                  multi = multistring(u"")
    263          else:
    264              multi = (gpo.po_message_msgstr(self._gpo_message) or "").decode(self._encoding)
    265          return multi
    266
    267 -    def settarget(self, target):
    268          # for plural strings: convert 'target' into a list
    269          if self.hasplural():
    270              if isinstance(target, multistring):
    271                  target = target.strings
    272              elif isinstance(target, basestring):
    273                  target = [target]
    274          # for non-plurals: check number of items in 'target'
    275          elif isinstance(target, (dict, list)):
    276              if len(target) == 1:
    277                  target = target[0]
    278              else:
    279                  raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
    280          # empty the previous list of messages
    281          # TODO: the "pypo" implementation does not remove the previous items of
    282          #   the target, if self.target == target (essentially: comparing only
    283          #   the first item of a plural string with the single new string)
    284          #   Maybe this behaviour should be unified.
    285          if isinstance(target, (dict, list)):
    286              i = 0
    287              message = gpo.po_message_msgstr_plural(self._gpo_message, i)
    288              while message is not None:
    289                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
    290                  i += 1
    291                  message = gpo.po_message_msgstr_plural(self._gpo_message, i)
    292          # add the items of a list
    293          if isinstance(target, list):
    294              for i in range(len(target)):
    295                  targetstring = target[i]
    296                  if isinstance(targetstring, unicode):
    297                      targetstring = targetstring.encode(self._encoding)
    298                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
    299          # add the values of a dict
    300          elif isinstance(target, dict):
    301              for i, targetstring in enumerate(target.itervalues()):
    302                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
    303          # add a single string
    304          else:
    305              if isinstance(target, unicode):
    306                  target = target.encode(self._encoding)
    307              if target is None:
    308                  gpo.po_message_set_msgstr(self._gpo_message, "")
    309              else:
    310                  gpo.po_message_set_msgstr(self._gpo_message, target)
    311      target = property(gettarget, settarget)
    312
    313 -    def getid(self):
    314          """The unique identifier for this unit according to the convensions in
    315          .mo files."""
    316          id = (gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding)
    317          # Gettext does not consider the plural to determine duplicates, only
    318          # the msgid. For generation of .mo files, we might want to use this
    319          # code to generate the entry for the hash table, but for now, it is
    320          # commented out for conformance to gettext.
    321  #        plural = gpo.po_message_msgid_plural(self._gpo_message)
    322  #        if not plural is None:
    323  #            id = '%s\0%s' % (id, plural)
    324          context = gpo.po_message_msgctxt(self._gpo_message)
    325          if context:
    326              id = u"%s\04%s" % (context.decode(self._encoding), id)
    327          return id
    328
    329 -    def getnotes(self, origin=None):
    330          if origin == None:
    331              comments = gpo.po_message_comments(self._gpo_message) + \
    332                         gpo.po_message_extracted_comments(self._gpo_message)
    333          elif origin == "translator":
    334              comments = gpo.po_message_comments(self._gpo_message)
    335          elif origin in ["programmer", "developer", "source code"]:
    336              comments = gpo.po_message_extracted_comments(self._gpo_message)
    337          else:
    338              raise ValueError("Comment type not valid")
    339
    340          if comments and get_libgettextpo_version() < (0, 17, 0):
    341              comments = "\n".join([line for line in comments.split("\n")])
    342          # Let's drop the last newline
    343          return comments[:-1].decode(self._encoding)
    344
    345 -    def addnote(self, text, origin=None, position="append"):
    346          # ignore empty strings and strings without non-space characters
    347          if not (text and text.strip()):
    348              return
    349          text = data.forceunicode(text)
    350          oldnotes = self.getnotes(origin)
    351          newnotes = None
    352          if oldnotes:
    353              if position == "append":
    354                  newnotes = oldnotes + "\n" + text
    355              elif position == "merge":
    356                  if oldnotes != text:
    357                      oldnoteslist = oldnotes.split("\n")
    358                      for newline in text.split("\n"):
    359                          newline = newline.rstrip("\r")
    360                          # avoid duplicate comment lines (this might cause some problems)
    361                          if newline not in oldnotes or len(newline) < 5:
    362                              oldnoteslist.append(newline)
    363                      newnotes = "\n".join(oldnoteslist)
    364              else:
    365                  newnotes = text + '\n' + oldnotes
    366          else:
    367              newnotes = "\n".join([line.rstrip("\r") for line in text.split("\n")])
    368
    369          if newnotes:
    370              newlines = []
    371              needs_space = get_libgettextpo_version() < (0, 17, 0)
    372              for line in newnotes.split("\n"):
    373                  if line and needs_space:
    374                      newlines.append(" " + line)
    375                  else:
    376                      newlines.append(line)
    377              newnotes = "\n".join(newlines).encode(self._encoding)
    378              if origin in ["programmer", "developer", "source code"]:
    379                  gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
    380              else:
    381                  gpo.po_message_set_comments(self._gpo_message, newnotes)
    382
    383 -    def removenotes(self):
    384          gpo.po_message_set_comments(self._gpo_message, "")
    385
    386 -    def copy(self):
    387          newpo = self.__class__()
    388          newpo._gpo_message = self._gpo_message
    389          return newpo
    390
    391 -    def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
    392          """Merges the otherpo (with the same msgid) into this one.
    393
    394          Overwrite non-blank self.msgstr only if overwrite is True
    395          merge comments only if comments is True
    396          """
    397
    398          if not isinstance(otherpo, pounit):
    399              super(pounit, self).merge(otherpo, overwrite, comments)
    400              return
    401          if comments:
    402              self.addnote(otherpo.getnotes("translator"), origin="translator", position="merge")
    403              # FIXME mergelists(self.typecomments, otherpo.typecomments)
    404              if not authoritative:
    405                  # We don't bring across otherpo.automaticcomments as we consider ourself
    406                  # to be the the authority.  Same applies to otherpo.msgidcomments
    407                  self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge")
    408                  self.msgidcomment = otherpo._extract_msgidcomments() or None
    409                  self.addlocations(otherpo.getlocations())
    410          if not self.istranslated() or overwrite:
    411              # Remove kde-style comments from the translation (if any).
    412              if self._extract_msgidcomments(otherpo.target):
    413                  otherpo.target = otherpo.target.replace('_: ' + otherpo._extract_msgidcomments() + '\n', '')
    414              self.target = otherpo.target
    415              if self.source != otherpo.source or self.getcontext() != otherpo.getcontext():
    416                  self.markfuzzy()
    417              else:
    418                  self.markfuzzy(otherpo.isfuzzy())
    419          elif not otherpo.istranslated():
    420              if self.source != otherpo.source:
    421                  self.markfuzzy()
    422          else:
    423              if self.target != otherpo.target:
    424                  self.markfuzzy()
    425
    426 -    def isheader(self):
    427          #return self.source == u"" and self.target != u""
    428          # we really want to make sure that there is no msgidcomment or msgctxt
    429          return self.getid() == "" and len(self.target) > 0
    430
    431 -    def isblank(self):
    432          return len(self.source) == len(self.target) == len(self.getcontext()) == 0
    433
    434 -    def hastypecomment(self, typecomment):
    435          return gpo.po_message_is_format(self._gpo_message, typecomment)
    436
    437 -    def settypecomment(self, typecomment, present=True):
    438          gpo.po_message_set_format(self._gpo_message, typecomment, present)
    439
    440 -    def hasmarkedcomment(self, commentmarker):
    441          commentmarker = "(%s)" % commentmarker
    442          for comment in self.getnotes("translator").split("\n"):
    443              if comment.startswith(commentmarker):
    444                  return True
    445          return False
    446
    447 -    def isfuzzy(self):
    448          return gpo.po_message_is_fuzzy(self._gpo_message)
    449
    450 -    def _domarkfuzzy(self, present=True):
    451          gpo.po_message_set_fuzzy(self._gpo_message, present)
    452
    453 -    def makeobsolete(self):
    454          # FIXME: libgettexpo currently does not reset other data, we probably want to do that
    455          # but a better solution would be for libgettextpo to output correct data on serialisation
    456          gpo.po_message_set_obsolete(self._gpo_message, True)
    457          self.infer_state()
    458
    459 -    def resurrect(self):
    460          gpo.po_message_set_obsolete(self._gpo_message, False)
    461          self.infer_state()
    462
    463 -    def hasplural(self):
    464          return gpo.po_message_msgid_plural(self._gpo_message) is not None
    465
    466 -    def _extract_msgidcomments(self, text=None):
    467          """Extract KDE style msgid comments from the unit.
    468
    469          @rtype: String
    470          @return: Returns the extracted msgidcomments found in this unit's msgid.
    471          """
    472          if not text:
    473              text = (gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding)
    474          if text:
    475              return pocommon.extract_msgid_comment(text)
    476          return u""
    477
    478 -    def setmsgidcomment(self, msgidcomment):
    479          if msgidcomment:
    480              self.source = u"_: %s\n%s" % (msgidcomment, self.source)
    481      msgidcomment = property(_extract_msgidcomments, setmsgidcomment)
    482
    483 -    def __str__(self):
    484          pf = pofile(noheader=True)
    485          pf.addunit(self)
    486          return str(pf)
    487
    488 -    def getlocations(self):
    489          locations = []
    490          i = 0
    491          location = gpo.po_message_filepos(self._gpo_message, i)
    492          while location:
    493              locname = gpo.po_filepos_file(location)
    494              locline = gpo.po_filepos_start_line(location)
    495              if locline == -1:
    496                  locstring = locname
    497              else:
    498                  locstring = locname + ":" + str(locline)
    499              locations.append(pocommon.unquote_plus(locstring))
    500              i += 1
    501              location = gpo.po_message_filepos(self._gpo_message, i)
    502          return locations
    503
    504 -    def addlocation(self, location):
    505          if location.find(" ") != -1:
    506              location = pocommon.quote_plus(location)
    507          parts = location.split(":")
    508          file = parts[0]
    509          if len(parts) == 2:
    510              line = int(parts[1] or "0")
    511          else:
    512              line = -1
    513          gpo.po_message_add_filepos(self._gpo_message, file, line)
    514
    515 -    def getcontext(self):
    516          msgctxt = gpo.po_message_msgctxt(self._gpo_message)
    517          if msgctxt:
    518              return msgctxt.decode(self._encoding)
    519          else:
    520              msgidcomment = self._extract_msgidcomments()
    521              return msgidcomment
    522
    523 -    def setcontext(self, context):
    524          context = data.forceunicode(context)
    525          gpo.po_message_set_msgctxt(self._gpo_message, context)
    526
    527 -    def buildfromunit(cls, unit, encoding=None):
    528          """Build a native unit from a foreign unit, preserving as much
    529          information as possible."""
    530          if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy):
    531              return unit.copy()
    532          elif isinstance(unit, pocommon.pounit):
    533              newunit = cls(unit.source, encoding)
    534              newunit.target = unit.target
    535              #context
    536              newunit.msgidcomment = unit._extract_msgidcomments()
    537              context = unit.getcontext()
    538              if not newunit.msgidcomment and context:
    539                  gpo.po_message_set_msgctxt(newunit._gpo_message, context)
    540
    541              locations = unit.getlocations()
    542              if locations:
    543                  newunit.addlocations(locations)
    544              notes = unit.getnotes("developer")
    545              if notes:
    546                  newunit.addnote(notes, "developer")
    547              notes = unit.getnotes("translator")
    548              if notes:
    549                  newunit.addnote(notes, "translator")
    550              if unit.isobsolete():
    551                  newunit.makeobsolete()
    552              newunit.markfuzzy(unit.isfuzzy())
    553              for tc in ['python-format', 'c-format', 'php-format']:
    554                  if unit.hastypecomment(tc):
    555                      newunit.settypecomment(tc)
    556                      # We assume/guess/hope that there will only be one
    557                      break
    558              return newunit
    559          else:
    560              return base.TranslationUnit.buildfromunit(unit)
    561      buildfromunit = classmethod(buildfromunit)
    562
    563
    564 -class pofile(pocommon.pofile):
    565      UnitClass = pounit
    566
    567 -    def __init__(self, inputfile=None, encoding=None, unitclass=pounit, noheader=False):
    568          self._gpo_memory_file = None
    569          self._gpo_message_iterator = None
    570          self.units = []
    571          self.sourcelanguage = None
    572          self.targetlanguage = None
    573          self._encoding = 'utf-8'
    574          if inputfile is None:
    575              self._gpo_memory_file = gpo.po_file_create()
    576              self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
    577              if not noheader:
    578                  self.init_headers()
    579          else:
    580              super(pofile, self).__init__(inputfile=inputfile, encoding=encoding)
    581
    582 -    def addunit(self, unit, new=True):
    583          if new:
    584              gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
    585          super(pofile, self).addunit(unit)
    586
    587 -    def _insert_header(self, header):
    588          header._store = self
    589          self.units.insert(0, header)
    590          gpo.po_message_iterator_free(self._gpo_message_iterator)
    591          self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
    592          gpo.po_message_insert(self._gpo_message_iterator, header._gpo_message)
    593          while gpo.po_next_message(self._gpo_message_iterator):
    594              pass
    595
    596 -    def removeduplicates(self, duplicatestyle="merge"):
    597          """make sure each msgid is unique ; merge comments etc from duplicates into original"""
    598          # TODO: can we handle consecutive calls to removeduplicates()? What
    599          # about files already containing msgctxt? - test
    600          id_dict = {}
    601          uniqueunits = []
    602          # TODO: this is using a list as the pos aren't hashable, but this is slow.
    603          # probably not used frequently enough to worry about it, though.
    604          markedpos = []
    605
    606          def addcomment(thepo):
    607              thepo.msgidcomment = " ".join(thepo.getlocations())
    608              markedpos.append(thepo)
    609          for thepo in self.units:
    610              id = thepo.getid()
    611              if thepo.isheader() and not thepo.getlocations():
    612                  # header msgids shouldn't be merged...
    613                  uniqueunits.append(thepo)
    614              elif id in id_dict:
    615                  if duplicatestyle == "merge":
    616                      if id:
    617                          id_dict[id].merge(thepo)
    618                      else:
    619                          addcomment(thepo)
    620                          uniqueunits.append(thepo)
    621                  elif duplicatestyle == "msgctxt":
    622                      origpo = id_dict[id]
    623                      if origpo not in markedpos:
    624                          gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
    625                          markedpos.append(thepo)
    626                      gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
    627                      uniqueunits.append(thepo)
    628              else:
    629                  if not id:
    630                      if duplicatestyle == "merge":
    631                          addcomment(thepo)
    632                      else:
    633                          gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
    634                  id_dict[id] = thepo
    635                  uniqueunits.append(thepo)
    636          new_gpo_memory_file = gpo.po_file_create()
    637          new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
    638          for unit in uniqueunits:
    639              gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
    640          gpo.po_message_iterator_free(self._gpo_message_iterator)
    641          self._gpo_message_iterator = new_gpo_message_iterator
    642          self._gpo_memory_file = new_gpo_memory_file
    643          self.units = uniqueunits
    644
    645 -    def __str__(self):
    646
    647          def obsolete_workaround():
    648              # Remove all items that are not output by msgmerge when a unit is obsolete.  This is a work
    649              # around for bug in libgettextpo
    650              # FIXME Do version test in case they fix this bug
    651              for unit in self.units:
    652                  if unit.isobsolete():
    653                      gpo.po_message_set_extracted_comments(unit._gpo_message, "")
    654                      location = gpo.po_message_filepos(unit._gpo_message, 0)
    655                      while location:
    656                          gpo.po_message_remove_filepos(unit._gpo_message, 0)
    657                          location = gpo.po_message_filepos(unit._gpo_message, 0)
    658          outputstring = ""
    659          if self._gpo_memory_file:
    660              obsolete_workaround()
    661              f, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
    662              os.close(f)
    663              self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, fname, xerror_handler)
    664              f = open(fname)
    665              outputstring = f.read()
    666              f.close()
    667              os.remove(fname)
    668          return outputstring
    669
    670 -    def isempty(self):
    671          """Returns True if the object doesn't contain any translation units."""
    672          if len(self.units) == 0:
    673              return True
    674          # Skip the first unit if it is a header.
    675          if self.units[0].isheader():
    676              units = self.units[1:]
    677          else:
    678              units = self.units
    679
    680          for unit in units:
    681              if not unit.isblank() and not unit.isobsolete():
    682                  return False
    683          return True
    684
    685 -    def parse(self, input):
    686          if hasattr(input, 'name'):
    687              self.filename = input.name
    688          elif not getattr(self, 'filename', ''):
    689              self.filename = ''
    690
    691          if hasattr(input, "read"):
    692              posrc = input.read()
    693              input.close()
    694              input = posrc
    695
    696          needtmpfile = not os.path.isfile(input)
    697          if needtmpfile:
    698              # This is not a file - we write the string to a temporary file
    699              fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
    700              os.write(fd, input)
    701              input = fname
    702              os.close(fd)
    703
    704          self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
    705          if self._gpo_memory_file is None:
    706              print >> sys.stderr, "Error:"
    707
    708          if needtmpfile:
    709              os.remove(input)
    710
    711          self.units = []
    712          # Handle xerrors here
    713          self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
    714          if self._header:
    715              charset = gpo.po_header_field(self._header, "Content-Type")
    716              if charset:
    717                  charset = re.search("charset=([^\\s]+)", charset).group(1)
    718              self._encoding = encodingToUse(charset)
    719          self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
    720          newmessage = gpo.po_next_message(self._gpo_message_iterator)
    721          while newmessage:
    722              newunit = pounit(gpo_message=newmessage, encoding=self._encoding)
    723              self.addunit(newunit, new=False)
    724              newmessage = gpo.po_next_message(self._gpo_message_iterator)
    725          self._free_iterator()
    726
    727 -    def __del__(self):
    728          # We currently disable this while we still get segmentation faults.
    729          # Note that this is definitely leaking memory because of this.
    730          return
    731          self._free_iterator()
    732          if self._gpo_memory_file is not None:
    733              gpo.po_file_free(self._gpo_memory_file)
    734              self._gpo_memory_file = None
    735
    736 -    def _free_iterator(self):
    737          # We currently disable this while we still get segmentation faults.
    738          # Note that this is definitely leaking memory because of this.
    739          return
    740          if self._gpo_message_iterator is not None:
    741              gpo.po_message_iterator_free(self._gpo_message_iterator)
    742              self._gpo_message_iterator = None
    743