1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects.
26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library.
31 """
32
33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon
35 from translate.storage.pocommon import encodingToUse
36 from translate.misc import quote
37 from translate.lang import data
38 from ctypes import *
39 import ctypes.util
40 try:
41 import cStringIO as StringIO
42 except ImportError:
43 import StringIO
44 import os
45 import pypo
46 import re
47 import sys
48 import tempfile
49
50 lsep = " "
51 """Seperator for #: entries"""
52
53 STRING = c_char_p
54
55
58
59
60 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
61 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
62
63
64
68
70 _fields_ = [
71 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
72 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
73 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
74 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
75 ]
76
77
78 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
79 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
80 if severity >= 1:
81 raise ValueError(message_text)
82
83 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
84 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
85 if severity >= 1:
86 raise ValueError(message_text1)
87
88
89
90
91 gpo = None
92
93
94 names = ['gettextpo', 'libgettextpo']
95 for name in names:
96 lib_location = ctypes.util.find_library(name)
97 if lib_location:
98 gpo = cdll.LoadLibrary(lib_location)
99 if gpo:
100 break
101 else:
102
103
104 try:
105 gpo = cdll.LoadLibrary('libgettextpo.so')
106 except OSError, e:
107 raise ImportError("gettext PO library not found")
108
109
110
111 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
113 gpo.po_file_write_v2.retype = c_int
114
115
116 gpo.po_file_domain_header.restype = STRING
117 gpo.po_header_field.restype = STRING
118 gpo.po_header_field.argtypes = [STRING, STRING]
119
120
121 gpo.po_filepos_file.restype = STRING
122 gpo.po_message_filepos.restype = c_int
123 gpo.po_message_filepos.argtypes = [c_int, c_int]
124 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
125
126
127 gpo.po_message_comments.restype = STRING
128 gpo.po_message_extracted_comments.restype = STRING
129 gpo.po_message_prev_msgctxt.restype = STRING
130 gpo.po_message_prev_msgid.restype = STRING
131 gpo.po_message_prev_msgid_plural.restype = STRING
132 gpo.po_message_is_format.restype = c_int
133 gpo.po_message_is_format.argtypes = [c_int, STRING]
134 gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
135 gpo.po_message_msgctxt.restype = STRING
136 gpo.po_message_msgid.restype = STRING
137 gpo.po_message_msgid_plural.restype = STRING
138 gpo.po_message_msgstr.restype = STRING
139 gpo.po_message_msgstr_plural.restype = STRING
140
141
142 gpo.po_message_set_comments.argtypes = [c_int, STRING]
143 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
144 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
145 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
146
147
148 xerror_handler = po_xerror_handler()
149 xerror_handler.xerror = xerror_prototype(xerror_cb)
150 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
151
154
157
160
162 """Returns the libgettextpo version
163
164 @rtype: three-value tuple
165 @return: libgettextpo version in the following format::
166 (major version, minor version, subminor version)
167 """
168 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
169 major = libversion.value >> 16
170 minor = libversion.value >> 8
171 subminor = libversion.value - (major << 16) - (minor << 8)
172 return major, minor, subminor
173
174
175 -class pounit(pocommon.pounit):
176 - def __init__(self, source=None, encoding='utf-8', gpo_message=None):
177 self._rich_source = None
178 self._rich_target = None
179 self._encoding = encoding
180 if not gpo_message:
181 self._gpo_message = gpo.po_message_create()
182 if source or source == "":
183 self.source = source
184 self.target = ""
185 elif gpo_message:
186 self._gpo_message = gpo_message
187
189 if isinstance(msgid_plural, list):
190 msgid_plural = "".join(msgid_plural)
191 gpo.po_message_set_msgid_plural(self._gpo_message, msgid_plural)
192 msgid_plural = property(None, setmsgid_plural)
193
195 def remove_msgid_comments(text):
196 if not text:
197 return text
198 if text.startswith("_:"):
199 remainder = re.search(r"_: .*\n(.*)", text)
200 if remainder:
201 return remainder.group(1)
202 else:
203 return u""
204 else:
205 return text
206 singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message).decode(self._encoding))
207 if singular:
208 if self.hasplural():
209 multi = multistring(singular, self._encoding)
210 pluralform = gpo.po_message_msgid_plural(self._gpo_message).decode(self._encoding)
211 multi.strings.append(pluralform)
212 return multi
213 else:
214 return singular
215 else:
216 return u""
217
230
231 source = property(getsource, setsource)
232
234 if self.hasplural():
235 plurals = []
236 nplural = 0
237 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
238 while plural:
239 plurals.append(plural.decode(self._encoding))
240 nplural += 1
241 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
242 if plurals:
243 multi = multistring(plurals, encoding=self._encoding)
244 else:
245 multi = multistring(u"")
246 else:
247 multi = (gpo.po_message_msgstr(self._gpo_message) or "").decode(self._encoding)
248 return multi
249
251
252 if self.hasplural():
253 if isinstance(target, multistring):
254 target = target.strings
255 elif isinstance(target, basestring):
256 target = [target]
257
258 elif isinstance(target, (dict, list)):
259 if len(target) == 1:
260 target = target[0]
261 else:
262 raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
263
264
265
266
267
268 if isinstance(target, (dict, list)):
269 i = 0
270 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
271 while message is not None:
272 gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
273 i += 1
274 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
275
276 if isinstance(target, list):
277 for i in range(len(target)):
278 targetstring = target[i]
279 if isinstance(targetstring, unicode):
280 targetstring = targetstring.encode(self._encoding)
281 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
282
283 elif isinstance(target, dict):
284 for i, targetstring in enumerate(target.itervalues()):
285 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
286
287 else:
288 if isinstance(target, unicode):
289 target = target.encode(self._encoding)
290 if target is None:
291 gpo.po_message_set_msgstr(self._gpo_message, "")
292 else:
293 gpo.po_message_set_msgstr(self._gpo_message, target)
294 target = property(gettarget, settarget)
295
297 """The unique identifier for this unit according to the convensions in
298 .mo files."""
299 id = (gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding)
300
301
302
303
304
305
306
307 context = gpo.po_message_msgctxt(self._gpo_message)
308 if context:
309 id = u"%s\04%s" % (context.decode(self._encoding), id)
310 return id
311
313 if origin == None:
314 comments = gpo.po_message_comments(self._gpo_message) + \
315 gpo.po_message_extracted_comments(self._gpo_message)
316 elif origin == "translator":
317 comments = gpo.po_message_comments(self._gpo_message)
318 elif origin in ["programmer", "developer", "source code"]:
319 comments = gpo.po_message_extracted_comments(self._gpo_message)
320 else:
321 raise ValueError("Comment type not valid")
322
323 if comments and get_libgettextpo_version() < (0, 17, 0):
324 comments = "\n".join([line.strip() for line in comments.split("\n")])
325
326 return comments[:-1].decode(self._encoding)
327
328 - def addnote(self, text, origin=None, position="append"):
329
330 if not (text and text.strip()):
331 return
332 text = data.forceunicode(text)
333 oldnotes = self.getnotes(origin)
334 newnotes = None
335 if oldnotes:
336 if position == "append":
337 newnotes = oldnotes + "\n" + text
338 elif position == "merge":
339 if oldnotes != text:
340 oldnoteslist = oldnotes.split("\n")
341 for newline in text.split("\n"):
342 newline = newline.rstrip()
343
344 if newline not in oldnotes or len(newline) < 5:
345 oldnoteslist.append(newline)
346 newnotes = "\n".join(oldnoteslist)
347 else:
348 newnotes = text + '\n' + oldnotes
349 else:
350 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
351
352 if newnotes:
353 newlines = []
354 needs_space = get_libgettextpo_version() < (0, 17, 0)
355 for line in newnotes.split("\n"):
356 if line and needs_space:
357 newlines.append(" " + line)
358 else:
359 newlines.append(line)
360 newnotes = "\n".join(newlines).encode(self._encoding)
361 if origin in ["programmer", "developer", "source code"]:
362 gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
363 else:
364 gpo.po_message_set_comments(self._gpo_message, newnotes)
365
367 gpo.po_message_set_comments(self._gpo_message, "")
368
370 newpo = self.__class__()
371 newpo._gpo_message = self._gpo_message
372 return newpo
373
374 - def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
408
410
411
412 return self.getid() == "" and len(self.target) > 0
413
416
419
422
429
431 return gpo.po_message_is_fuzzy(self._gpo_message)
432
434 gpo.po_message_set_fuzzy(self._gpo_message, present)
435
437 return gpo.po_message_is_obsolete(self._gpo_message)
438
440
441
442 gpo.po_message_set_obsolete(self._gpo_message, True)
443
445 gpo.po_message_set_obsolete(self._gpo_message, False)
446
448 return gpo.po_message_msgid_plural(self._gpo_message) is not None
449
461
465 msgidcomment = property(_extract_msgidcomments, setmsgidcomment)
466
471
473 locations = []
474 i = 0
475 location = gpo.po_message_filepos(self._gpo_message, i)
476 while location:
477 locname = gpo.po_filepos_file(location)
478 locline = gpo.po_filepos_start_line(location)
479 if locline == -1:
480 locstring = locname
481 else:
482 locstring = locname + ":" + str(locline)
483 locations.append(locstring)
484 i += 1
485 location = gpo.po_message_filepos(self._gpo_message, i)
486 return locations
487
489 for loc in location.split():
490 parts = loc.split(":")
491 file = parts[0]
492 if len(parts) == 2:
493 line = int(parts[1] or "0")
494 else:
495 line = -1
496 gpo.po_message_add_filepos(self._gpo_message, file, line)
497
498 - def getcontext(self):
499 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
500 if msgctxt:
501 return msgctxt.decode(self._encoding)
502 else:
503 msgidcomment = self._extract_msgidcomments()
504 return msgidcomment
505
540 buildfromunit = classmethod(buildfromunit)
541
542 -class pofile(pocommon.pofile):
543 UnitClass = pounit
545 self._gpo_memory_file = None
546 self._gpo_message_iterator = None
547 super(pofile, self).__init__(inputfile=inputfile, encoding=encoding)
548 if inputfile is None:
549 self._gpo_memory_file = gpo.po_file_create()
550 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
551
552 - def addunit(self, unit, new=True):
553 if new:
554 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
555 super(pofile, self).addunit(unit)
556
558 """make sure each msgid is unique ; merge comments etc from duplicates into original"""
559
560
561 id_dict = {}
562 uniqueunits = []
563
564
565 markedpos = []
566 def addcomment(thepo):
567 thepo.msgidcomment = " ".join(thepo.getlocations())
568 markedpos.append(thepo)
569 for thepo in self.units:
570 id = thepo.getid()
571 if thepo.isheader() and not thepo.getlocations():
572
573 uniqueunits.append(thepo)
574 elif id in id_dict:
575 if duplicatestyle == "merge":
576 if id:
577 id_dict[id].merge(thepo)
578 else:
579 addcomment(thepo)
580 uniqueunits.append(thepo)
581 elif duplicatestyle == "msgctxt":
582 origpo = id_dict[id]
583 if origpo not in markedpos:
584 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
585 markedpos.append(thepo)
586 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
587 uniqueunits.append(thepo)
588 else:
589 if not id:
590 if duplicatestyle == "merge":
591 addcomment(thepo)
592 else:
593 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
594 id_dict[id] = thepo
595 uniqueunits.append(thepo)
596 new_gpo_memory_file = gpo.po_file_create()
597 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
598 for unit in uniqueunits:
599 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
600 gpo.po_message_iterator_free(self._gpo_message_iterator)
601 self._gpo_message_iterator = new_gpo_message_iterator
602 self._gpo_memory_file = new_gpo_memory_file
603 self.units = uniqueunits
604
606 def obsolete_workaround():
607
608
609
610 for unit in self.units:
611 if unit.isobsolete():
612 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
613 location = gpo.po_message_filepos(unit._gpo_message, 0)
614 while location:
615 gpo.po_message_remove_filepos(unit._gpo_message, 0)
616 location = gpo.po_message_filepos(unit._gpo_message, 0)
617 outputstring = ""
618 if self._gpo_memory_file:
619 obsolete_workaround()
620 f, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
621 os.close(f)
622 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, fname, xerror_handler)
623 f = open(fname)
624 outputstring = f.read()
625 f.close()
626 os.remove(fname)
627 return outputstring
628
630 """Returns True if the object doesn't contain any translation units."""
631 if len(self.units) == 0:
632 return True
633
634 if self.units[0].isheader():
635 units = self.units[1:]
636 else:
637 units = self.units
638
639 for unit in units:
640 if not unit.isblank() and not unit.isobsolete():
641 return False
642 return True
643
645 if hasattr(input, 'name'):
646 self.filename = input.name
647 elif not getattr(self, 'filename', ''):
648 self.filename = ''
649
650 if hasattr(input, "read"):
651 posrc = input.read()
652 input.close()
653 input = posrc
654
655 needtmpfile = not os.path.isfile(input)
656 if needtmpfile:
657
658 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
659 os.write(fd, input)
660 input = fname
661 os.close(fd)
662
663 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
664 if self._gpo_memory_file is None:
665 print >> sys.stderr, "Error:"
666
667 if needtmpfile:
668 os.remove(input)
669
670
671 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
672 if self._header:
673 charset = gpo.po_header_field(self._header, "Content-Type")
674 if charset:
675 charset = re.search("charset=([^\\s]+)", charset).group(1)
676 self._encoding = encodingToUse(charset)
677 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
678 newmessage = gpo.po_next_message(self._gpo_message_iterator)
679 while newmessage:
680 newunit = pounit(gpo_message=newmessage, encoding=self._encoding)
681 self.addunit(newunit, new=False)
682 newmessage = gpo.po_next_message(self._gpo_message_iterator)
683 self._free_iterator()
684
686
687
688 return
689 self._free_iterator()
690 if self._gpo_memory_file is not None:
691 gpo.po_file_free(self._gpo_memory_file)
692 self._gpo_memory_file = None
693
695
696
697 return
698 if self._gpo_message_iterator is not None:
699 gpo.po_message_iterator_free(self._gpo_message_iterator)
700 self._gpo_message_iterator = None
701