Package translate :: Package storage :: Module base
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.base

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # 
  4  # Copyright 2006-2009 Zuza Software Foundation 
  5  # 
  6  # This file is part of the Translate Toolkit. 
  7  # 
  8  # This program is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  # 
 13  # This program is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with this program; if not, see <http://www.gnu.org/licenses/>. 
 20   
 21  """Base classes for storage interfaces. 
 22   
 23  @organization: Zuza Software Foundation 
 24  @copyright: 2006-2009 Zuza Software Foundation 
 25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>} 
 26  """ 
 27   
 28  try: 
 29      import cPickle as pickle 
 30  except: 
 31      import pickle 
 32  from exceptions import NotImplementedError 
 33  import translate.i18n 
 34  from translate.storage.placeables import StringElem, general, parse as rich_parse 
 35  from translate.misc.typecheck import accepts, Self, IsOneOf 
 36  from translate.misc.multistring import multistring 
 37   
38 -def force_override(method, baseclass):
39 """Forces derived classes to override method.""" 40 41 if type(method.im_self) == type(baseclass): 42 # then this is a classmethod and im_self is the actual class 43 actualclass = method.im_self 44 else: 45 actualclass = method.im_class 46 if actualclass != baseclass: 47 raise NotImplementedError( 48 "%s does not reimplement %s as required by %s" % \ 49 (actualclass.__name__, method.__name__, baseclass.__name__) 50 )
51 52
53 -class ParseError(Exception):
54 - def __init__(self, inner_exc):
55 self.inner_exc = inner_exc
56
57 - def __str__(self):
58 return repr(self.inner_exc)
59 60
61 -class TranslationUnit(object):
62 """Base class for translation units. 63 64 Our concept of a I{translation unit} is influenced heavily by XLIFF: 65 U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm} 66 67 As such most of the method- and variable names borrows from XLIFF terminology. 68 69 A translation unit consists of the following: 70 - A I{source} string. This is the original translatable text. 71 - A I{target} string. This is the translation of the I{source}. 72 - Zero or more I{notes} on the unit. Notes would typically be some 73 comments from a translator on the unit, or some comments originating from 74 the source code. 75 - Zero or more I{locations}. Locations indicate where in the original 76 source code this unit came from. 77 - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on 78 translations and produce error messages. 79 80 @group Source: *source* 81 @group Target: *target* 82 @group Notes: *note* 83 @group Locations: *location* 84 @group Errors: *error* 85 """ 86 87 rich_parsers = [] 88 """A list of functions to use for parsing a string into a rich string tree.""" 89
90 - def __init__(self, source):
91 """Constructs a TranslationUnit containing the given source string.""" 92 self.notes = "" 93 self._store = None 94 self.source = source 95 self._target = None 96 self._rich_source = None 97 self._rich_target = None
98
99 - def __eq__(self, other):
100 """Compares two TranslationUnits. 101 102 @type other: L{TranslationUnit} 103 @param other: Another L{TranslationUnit} 104 @rtype: Boolean 105 @return: Returns True if the supplied TranslationUnit equals this unit. 106 """ 107 return self.source == other.source and self.target == other.target
108
109 - def __str__(self):
110 """Converts to a string representation that can be parsed back using L{parsestring()}.""" 111 # no point in pickling store object, so let's hide it for a while. 112 store = getattr(self, "_store", None) 113 self._store = None 114 dump = pickle.dumps(self) 115 self._store = store 116 return dump
117
118 - def rich_to_multistring(cls, elem_list):
119 """Convert a "rich" string tree to a C{multistring}: 120 121 >>> from translate.storage.placeables.interfaces import X 122 >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])] 123 >>> TranslationUnit.rich_to_multistring(rich) 124 multistring(u'foo bar') 125 """ 126 return multistring([unicode(elem) for elem in elem_list])
127 rich_to_multistring = classmethod(rich_to_multistring) 128
129 - def multistring_to_rich(cls, mulstring):
130 """Convert a multistring to a list of "rich" string trees: 131 132 >>> target = multistring([u'foo', u'bar', u'baz']) 133 >>> TranslationUnit.multistring_to_rich(target) 134 [<StringElem([<StringElem([u'foo'])>])>, 135 <StringElem([<StringElem([u'bar'])>])>, 136 <StringElem([<StringElem([u'baz'])>])>] 137 """ 138 if isinstance(mulstring, multistring): 139 return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings] 140 return [rich_parse(mulstring, cls.rich_parsers)]
141
142 - def setsource(self, source):
143 """Sets the source string to the given value.""" 144 self._rich_source = None 145 self._source = source
146 source = property(lambda self: self._source, setsource) 147
148 - def settarget(self, target):
149 """Sets the target string to the given value.""" 150 self._rich_target = None 151 self._target = target
152 target = property(lambda self: self._target, settarget) 153
154 - def _get_rich_source(self):
155 if self._rich_source is None: 156 self._rich_source = self.multistring_to_rich(self.source) 157 return self._rich_source
158 - def _set_rich_source(self, value):
159 if not hasattr(value, '__iter__'): 160 raise ValueError('value must be iterable') 161 if len(value) < 1: 162 raise ValueError('value must have at least one element.') 163 if not isinstance(value[0], StringElem): 164 raise ValueError('value[0] must be of type StringElem.') 165 self._rich_source = list(value) 166 self.source = self.rich_to_multistring(value)
167 rich_source = property(_get_rich_source, _set_rich_source) 168 """ @see: rich_to_multistring 169 @see: multistring_to_rich""" 170
171 - def _get_rich_target(self):
172 if self._rich_target is None: 173 self._rich_target = self.multistring_to_rich(self.target) 174 return self._rich_target
175 - def _set_rich_target(self, value):
176 if not hasattr(value, '__iter__'): 177 raise ValueError('value must be iterable') 178 if len(value) < 1: 179 raise ValueError('value must have at least one element.') 180 if not isinstance(value[0], StringElem): 181 raise ValueError('value[0] must be of type StringElem.') 182 self._rich_target = list(value) 183 self.target = self.rich_to_multistring(value)
184 rich_target = property(_get_rich_target, _set_rich_target) 185 """ @see: rich_to_multistring 186 @see: multistring_to_rich""" 187
188 - def gettargetlen(self):
189 """Returns the length of the target string. 190 191 @note: Plural forms might be combined. 192 @rtype: Integer 193 """ 194 length = len(self.target or "") 195 strings = getattr(self.target, "strings", []) 196 if strings: 197 length += sum([len(pluralform) for pluralform in strings[1:]]) 198 return length
199
200 - def getid(self):
201 """A unique identifier for this unit. 202 203 @rtype: string 204 @return: an identifier for this unit that is unique in the store 205 206 Derived classes should override this in a way that guarantees a unique 207 identifier for each unit in the store. 208 """ 209 return self.source
210
211 - def setid(self, value):
212 """Sets the unique identified for this unit. 213 214 only implemented if format allows ids independant from other 215 unit properties like source or context""" 216 pass
217
218 - def getlocations(self):
219 """A list of source code locations. 220 221 @note: Shouldn't be implemented if the format doesn't support it. 222 @rtype: List 223 """ 224 return []
225
226 - def addlocation(self, location):
227 """Add one location to the list of locations. 228 229 @note: Shouldn't be implemented if the format doesn't support it. 230 """ 231 pass
232
233 - def addlocations(self, location):
234 """Add a location or a list of locations. 235 236 @note: Most classes shouldn't need to implement this, 237 but should rather implement L{addlocation()}. 238 @warning: This method might be removed in future. 239 """ 240 if isinstance(location, list): 241 for item in location: 242 self.addlocation(item) 243 else: 244 self.addlocation(location)
245
246 - def getcontext(self):
247 """Get the message context.""" 248 return ""
249
250 - def setcontext(self, context):
251 """Set the message context""" 252 pass
253
254 - def getnotes(self, origin=None):
255 """Returns all notes about this unit. 256 257 It will probably be freeform text or something reasonable that can be 258 synthesised by the format. 259 It should not include location comments (see L{getlocations()}). 260 """ 261 return getattr(self, "notes", "")
262
263 - def addnote(self, text, origin=None, position="append"):
264 """Adds a note (comment). 265 266 @type text: string 267 @param text: Usually just a sentence or two. 268 @type origin: string 269 @param origin: Specifies who/where the comment comes from. 270 Origin can be one of the following text strings: 271 - 'translator' 272 - 'developer', 'programmer', 'source code' (synonyms) 273 """ 274 if getattr(self, "notes", None): 275 self.notes += '\n'+text 276 else: 277 self.notes = text
278
279 - def removenotes(self):
280 """Remove all the translator's notes.""" 281 self.notes = u''
282
283 - def adderror(self, errorname, errortext):
284 """Adds an error message to this unit. 285 286 @type errorname: string 287 @param errorname: A single word to id the error. 288 @type errortext: string 289 @param errortext: The text describing the error. 290 """ 291 pass
292
293 - def geterrors(self):
294 """Get all error messages. 295 296 @rtype: Dictionary 297 """ 298 return {}
299
300 - def markreviewneeded(self, needsreview=True, explanation=None):
301 """Marks the unit to indicate whether it needs review. 302 303 @keyword needsreview: Defaults to True. 304 @keyword explanation: Adds an optional explanation as a note. 305 """ 306 pass
307
308 - def istranslated(self):
309 """Indicates whether this unit is translated. 310 311 This should be used rather than deducing it from .target, 312 to ensure that other classes can implement more functionality 313 (as XLIFF does). 314 """ 315 return bool(self.target) and not self.isfuzzy()
316
317 - def istranslatable(self):
318 """Indicates whether this unit can be translated. 319 320 This should be used to distinguish real units for translation from 321 header, obsolete, binary or other blank units. 322 """ 323 return True
324
325 - def isfuzzy(self):
326 """Indicates whether this unit is fuzzy.""" 327 return False
328
329 - def markfuzzy(self, value=True):
330 """Marks the unit as fuzzy or not.""" 331 pass
332
333 - def isobsolete(self):
334 """indicate whether a unit is obsolete""" 335 return False
336
337 - def makeobsolete(self):
338 """Make a unit obsolete""" 339 pass
340
341 - def isheader(self):
342 """Indicates whether this unit is a header.""" 343 return False
344
345 - def isreview(self):
346 """Indicates whether this unit needs review.""" 347 return False
348
349 - def isblank(self):
350 """Used to see if this unit has no source or target string. 351 352 @note: This is probably used more to find translatable units, 353 and we might want to move in that direction rather and get rid of this. 354 """ 355 return not (self.source or self.target)
356
357 - def hasplural(self):
358 """Tells whether or not this specific unit has plural strings.""" 359 #TODO: Reconsider 360 return False
361
362 - def getsourcelanguage(self):
363 return getattr(self._store, "sourcelanguage", "en")
364
365 - def gettargetlanguage(self):
366 return getattr(self._store, "targetlanguage", None)
367
368 - def merge(self, otherunit, overwrite=False, comments=True, authoritative=False):
369 """Do basic format agnostic merging.""" 370 if not self.target or overwrite: 371 self.rich_target = otherunit.rich_target
372
373 - def unit_iter(self):
374 """Iterator that only returns this unit.""" 375 yield self
376
377 - def getunits(self):
378 """This unit in a list.""" 379 return [self]
380
381 - def buildfromunit(cls, unit):
382 """Build a native unit from a foreign unit, preserving as much 383 information as possible.""" 384 if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy): 385 return unit.copy() 386 newunit = cls(unit.source) 387 newunit.target = unit.target 388 newunit.markfuzzy(unit.isfuzzy()) 389 locations = unit.getlocations() 390 if locations: 391 newunit.addlocations(locations) 392 notes = unit.getnotes() 393 if notes: 394 newunit.addnote(notes) 395 return newunit
396 buildfromunit = classmethod(buildfromunit) 397 398 xid = property(lambda self: None, lambda self, value: None) 399 rid = property(lambda self: None, lambda self, value: None)
400 401
402 -class TranslationStore(object):
403 """Base class for stores for multiple translation units of type UnitClass.""" 404 405 UnitClass = TranslationUnit 406 """The class of units that will be instantiated and used by this class""" 407 Name = "Base translation store" 408 """The human usable name of this store type""" 409 Mimetypes = None 410 """A list of MIME types associated with this store type""" 411 Extensions = None 412 """A list of file extentions associated with this store type""" 413 _binary = False 414 """Indicates whether a file should be accessed as a binary file.""" 415 suggestions_in_format = False 416 """Indicates if format can store suggestions and alternative translation for a unit""" 417
418 - def __init__(self, unitclass=None):
419 """Constructs a blank TranslationStore.""" 420 self.units = [] 421 self.sourcelanguage = None 422 self.targetlanguage = None 423 if unitclass: 424 self.UnitClass = unitclass 425 super(TranslationStore, self).__init__()
426
427 - def getsourcelanguage(self):
428 """Gets the source language for this store""" 429 return self.sourcelanguage
430
431 - def setsourcelanguage(self, sourcelanguage):
432 """Sets the source language for this store""" 433 self.sourcelanguage = sourcelanguage
434
435 - def gettargetlanguage(self):
436 """Gets the target language for this store""" 437 return self.targetlanguage
438
439 - def settargetlanguage(self, targetlanguage):
440 """Sets the target language for this store""" 441 self.targetlanguage = targetlanguage
442
443 - def unit_iter(self):
444 """Iterator over all the units in this store.""" 445 for unit in self.units: 446 yield unit
447
448 - def getunits(self):
449 """Return a list of all units in this store.""" 450 return [unit for unit in self.unit_iter()]
451
452 - def addunit(self, unit):
453 """Appends the given unit to the object's list of units. 454 455 This method should always be used rather than trying to modify the 456 list manually. 457 458 @type unit: L{TranslationUnit} 459 @param unit: The unit that will be added. 460 """ 461 unit._store = self 462 self.units.append(unit)
463
464 - def addsourceunit(self, source):
465 """Adds and returns a new unit with the given source string. 466 467 @rtype: L{TranslationUnit} 468 """ 469 unit = self.UnitClass(source) 470 self.addunit(unit) 471 return unit
472
473 - def findid(self, id):
474 """find unit with matching id by checking id_index""" 475 self.require_index() 476 return self.id_index.get(id, None)
477
478 - def findunit(self, source):
479 """Finds the unit with the given source string. 480 481 @rtype: L{TranslationUnit} or None 482 """ 483 if len(getattr(self, "sourceindex", [])): 484 if source in self.sourceindex: 485 return self.sourceindex[source][0] 486 else: 487 for unit in self.units: 488 if unit.source == source: 489 return unit 490 return None
491 492
493 - def findunits(self, source):
494 """Finds the units with the given source string. 495 496 @rtype: L{TranslationUnit} or None 497 """ 498 if len(getattr(self, "sourceindex", [])): 499 if source in self.sourceindex: 500 return self.sourceindex[source] 501 else: 502 #FIXME: maybe we should generate index here instead since 503 #we'll scan all units anyway 504 result = [] 505 for unit in self.units: 506 if unit.source == source: 507 result.append(unit) 508 return result 509 return None
510
511 - def translate(self, source):
512 """Returns the translated string for a given source string. 513 514 @rtype: String or None 515 """ 516 unit = self.findunit(source) 517 if unit and unit.target: 518 return unit.target 519 else: 520 return None
521
522 - def remove_unit_from_index(self, unit):
523 """Remove a unit from source and locaton indexes""" 524 def remove_unit(source): 525 if source in self.sourceindex: 526 try: 527 self.sourceindex[source].remove(unit) 528 if len(self.sourceindex[source]) == 0: 529 del(self.sourceindex[source]) 530 except ValueError: 531 pass
532 533 if unit.hasplural(): 534 for source in unit.source.strings: 535 remove_unit(source) 536 else: 537 remove_unit(unit.source) 538 539 for location in unit.getlocations(): 540 if location in self.locationindex and self.locationindex[location] is not None \ 541 and self.locationindex[location] == unit: 542 del(self.locationindex[location])
543 544
545 - def add_unit_to_index(self, unit):
546 """Add a unit to source and location idexes""" 547 self.id_index[unit.getid()] = unit 548 549 def insert_unit(source): 550 if not source in self.sourceindex: 551 self.sourceindex[source] = [unit] 552 else: 553 self.sourceindex[source].append(unit)
554 555 if unit.hasplural(): 556 for source in unit.source.strings: 557 insert_unit(source) 558 else: 559 insert_unit(unit.source) 560 561 for location in unit.getlocations(): 562 if location in self.locationindex: 563 # if sources aren't unique, don't use them 564 #FIXME: maybe better store a list of units like sourceindex 565 self.locationindex[location] = None 566 else: 567 self.locationindex[location] = unit 568
569 - def makeindex(self):
570 """Indexes the items in this store. At least .sourceindex should be usefull.""" 571 self.locationindex = {} 572 self.sourceindex = {} 573 self.id_index = {} 574 for index, unit in enumerate(self.units): 575 unit.index = index 576 if unit.istranslatable(): 577 self.add_unit_to_index(unit)
578
579 - def require_index(self):
580 """make sure source index exists""" 581 if not hasattr(self, "sourceindex"): 582 self.makeindex()
583
584 - def getids(self):
585 """return a list of unit ids""" 586 self.require_index() 587 return self.id_index.keys()
588
589 - def __getstate__(self):
590 odict = self.__dict__.copy() 591 odict['fileobj'] = None 592 return odict
593
594 - def __setstate__(self, dict):
595 self.__dict__.update(dict) 596 if getattr(self, "filename", False): 597 self.fileobj = open(self.filename)
598
599 - def __str__(self):
600 """Converts to a string representation that can be parsed back using L{parsestring()}.""" 601 # We can't pickle fileobj if it is there, so let's hide it for a while. 602 fileobj = getattr(self, "fileobj", None) 603 self.fileobj = None 604 dump = pickle.dumps(self) 605 self.fileobj = fileobj 606 return dump
607
608 - def isempty(self):
609 """Returns True if the object doesn't contain any translation units.""" 610 if len(self.units) == 0: 611 return True 612 for unit in self.units: 613 if unit.istranslatable(): 614 return False 615 return True
616
617 - def _assignname(self):
618 """Tries to work out what the name of the filesystem file is and 619 assigns it to .filename.""" 620 fileobj = getattr(self, "fileobj", None) 621 if fileobj: 622 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 623 if filename: 624 self.filename = filename
625
626 - def parsestring(cls, storestring):
627 """Converts the string representation back to an object.""" 628 newstore = cls() 629 if storestring: 630 newstore.parse(storestring) 631 return newstore
632 parsestring = classmethod(parsestring) 633
634 - def parse(self, data):
635 """parser to process the given source string""" 636 self.units = pickle.loads(data).units
637
638 - def savefile(self, storefile):
639 """Writes the string representation to the given file (or filename).""" 640 if isinstance(storefile, basestring): 641 mode = 'w' 642 if self._binary: 643 mode = 'wb' 644 storefile = open(storefile, mode) 645 self.fileobj = storefile 646 self._assignname() 647 storestring = str(self) 648 storefile.write(storestring) 649 storefile.close()
650
651 - def save(self):
652 """Save to the file that data was originally read from, if available.""" 653 fileobj = getattr(self, "fileobj", None) 654 mode = 'w' 655 if self._binary: 656 mode = 'wb' 657 if not fileobj: 658 filename = getattr(self, "filename", None) 659 if filename: 660 fileobj = file(filename, mode) 661 else: 662 fileobj.close() 663 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 664 if not filename: 665 raise ValueError("No file or filename to save to") 666 fileobj = fileobj.__class__(filename, mode) 667 self.savefile(fileobj)
668
669 - def parsefile(cls, storefile):
670 """Reads the given file (or opens the given filename) and parses back to an object.""" 671 mode = 'r' 672 if cls._binary: 673 mode = 'rb' 674 if isinstance(storefile, basestring): 675 storefile = open(storefile, mode) 676 mode = getattr(storefile, "mode", mode) 677 #For some reason GzipFile returns 1, so we have to test for that here 678 if mode == 1 or "r" in mode: 679 storestring = storefile.read() 680 storefile.close() 681 else: 682 storestring = "" 683 newstore = cls.parsestring(storestring) 684 newstore.fileobj = storefile 685 newstore._assignname() 686 return newstore
687 parsefile = classmethod(parsefile) 688