Package translate :: Package storage :: Module xpi
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.xpi

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  #  
  4  # Copyright 2004, 2005 Zuza Software Foundation 
  5  #  
  6  # This file is part of translate. 
  7  # 
  8  # translate is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  #  
 13  # translate is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with translate; if not, write to the Free Software 
 20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 21   
 22  """module for accessing mozilla xpi packages""" 
 23   
 24  from __future__ import generators 
 25  import zipfile 
 26  import os.path 
 27  from translate import __version__ 
 28  import StringIO 
 29  import re 
 30   
 31  # we have some enhancements to zipfile in a file called zipfileext 
 32  # hopefully they will be included in a future version of python 
 33  from translate.misc import zipfileext 
 34  ZipFileBase = zipfileext.ZipFileExt 
 35   
 36  from translate.misc import wStringIO 
 37  # this is a fix to the StringIO in Python 2.3.3 
 38  # submitted as patch 951915 on sourceforge 
39 -class FixedStringIO(wStringIO.StringIO):
40 - def truncate(self, size=None):
41 StringIO.StringIO.truncate(self, size) 42 self.len = len(self.buf)
43 44 NamedStringInput = wStringIO.StringIO 45 NamedStringOutput = wStringIO.StringIO 46
47 -def _commonprefix(itemlist):
48 def cp(a, b): 49 l = min(len(a), len(b)) 50 for n in range(l): 51 if a[n] != b[n]: return a[:n] 52 return a[:l]
53 if itemlist: 54 return reduce(cp, itemlist) 55 else: 56 return '' 57
58 -def rememberchanged(self, method):
59 def changed(*args, **kwargs): 60 self.changed = True 61 method(*args, **kwargs)
62 return changed 63
64 -class CatchPotentialOutput(NamedStringInput, object):
65 """catches output if there has been, before closing"""
66 - def __init__(self, contents, onclose):
67 """Set up the output stream, and remember a method to call on closing""" 68 NamedStringInput.__init__(self, contents) 69 self.onclose = onclose 70 self.changed = False 71 s = super(CatchPotentialOutput, self) 72 self.write = rememberchanged(self, s.write) 73 self.writelines = rememberchanged(self, s.writelines) 74 self.truncate = rememberchanged(self, s.truncate)
75
76 - def close(self):
77 """wrap the underlying close method, to pass the value to onclose before it goes""" 78 if self.changed: 79 value = self.getvalue() 80 self.onclose(value) 81 NamedStringInput.close(self)
82
83 - def flush(self):
84 """zip files call flush, not close, on file-like objects""" 85 value = self.getvalue() 86 self.onclose(value) 87 NamedStringInput.flush(self)
88
89 - def slam(self):
90 """use this method to force the closing of the stream if it isn't closed yet""" 91 if not self.closed: 92 self.close()
93
94 -class ZipFileCatcher(ZipFileBase, object):
95 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
96 - def __init__(self, *args, **kwargs):
97 """initialize the ZipFileCatcher""" 98 # storing oldclose as attribute, since if close is called from __del__ it has no access to external variables 99 self.oldclose = super(ZipFileCatcher, self).close 100 super(ZipFileCatcher, self).__init__(*args, **kwargs)
101
102 - def addcatcher(self, pendingsave):
103 """remember to call the given method before closing""" 104 if hasattr(self, "pendingsaves"): 105 if not pendingsave in self.pendingsaves: 106 self.pendingsaves.append(pendingsave) 107 else: 108 self.pendingsaves = [pendingsave]
109
110 - def close(self):
111 """close the stream, remembering to call any addcatcher methods first""" 112 if hasattr(self, "pendingsaves"): 113 for pendingsave in self.pendingsaves: 114 pendingsave() 115 # if close is called from __del__, it somehow can't see ZipFileCatcher, so we've cached oldclose... 116 if ZipFileCatcher is None: 117 self.oldclose() 118 else: 119 super(ZipFileCatcher, self).close()
120
121 - def overwritestr(self, zinfo_or_arcname, bytes):
122 """writes the string into the archive, overwriting the file if it exists...""" 123 if isinstance(zinfo_or_arcname, zipfile.ZipInfo): 124 filename = zinfo_or_arcname.filename 125 else: 126 filename = zinfo_or_arcname 127 if filename in self.NameToInfo: 128 self.delete(filename) 129 self.writestr(zinfo_or_arcname, bytes) 130 self.writeendrec()
131
132 -class XpiFile(ZipFileCatcher):
133 - def __init__(self, *args, **kwargs):
134 """sets up the xpi file""" 135 self.includenonloc = kwargs.get("includenonloc", True) 136 if "includenonloc" in kwargs: 137 del kwargs["includenonloc"] 138 if "compression" not in kwargs: 139 kwargs["compression"] = zipfile.ZIP_DEFLATED 140 self.locale = kwargs.pop("locale", None) 141 self.region = kwargs.pop("region", None) 142 super(XpiFile, self).__init__(*args, **kwargs) 143 self.jarfiles = {} 144 self.findlangreg() 145 self.jarprefixes = self.findjarprefixes() 146 self.reverseprefixes = dict([ 147 (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix]) 148 self.reverseprefixes["package/"] = None
149
150 - def iterjars(self):
151 """iterate through the jar files in the xpi as ZipFile objects""" 152 for filename in self.namelist(): 153 if filename.lower().endswith('.jar'): 154 if filename not in self.jarfiles: 155 jarstream = self.openinputstream(None, filename) 156 jarfile = ZipFileCatcher(jarstream, mode=self.mode) 157 self.jarfiles[filename] = jarfile 158 else: 159 jarfile = self.jarfiles[filename] 160 yield filename, jarfile
161
162 - def islocfile(self, filename):
163 """returns whether the given file is needed for localization (basically .dtd and .properties)""" 164 base, ext = os.path.splitext(filename) 165 return ext in (os.extsep + "dtd", os.extsep + "properties")
166
167 - def findlangreg(self):
168 """finds the common prefix of all the files stored in the jar files""" 169 dirstructure = {} 170 locale = self.locale 171 region = self.region 172 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$") 173 regionmatch = re.compile("^[a-zA-Z]{2,3}$") 174 # exclude en-mac, en-win, en-unix for seamonkey 175 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$") 176 for jarfilename, jarfile in self.iterjars(): 177 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1) 178 if localematch.match(jarname) and not osmatch.match(jarname): 179 if locale is None: 180 locale = jarname 181 elif locale != jarname: 182 locale = 0 183 elif regionmatch.match(jarname): 184 if region is None: 185 region = jarname 186 elif region != jarname: 187 region = 0 188 for filename in jarfile.namelist(): 189 if filename.endswith('/'): continue 190 if not self.islocfile(filename) and not self.includenonloc: continue 191 parts = filename.split('/')[:-1] 192 treepoint = dirstructure 193 for partnum in range(len(parts)): 194 part = parts[partnum] 195 if part in treepoint: 196 treepoint = treepoint[part] 197 else: 198 treepoint[part] = {} 199 treepoint = treepoint[part] 200 localeentries = {} 201 if 'locale' in dirstructure: 202 for dirname in dirstructure['locale']: 203 localeentries[dirname] = 1 204 if localematch.match(dirname) and not osmatch.match(dirname): 205 if locale is None: 206 locale = dirname 207 elif locale != dirname: 208 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0" 209 locale = 0 210 elif regionmatch.match(dirname): 211 if region is None: 212 region = dirname 213 elif region != dirname: 214 region = 0 215 if locale and locale in localeentries: 216 del localeentries[locale] 217 if region and region in localeentries: 218 del localeentries[region] 219 if locale and not region: 220 if "-" in locale: 221 region = locale.split("-", 1)[1] 222 else: 223 region = "" 224 self.setlangreg(locale, region)
225
226 - def setlangreg(self, locale, region):
227 """set the locale and region of this xpi""" 228 if locale == 0 or locale is None: 229 raise ValueError("unable to determine locale") 230 self.locale = locale 231 self.region = region 232 self.dirmap = {} 233 if self.locale is not None: 234 self.dirmap[('locale', self.locale)] = ('lang-reg',) 235 if self.region: 236 self.dirmap[('locale', self.region)] = ('reg',)
237
238 - def findjarprefixes(self):
239 """checks the uniqueness of the jar files contents""" 240 uniquenames = {} 241 jarprefixes = {} 242 for jarfilename, jarfile in self.iterjars(): 243 jarprefixes[jarfilename] = "" 244 for filename in jarfile.namelist(): 245 if filename.endswith('/'): continue 246 if filename in uniquenames: 247 jarprefixes[jarfilename] = True 248 jarprefixes[uniquenames[filename]] = True 249 else: 250 uniquenames[filename] = jarfilename 251 for jarfilename, hasconflicts in jarprefixes.items(): 252 if hasconflicts: 253 shortjarfilename = os.path.split(jarfilename)[1] 254 shortjarfilename = os.path.splitext(shortjarfilename)[0] 255 jarprefixes[jarfilename] = shortjarfilename+'/' 256 # this is a clever trick that will e.g. remove zu- from zu-win, zu-mac, zu-unix 257 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix]) 258 if commonjarprefix: 259 for jarfilename, prefix in jarprefixes.items(): 260 if prefix: 261 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1) 262 return jarprefixes
263
264 - def ziptoospath(self, zippath):
265 """converts a zipfile filepath to an os-style filepath""" 266 return os.path.join(*zippath.split('/'))
267
268 - def ostozippath(self, ospath):
269 """converts an os-style filepath to a zipfile filepath""" 270 return '/'.join(ospath.split(os.sep))
271
272 - def mapfilename(self, filename):
273 """uses a map to simplify the directory structure""" 274 parts = tuple(filename.split('/')) 275 possiblematch = None 276 for prefix, mapto in self.dirmap.iteritems(): 277 if parts[:len(prefix)] == prefix: 278 if possiblematch is None or len(possiblematch[0]) < len(prefix): 279 possiblematch = prefix, mapto 280 if possiblematch is not None: 281 prefix, mapto = possiblematch 282 mapped = mapto + parts[len(prefix):] 283 return '/'.join(mapped) 284 return filename
285
286 - def mapxpifilename(self, filename):
287 """uses a map to rename files that occur straight in the xpi""" 288 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"): 289 return 'bin/chrome/lang-reg.manifest' 290 return filename
291
292 - def reversemapfile(self, filename):
293 """unmaps the filename...""" 294 possiblematch = None 295 parts = tuple(filename.split('/')) 296 for prefix, mapto in self.dirmap.iteritems(): 297 if parts[:len(mapto)] == mapto: 298 if possiblematch is None or len(possiblematch[0]) < len(mapto): 299 possiblematch = (mapto, prefix) 300 if possiblematch is None: 301 return filename 302 mapto, prefix = possiblematch 303 reversemapped = prefix + parts[len(mapto):] 304 return '/'.join(reversemapped)
305
306 - def reversemapxpifilename(self, filename):
307 """uses a map to rename files that occur straight in the xpi""" 308 if filename == 'bin/chrome/lang-reg.manifest': 309 if self.locale: 310 return '/'.join(('bin', 'chrome', self.locale + '.manifest')) 311 else: 312 for otherfilename in self.namelist(): 313 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"): 314 return otherfilename 315 return filename
316
317 - def jartoospath(self, jarfilename, filename):
318 """converts a filename from within a jarfile to an os-style filepath""" 319 if jarfilename: 320 jarprefix = self.jarprefixes[jarfilename] 321 return self.ziptoospath(jarprefix+self.mapfilename(filename)) 322 else: 323 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
324
325 - def ostojarpath(self, ospath):
326 """converts an extracted os-style filepath to a jarfilename and filename""" 327 zipparts = ospath.split(os.sep) 328 prefix = zipparts[0] + '/' 329 if prefix in self.reverseprefixes: 330 jarfilename = self.reverseprefixes[prefix] 331 filename = self.reversemapfile('/'.join(zipparts[1:])) 332 if jarfilename is None: 333 filename = self.reversemapxpifilename(filename) 334 return jarfilename, filename 335 else: 336 filename = self.ostozippath(ospath) 337 if filename in self.namelist(): 338 return None, filename 339 filename = self.reversemapfile('/'.join(zipparts)) 340 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix] 341 for jarfilename in possiblejarfilenames: 342 jarfile = self.jarfiles[jarfilename] 343 if filename in jarfile.namelist(): 344 return jarfilename, filename 345 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
346
347 - def jarfileexists(self, jarfilename, filename):
348 """checks whether the given file exists inside the xpi""" 349 if jarfilename is None: 350 return filename in self.namelist() 351 else: 352 jarfile = self.jarfiles[jarfilename] 353 return filename in jarfile.namelist()
354
355 - def ospathexists(self, ospath):
356 """checks whether the given file exists inside the xpi""" 357 jarfilename, filename = self.ostojarpath(ospath) 358 if jarfilename is None: 359 return filename in self.namelist() 360 else: 361 jarfile = self.jarfiles[jarfilename] 362 return filename in jarfile.namelist()
363
364 - def openinputstream(self, jarfilename, filename):
365 """opens a file (possibly inside a jarfile as a StringIO""" 366 if jarfilename is None: 367 contents = self.read(filename) 368 def onclose(contents): 369 if contents != self.read(filename): 370 self.overwritestr(filename, contents)
371 inputstream = CatchPotentialOutput(contents, onclose) 372 self.addcatcher(inputstream.slam) 373 else: 374 jarfile = self.jarfiles[jarfilename] 375 contents = jarfile.read(filename) 376 inputstream = NamedStringInput(contents) 377 inputstream.name = self.jartoospath(jarfilename, filename) 378 if hasattr(self.fp, 'name'): 379 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name) 380 return inputstream
381
382 - def openoutputstream(self, jarfilename, filename):
383 """opens a file for writing (possibly inside a jarfile as a StringIO""" 384 if jarfilename is None: 385 def onclose(contents): 386 self.overwritestr(filename, contents)
387 else: 388 if jarfilename in self.jarfiles: 389 jarfile = self.jarfiles[jarfilename] 390 else: 391 jarstream = self.openoutputstream(None, jarfilename) 392 jarfile = ZipFileCatcher(jarstream, "w") 393 self.jarfiles[jarfilename] = jarfile 394 self.addcatcher(jarstream.slam) 395 def onclose(contents): 396 jarfile.overwritestr(filename, contents) 397 outputstream = wStringIO.CatchStringOutput(onclose) 398 outputstream.name = "%s %s" % (jarfilename, filename) 399 if jarfilename is None: 400 self.addcatcher(outputstream.slam) 401 else: 402 jarfile.addcatcher(outputstream.slam) 403 return outputstream 404
405 - def close(self):
406 """Close the file, and for mode "w" and "a" write the ending records.""" 407 for jarfile in self.jarfiles.itervalues(): 408 jarfile.close() 409 super(XpiFile, self).close()
410
411 - def testzip(self):
412 """test the xpi zipfile and all enclosed jar files...""" 413 for jarfile in self.jarfiles.itervalues(): 414 jarfile.testzip() 415 super(XpiFile, self).testzip()
416
417 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
418 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream""" 419 jarfile = self.jarfiles[origjarfilename] 420 origlang = self.locale[:self.locale.find("-")] 421 if newregion: 422 newlocale = "%s-%s" % (newlang, newregion) 423 else: 424 newlocale = newlang 425 for filename in jarfile.namelist(): 426 filenameparts = filename.split("/") 427 for i in range(len(filenameparts)): 428 part = filenameparts[i] 429 if part == origlang: 430 filenameparts[i] = newlang 431 elif part == self.locale: 432 filenameparts[i] = newlocale 433 elif part == self.region: 434 filenameparts[i] = newregion 435 newfilename = '/'.join(filenameparts) 436 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename) 437 fileinputstream = self.openinputstream(origjarfilename, filename) 438 fileoutputstream.write(fileinputstream.read()) 439 fileinputstream.close() 440 fileoutputstream.close()
441
442 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
443 """Create a new .xpi file with the same contents as this one...""" 444 other = XpiFile(newfilename, "w", locale=newlang, region=newregion) 445 origlang = self.locale[:self.locale.find("-")] 446 # TODO: check if this language replacement code is still neccessary 447 if newlang is None: 448 newlang = origlang 449 if newregion is None: 450 newregion = self.region 451 if newregion: 452 newlocale = "%s-%s" % (newlang, newregion) 453 else: 454 newlocale = newlang 455 for filename in self.namelist(): 456 filenameparts = filename.split('/') 457 basename = filenameparts[-1] 458 if basename.startswith(self.locale): 459 newbasename = basename.replace(self.locale, newlocale) 460 elif basename.startswith(origlang): 461 newbasename = basename.replace(origlang, newlang) 462 elif basename.startswith(self.region): 463 newbasename = basename.replace(self.region, newregion) 464 else: 465 newbasename = basename 466 if newbasename != basename: 467 filenameparts[-1] = newbasename 468 renamefilename = "/".join(filenameparts) 469 print "cloning", filename, "and renaming to", renamefilename 470 else: 471 print "cloning", filename 472 renamefilename = filename 473 if filename.lower().endswith(".jar"): 474 self.restructurejar(filename, renamefilename, other, newlang, newregion) 475 else: 476 inputstream = self.openinputstream(None, filename) 477 outputstream = other.openoutputstream(None, renamefilename) 478 outputstream.write(inputstream.read()) 479 inputstream.close() 480 outputstream.close() 481 other.close() 482 if newmode is None: newmode = self.mode 483 if newmode == "w": newmode = "a" 484 other = XpiFile(newfilename, newmode) 485 other.setlangreg(newlocale, newregion) 486 return other
487
488 - def iterextractnames(self, includenonjars=False, includedirs=False):
489 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary""" 490 if includenonjars: 491 for filename in self.namelist(): 492 if filename.endswith('/') and not includedirs: continue 493 if not self.islocfile(filename) and not self.includenonloc: continue 494 if not filename.lower().endswith(".jar"): 495 yield self.jartoospath(None, filename) 496 for jarfilename, jarfile in self.iterjars(): 497 for filename in jarfile.namelist(): 498 if filename.endswith('/'): 499 if not includedirs: continue 500 if not self.islocfile(filename) and not self.includenonloc: continue 501 yield self.jartoospath(jarfilename, filename)
502 503 # the following methods are required by translate.convert.ArchiveConvertOptionParser #
504 - def __iter__(self):
505 """iterates through all the files. this is the method use by the converters""" 506 for inputpath in self.iterextractnames(includenonjars=True): 507 yield inputpath
508
509 - def __contains__(self, fullpath):
510 """returns whether the given pathname exists in the archive""" 511 try: 512 jarfilename, filename = self.ostojarpath(fullpath) 513 except IndexError: 514 return False 515 return self.jarfileexists(jarfilename, filename)
516
517 - def openinputfile(self, fullpath):
518 """opens an input file given the full pathname""" 519 jarfilename, filename = self.ostojarpath(fullpath) 520 return self.openinputstream(jarfilename, filename)
521
522 - def openoutputfile(self, fullpath):
523 """opens an output file given the full pathname""" 524 try: 525 jarfilename, filename = self.ostojarpath(fullpath) 526 except IndexError: 527 return None 528 return self.openoutputstream(jarfilename, filename)
529 530 if __name__ == '__main__': 531 import optparse 532 optparser = optparse.OptionParser(version="%prog "+__version__.sver) 533 optparser.usage = "%prog [-l|-x] [options] file.xpi" 534 optparser.add_option("-l", "--list", help="list files", \ 535 action="store_true", dest="listfiles", default=False) 536 optparser.add_option("-p", "--prefix", help="show common prefix", \ 537 action="store_true", dest="showprefix", default=False) 538 optparser.add_option("-x", "--extract", help="extract files", \ 539 action="store_true", dest="extractfiles", default=False) 540 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \ 541 default=".", metavar="EXTRACTDIR") 542 (options, args) = optparser.parse_args() 543 if len(args) < 1: 544 optparser.error("need at least one argument") 545 xpifile = XpiFile(args[0]) 546 if options.showprefix: 547 for prefix, mapto in xpifile.dirmap.iteritems(): 548 print "/".join(prefix), "->", "/".join(mapto) 549 if options.listfiles: 550 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True): 551 print name #, xpifile.ostojarpath(name) 552 if options.extractfiles: 553 if options.extractdir and not os.path.isdir(options.extractdir): 554 os.mkdir(options.extractdir) 555 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False): 556 abspath = os.path.join(options.extractdir, name) 557 # check neccessary directories exist - this way we don't create empty directories 558 currentpath = options.extractdir 559 subparts = os.path.dirname(name).split(os.sep) 560 for part in subparts: 561 currentpath = os.path.join(currentpath, part) 562 if not os.path.isdir(currentpath): 563 os.mkdir(currentpath) 564 outputstream = open(abspath, 'w') 565 jarfilename, filename = xpifile.ostojarpath(name) 566 inputstream = xpifile.openinputstream(jarfilename, filename) 567 outputstream.write(inputstream.read()) 568 outputstream.close() 569