1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """module for accessing mozilla xpi packages"""
23
24 from __future__ import generators
25 import zipfile
26 import os.path
27 from translate import __version__
28 import StringIO
29 import re
30
31
32
33 from translate.misc import zipfileext
34 ZipFileBase = zipfileext.ZipFileExt
35
36 from translate.misc import wStringIO
37
38
43
44 NamedStringInput = wStringIO.StringIO
45 NamedStringOutput = wStringIO.StringIO
46
48 def cp(a, b):
49 l = min(len(a), len(b))
50 for n in range(l):
51 if a[n] != b[n]: return a[:n]
52 return a[:l]
53 if itemlist:
54 return reduce(cp, itemlist)
55 else:
56 return ''
57
59 def changed(*args, **kwargs):
60 self.changed = True
61 method(*args, **kwargs)
62 return changed
63
65 """catches output if there has been, before closing"""
75
77 """wrap the underlying close method, to pass the value to onclose before it goes"""
78 if self.changed:
79 value = self.getvalue()
80 self.onclose(value)
81 NamedStringInput.close(self)
82
84 """zip files call flush, not close, on file-like objects"""
85 value = self.getvalue()
86 self.onclose(value)
87 NamedStringInput.flush(self)
88
90 """use this method to force the closing of the stream if it isn't closed yet"""
91 if not self.closed:
92 self.close()
93
95 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
101
103 """remember to call the given method before closing"""
104 if hasattr(self, "pendingsaves"):
105 if not pendingsave in self.pendingsaves:
106 self.pendingsaves.append(pendingsave)
107 else:
108 self.pendingsaves = [pendingsave]
109
111 """close the stream, remembering to call any addcatcher methods first"""
112 if hasattr(self, "pendingsaves"):
113 for pendingsave in self.pendingsaves:
114 pendingsave()
115
116 if ZipFileCatcher is None:
117 self.oldclose()
118 else:
119 super(ZipFileCatcher, self).close()
120
122 """writes the string into the archive, overwriting the file if it exists..."""
123 if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
124 filename = zinfo_or_arcname.filename
125 else:
126 filename = zinfo_or_arcname
127 if filename in self.NameToInfo:
128 self.delete(filename)
129 self.writestr(zinfo_or_arcname, bytes)
130 self.writeendrec()
131
134 """sets up the xpi file"""
135 self.includenonloc = kwargs.get("includenonloc", True)
136 if "includenonloc" in kwargs:
137 del kwargs["includenonloc"]
138 if "compression" not in kwargs:
139 kwargs["compression"] = zipfile.ZIP_DEFLATED
140 self.locale = kwargs.pop("locale", None)
141 self.region = kwargs.pop("region", None)
142 super(XpiFile, self).__init__(*args, **kwargs)
143 self.jarfiles = {}
144 self.findlangreg()
145 self.jarprefixes = self.findjarprefixes()
146 self.reverseprefixes = dict([
147 (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix])
148 self.reverseprefixes["package/"] = None
149
151 """iterate through the jar files in the xpi as ZipFile objects"""
152 for filename in self.namelist():
153 if filename.lower().endswith('.jar'):
154 if filename not in self.jarfiles:
155 jarstream = self.openinputstream(None, filename)
156 jarfile = ZipFileCatcher(jarstream, mode=self.mode)
157 self.jarfiles[filename] = jarfile
158 else:
159 jarfile = self.jarfiles[filename]
160 yield filename, jarfile
161
163 """returns whether the given file is needed for localization (basically .dtd and .properties)"""
164 base, ext = os.path.splitext(filename)
165 return ext in (os.extsep + "dtd", os.extsep + "properties")
166
168 """finds the common prefix of all the files stored in the jar files"""
169 dirstructure = {}
170 locale = self.locale
171 region = self.region
172 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$")
173 regionmatch = re.compile("^[a-zA-Z]{2,3}$")
174
175 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$")
176 for jarfilename, jarfile in self.iterjars():
177 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1)
178 if localematch.match(jarname) and not osmatch.match(jarname):
179 if locale is None:
180 locale = jarname
181 elif locale != jarname:
182 locale = 0
183 elif regionmatch.match(jarname):
184 if region is None:
185 region = jarname
186 elif region != jarname:
187 region = 0
188 for filename in jarfile.namelist():
189 if filename.endswith('/'): continue
190 if not self.islocfile(filename) and not self.includenonloc: continue
191 parts = filename.split('/')[:-1]
192 treepoint = dirstructure
193 for partnum in range(len(parts)):
194 part = parts[partnum]
195 if part in treepoint:
196 treepoint = treepoint[part]
197 else:
198 treepoint[part] = {}
199 treepoint = treepoint[part]
200 localeentries = {}
201 if 'locale' in dirstructure:
202 for dirname in dirstructure['locale']:
203 localeentries[dirname] = 1
204 if localematch.match(dirname) and not osmatch.match(dirname):
205 if locale is None:
206 locale = dirname
207 elif locale != dirname:
208 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0"
209 locale = 0
210 elif regionmatch.match(dirname):
211 if region is None:
212 region = dirname
213 elif region != dirname:
214 region = 0
215 if locale and locale in localeentries:
216 del localeentries[locale]
217 if region and region in localeentries:
218 del localeentries[region]
219 if locale and not region:
220 if "-" in locale:
221 region = locale.split("-", 1)[1]
222 else:
223 region = ""
224 self.setlangreg(locale, region)
225
227 """set the locale and region of this xpi"""
228 if locale == 0 or locale is None:
229 raise ValueError("unable to determine locale")
230 self.locale = locale
231 self.region = region
232 self.dirmap = {}
233 if self.locale is not None:
234 self.dirmap[('locale', self.locale)] = ('lang-reg',)
235 if self.region:
236 self.dirmap[('locale', self.region)] = ('reg',)
237
239 """checks the uniqueness of the jar files contents"""
240 uniquenames = {}
241 jarprefixes = {}
242 for jarfilename, jarfile in self.iterjars():
243 jarprefixes[jarfilename] = ""
244 for filename in jarfile.namelist():
245 if filename.endswith('/'): continue
246 if filename in uniquenames:
247 jarprefixes[jarfilename] = True
248 jarprefixes[uniquenames[filename]] = True
249 else:
250 uniquenames[filename] = jarfilename
251 for jarfilename, hasconflicts in jarprefixes.items():
252 if hasconflicts:
253 shortjarfilename = os.path.split(jarfilename)[1]
254 shortjarfilename = os.path.splitext(shortjarfilename)[0]
255 jarprefixes[jarfilename] = shortjarfilename+'/'
256
257 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix])
258 if commonjarprefix:
259 for jarfilename, prefix in jarprefixes.items():
260 if prefix:
261 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1)
262 return jarprefixes
263
265 """converts a zipfile filepath to an os-style filepath"""
266 return os.path.join(*zippath.split('/'))
267
269 """converts an os-style filepath to a zipfile filepath"""
270 return '/'.join(ospath.split(os.sep))
271
273 """uses a map to simplify the directory structure"""
274 parts = tuple(filename.split('/'))
275 possiblematch = None
276 for prefix, mapto in self.dirmap.iteritems():
277 if parts[:len(prefix)] == prefix:
278 if possiblematch is None or len(possiblematch[0]) < len(prefix):
279 possiblematch = prefix, mapto
280 if possiblematch is not None:
281 prefix, mapto = possiblematch
282 mapped = mapto + parts[len(prefix):]
283 return '/'.join(mapped)
284 return filename
285
287 """uses a map to rename files that occur straight in the xpi"""
288 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"):
289 return 'bin/chrome/lang-reg.manifest'
290 return filename
291
293 """unmaps the filename..."""
294 possiblematch = None
295 parts = tuple(filename.split('/'))
296 for prefix, mapto in self.dirmap.iteritems():
297 if parts[:len(mapto)] == mapto:
298 if possiblematch is None or len(possiblematch[0]) < len(mapto):
299 possiblematch = (mapto, prefix)
300 if possiblematch is None:
301 return filename
302 mapto, prefix = possiblematch
303 reversemapped = prefix + parts[len(mapto):]
304 return '/'.join(reversemapped)
305
307 """uses a map to rename files that occur straight in the xpi"""
308 if filename == 'bin/chrome/lang-reg.manifest':
309 if self.locale:
310 return '/'.join(('bin', 'chrome', self.locale + '.manifest'))
311 else:
312 for otherfilename in self.namelist():
313 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"):
314 return otherfilename
315 return filename
316
318 """converts a filename from within a jarfile to an os-style filepath"""
319 if jarfilename:
320 jarprefix = self.jarprefixes[jarfilename]
321 return self.ziptoospath(jarprefix+self.mapfilename(filename))
322 else:
323 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
324
326 """converts an extracted os-style filepath to a jarfilename and filename"""
327 zipparts = ospath.split(os.sep)
328 prefix = zipparts[0] + '/'
329 if prefix in self.reverseprefixes:
330 jarfilename = self.reverseprefixes[prefix]
331 filename = self.reversemapfile('/'.join(zipparts[1:]))
332 if jarfilename is None:
333 filename = self.reversemapxpifilename(filename)
334 return jarfilename, filename
335 else:
336 filename = self.ostozippath(ospath)
337 if filename in self.namelist():
338 return None, filename
339 filename = self.reversemapfile('/'.join(zipparts))
340 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix]
341 for jarfilename in possiblejarfilenames:
342 jarfile = self.jarfiles[jarfilename]
343 if filename in jarfile.namelist():
344 return jarfilename, filename
345 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
346
348 """checks whether the given file exists inside the xpi"""
349 if jarfilename is None:
350 return filename in self.namelist()
351 else:
352 jarfile = self.jarfiles[jarfilename]
353 return filename in jarfile.namelist()
354
356 """checks whether the given file exists inside the xpi"""
357 jarfilename, filename = self.ostojarpath(ospath)
358 if jarfilename is None:
359 return filename in self.namelist()
360 else:
361 jarfile = self.jarfiles[jarfilename]
362 return filename in jarfile.namelist()
363
371 inputstream = CatchPotentialOutput(contents, onclose)
372 self.addcatcher(inputstream.slam)
373 else:
374 jarfile = self.jarfiles[jarfilename]
375 contents = jarfile.read(filename)
376 inputstream = NamedStringInput(contents)
377 inputstream.name = self.jartoospath(jarfilename, filename)
378 if hasattr(self.fp, 'name'):
379 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name)
380 return inputstream
381
383 """opens a file for writing (possibly inside a jarfile as a StringIO"""
384 if jarfilename is None:
385 def onclose(contents):
386 self.overwritestr(filename, contents)
387 else:
388 if jarfilename in self.jarfiles:
389 jarfile = self.jarfiles[jarfilename]
390 else:
391 jarstream = self.openoutputstream(None, jarfilename)
392 jarfile = ZipFileCatcher(jarstream, "w")
393 self.jarfiles[jarfilename] = jarfile
394 self.addcatcher(jarstream.slam)
395 def onclose(contents):
396 jarfile.overwritestr(filename, contents)
397 outputstream = wStringIO.CatchStringOutput(onclose)
398 outputstream.name = "%s %s" % (jarfilename, filename)
399 if jarfilename is None:
400 self.addcatcher(outputstream.slam)
401 else:
402 jarfile.addcatcher(outputstream.slam)
403 return outputstream
404
406 """Close the file, and for mode "w" and "a" write the ending records."""
407 for jarfile in self.jarfiles.itervalues():
408 jarfile.close()
409 super(XpiFile, self).close()
410
412 """test the xpi zipfile and all enclosed jar files..."""
413 for jarfile in self.jarfiles.itervalues():
414 jarfile.testzip()
415 super(XpiFile, self).testzip()
416
417 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
418 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream"""
419 jarfile = self.jarfiles[origjarfilename]
420 origlang = self.locale[:self.locale.find("-")]
421 if newregion:
422 newlocale = "%s-%s" % (newlang, newregion)
423 else:
424 newlocale = newlang
425 for filename in jarfile.namelist():
426 filenameparts = filename.split("/")
427 for i in range(len(filenameparts)):
428 part = filenameparts[i]
429 if part == origlang:
430 filenameparts[i] = newlang
431 elif part == self.locale:
432 filenameparts[i] = newlocale
433 elif part == self.region:
434 filenameparts[i] = newregion
435 newfilename = '/'.join(filenameparts)
436 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename)
437 fileinputstream = self.openinputstream(origjarfilename, filename)
438 fileoutputstream.write(fileinputstream.read())
439 fileinputstream.close()
440 fileoutputstream.close()
441
442 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
443 """Create a new .xpi file with the same contents as this one..."""
444 other = XpiFile(newfilename, "w", locale=newlang, region=newregion)
445 origlang = self.locale[:self.locale.find("-")]
446
447 if newlang is None:
448 newlang = origlang
449 if newregion is None:
450 newregion = self.region
451 if newregion:
452 newlocale = "%s-%s" % (newlang, newregion)
453 else:
454 newlocale = newlang
455 for filename in self.namelist():
456 filenameparts = filename.split('/')
457 basename = filenameparts[-1]
458 if basename.startswith(self.locale):
459 newbasename = basename.replace(self.locale, newlocale)
460 elif basename.startswith(origlang):
461 newbasename = basename.replace(origlang, newlang)
462 elif basename.startswith(self.region):
463 newbasename = basename.replace(self.region, newregion)
464 else:
465 newbasename = basename
466 if newbasename != basename:
467 filenameparts[-1] = newbasename
468 renamefilename = "/".join(filenameparts)
469 print "cloning", filename, "and renaming to", renamefilename
470 else:
471 print "cloning", filename
472 renamefilename = filename
473 if filename.lower().endswith(".jar"):
474 self.restructurejar(filename, renamefilename, other, newlang, newregion)
475 else:
476 inputstream = self.openinputstream(None, filename)
477 outputstream = other.openoutputstream(None, renamefilename)
478 outputstream.write(inputstream.read())
479 inputstream.close()
480 outputstream.close()
481 other.close()
482 if newmode is None: newmode = self.mode
483 if newmode == "w": newmode = "a"
484 other = XpiFile(newfilename, newmode)
485 other.setlangreg(newlocale, newregion)
486 return other
487
489 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary"""
490 if includenonjars:
491 for filename in self.namelist():
492 if filename.endswith('/') and not includedirs: continue
493 if not self.islocfile(filename) and not self.includenonloc: continue
494 if not filename.lower().endswith(".jar"):
495 yield self.jartoospath(None, filename)
496 for jarfilename, jarfile in self.iterjars():
497 for filename in jarfile.namelist():
498 if filename.endswith('/'):
499 if not includedirs: continue
500 if not self.islocfile(filename) and not self.includenonloc: continue
501 yield self.jartoospath(jarfilename, filename)
502
503
505 """iterates through all the files. this is the method use by the converters"""
506 for inputpath in self.iterextractnames(includenonjars=True):
507 yield inputpath
508
510 """returns whether the given pathname exists in the archive"""
511 try:
512 jarfilename, filename = self.ostojarpath(fullpath)
513 except IndexError:
514 return False
515 return self.jarfileexists(jarfilename, filename)
516
521
523 """opens an output file given the full pathname"""
524 try:
525 jarfilename, filename = self.ostojarpath(fullpath)
526 except IndexError:
527 return None
528 return self.openoutputstream(jarfilename, filename)
529
530 if __name__ == '__main__':
531 import optparse
532 optparser = optparse.OptionParser(version="%prog "+__version__.sver)
533 optparser.usage = "%prog [-l|-x] [options] file.xpi"
534 optparser.add_option("-l", "--list", help="list files", \
535 action="store_true", dest="listfiles", default=False)
536 optparser.add_option("-p", "--prefix", help="show common prefix", \
537 action="store_true", dest="showprefix", default=False)
538 optparser.add_option("-x", "--extract", help="extract files", \
539 action="store_true", dest="extractfiles", default=False)
540 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \
541 default=".", metavar="EXTRACTDIR")
542 (options, args) = optparser.parse_args()
543 if len(args) < 1:
544 optparser.error("need at least one argument")
545 xpifile = XpiFile(args[0])
546 if options.showprefix:
547 for prefix, mapto in xpifile.dirmap.iteritems():
548 print "/".join(prefix), "->", "/".join(mapto)
549 if options.listfiles:
550 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True):
551 print name
552 if options.extractfiles:
553 if options.extractdir and not os.path.isdir(options.extractdir):
554 os.mkdir(options.extractdir)
555 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False):
556 abspath = os.path.join(options.extractdir, name)
557
558 currentpath = options.extractdir
559 subparts = os.path.dirname(name).split(os.sep)
560 for part in subparts:
561 currentpath = os.path.join(currentpath, part)
562 if not os.path.isdir(currentpath):
563 os.mkdir(currentpath)
564 outputstream = open(abspath, 'w')
565 jarfilename, filename = xpifile.ostojarpath(name)
566 inputstream = xpifile.openinputstream(jarfilename, filename)
567 outputstream.write(inputstream.read())
568 outputstream.close()
569