1 #!/usr/bin/python2.4
2 #
3 # CDDL HEADER START
4 #
5 # The contents of this file are subject to the terms of the
6 # Common Development and Distribution License (the "License").
7 # You may not use this file except in compliance with the License.
8 #
9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 # or http://www.opensolaris.org/os/licensing.
11 # See the License for the specific language governing permissions
12 # and limitations under the License.
13 #
14 # When distributing Covered Code, include this CDDL HEADER in each
15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 # If applicable, add the following below this CDDL HEADER, with the
17 # fields enclosed by brackets "[]" replaced with your own identifying
18 # information: Portions Copyright [yyyy] [name of copyright owner]
19 #
20 # CDDL HEADER END
21 #
22 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 # Use is subject to license terms.
24
25
26 import fnmatch
27 import getopt
28 import gettext
29 import os
30 import pkg.depotcontroller as depotcontroller
31 import pkg.publish.transaction as trans
32 import re
33 import shlex
34 import sys
35 import urllib
36 import urlparse
37
38 from datetime import datetime
39 from itertools import groupby
40 from pkg import actions, elf
41 from pkg.bundle.SolarisPackageDirBundle import SolarisPackageDirBundle
42 from pkg.sysvpkg import SolarisPackage
43 from tempfile import mkstemp
44
45 gettext.install("import", "/usr/lib/locale")
46
47 class package(object):
48 def __init__(self, name):
49 self.name = name
50 self.files = []
51 self.depend = []
52 self.file_depend = []
53 self.idepend = [] #svr4 pkg deps, if any
54 self.undepend = []
55 self.extra = []
56 self.dropped_licenses = []
57 self.nonhollow_dirs = {}
58 self.srcpkgs = []
59 self.classification = []
60 self.desc = ""
61 self.summary = ""
62 self.version = ""
63 self.imppkg = None
64 pkgdict[name] = self
65
66 def import_pkg(self, imppkg, line):
67 try:
68 p = SolarisPackage(pkg_path(imppkg))
69 except:
70 raise RuntimeError("No such package: '%s'" % imppkg)
71
72 self.imppkg = p
73
74 svr4pkgpaths[p.pkginfo["PKG.PLAT"]] = pkg_path(imppkg)
75
76 # filename NOT always same as pkgname
77 imppkg = p.pkginfo["PKG.PLAT"]
78 svr4pkgsseen[imppkg] = p
79
80 if "SUNW_PKG_HOLLOW" in p.pkginfo and \
81 p.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true":
82 hollow_pkgs[imppkg] = True
83
84 excludes = dict((f, True) for f in line.split())
85
86 # XXX This isn't thread-safe. We want a dict method that adds
87 # the key/value pair, but throws an exception if the key is
88 # already present.
89 for o in p.manifest:
90 if o.pathname in excludes:
91 print "excluding %s from %s" % \
92 (o.pathname, imppkg)
93 continue
94
95 if o.pathname in elided_files:
96 print "ignoring %s in %s" % (o.pathname, imppkg)
97 continue
98
99 if o.type == "e":
100 if o.pathname not in editable_files:
101 editable_files[o.pathname] = \
102 [(imppkg, self)]
103 else:
104 editable_files[o.pathname].append(
105 (imppkg, self))
106
107 # XXX This decidedly ignores "e"-type files.
108
109 if o.type in "fv" and o.pathname in usedlist:
110 s = reuse_err % (
111 o.pathname,
112 self.name,
113 imppkg,
114 svr4pkgpaths[imppkg],
115 usedlist[o.pathname][1].name,
116 usedlist[o.pathname][0],
117 svr4pkgpaths[usedlist[o.pathname][0]])
118 print s
119 raise RuntimeError(s)
120 elif o.type == "i" and o.pathname == "copyright":
121 # Fake up a unique path for each license.
122 o.pathname = "//license/%s" % imppkg
123 usedlist[o.pathname] = (imppkg, self)
124 self.files.append(o)
125 elif o.type != "i":
126 if o.type in "dx" and imppkg not in hollow_pkgs:
127 self.nonhollow_dirs[o.pathname] = True
128
129 usedlist[o.pathname] = (imppkg, self)
130 self.check_perms(o)
131 self.files.append(o)
132
133 if not self.version:
134 self.version = "%s-%s" % (def_vers,
135 get_branch(self.name))
136 if not self.desc:
137 try:
138 self.desc = p.pkginfo["DESC"]
139 except KeyError:
140 self.desc = None
141 if not self.summary:
142 self.summary = zap_strings(p.pkginfo["NAME"],
143 summary_detritus)
144
145 # This is how we'd import dependencies, but we'll use
146 # file-specific dependencies only, since these tend to be
147 # broken.
148 # self.depend.extend(
149 # d.req_pkg_fmri
150 # for d in p.deps
151 # )
152
153 self.add_svr4_src(imppkg)
154
155 def add_svr4_src(self, imppkg):
156 if imppkg in destpkgs:
157 destpkgs[imppkg].append(self.name)
158 else:
159 destpkgs[imppkg] = [self.name]
160 self.srcpkgs.append(imppkg)
161
162 def import_file(self, fname, line):
163 imppkgname = self.imppkg.pkginfo["PKG.PLAT"]
164
165 if "SUNW_PKG_HOLLOW" in self.imppkg.pkginfo and \
166 self.imppkg.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true":
167 hollow_pkgs[imppkgname] = True
168
169 if fname in usedlist:
170 t = [
171 f for f in usedlist[fname][1].files
172 if f.pathname == fname
173 ][0].type
174 if t in "fv":
175 assert imppkgname == usedlist[fname][0]
176 raise RuntimeError(reuse_err % (
177 fname,
178 self.name,
179 self.imppkg,
180 svr4pkgpaths[self.imppkg],
181 usedlist[fname][1].name,
182 usedlist[fname][0],
183 svr4pkgpaths[usedlist[fname][0]]))
184
185 usedlist[fname] = (imppkgname, self)
186 o = [
187 o
188 for o in self.imppkg.manifest
189 if o.pathname == fname
190 ]
191 # There should be only one file with a given pathname in a
192 # single package.
193 if len(o) != 1:
194 print "ERROR: %s %s" % (imppkgname, fname)
195 assert len(o) == 1
196
197 if line:
198 a = actions.fromstr(
199 "%s path=%s %s" % \
200 (
201 self.convert_type(o[0].type),
202 o[0].pathname,
203 line
204 )
205 )
206 for attr in a.attrs:
207 if attr == "owner":
208 o[0].owner = a.attrs[attr]
209 elif attr == "group":
210 o[0].group = a.attrs[attr]
211 elif attr == "mode":
212 o[0].mode = a.attrs[attr]
213 self.check_perms(o[0])
214 self.files.extend(o)
215
216 def convert_type(self, svrtype):
217 """ given sv4r type, return IPS type"""
218 return {
219 "f": "file", "e": "file", "v": "file",
220 "d": "dir", "x": "dir",
221 "s": "link",
222 "l": "hardlink"
223 }[svrtype]
224
225 def type_convert(self, ipstype):
226 """ given IPS type, return svr4 type(s)"""
227 return {
228 "file": "fev", "dir": "dx", "link": "s",
229 "hardlink": "l"
230 }[ipstype]
231
232 def file_to_action(self, f):
233
234 if f.type in "dx":
235 action = actions.directory.DirectoryAction(
236 None, mode = f.mode, owner = f.owner,
237 group = f.group, path = f.pathname)
238 elif f.type in "efv":
239 action = actions.file.FileAction(
240 None, mode = f.mode, owner = f.owner,
241 group = f.group, path = f.pathname)
242 elif f.type == "s":
243 action = actions.link.LinkAction(None,
244 target = f.target, path = f.pathname)
245 elif f.type == "l":
246 action = actions.hardlink.HardLinkAction(None,
247 target = f.target, path = f.pathname)
248 else:
249 print "unknown type %s - path %s" % \
250 ( f.type, f.pathname)
251
252 return action
253
254 def check_perms(self, manifest):
255 if manifest.type not in "fevdxbc":
256 return
257
258 if manifest.owner == "?":
259 manifest.owner = "root"
260 print "File %s in pkg %s owned by '?': mapping to %s" \
261 % (manifest.pathname, self.name, manifest.owner)
262
263 if manifest.group == "?":
264 manifest.group = "bin"
265 print "File %s in pkg %s of group '?': mapping to %s" \
266 % (manifest.pathname, self.name, manifest.group)
267 if manifest.mode == "?":
268 if manifest.type in "dx":
269 manifest.mode = "0755"
270 else:
271 manifest.mode = "0444"
272 print "File %s in pkg %s mode '?': mapping to %s" % \
273 (manifest.pathname, self.name, manifest.mode)
274
275
276 def chattr(self, fname, line):
277 o = [f for f in self.files if f.pathname == fname]
278 if not o:
279 raise RuntimeError("No file '%s' in package '%s'" % \
280 (fname, curpkg.name))
281
282 line = line.rstrip()
283
284 # is this a deletion?
285 if line.startswith("drop"):
286 for f in o:
287 # deletion of existing attribute
288 if not hasattr(f, "deleted_attrs"):
289 f.deleted_attrs = []
290 print "Adding drop on %s of %s" % \
291 (fname, line.split()[1:])
292 f.deleted_attrs.extend(line.split()[1:])
293 return
294
295 # handle insertion/modification case
296 for f in o:
297 # create attribute dictionary from line
298 new_type = self.convert_type(f.type)
299 new_attrs = actions._fromstr("%s %s" %
300 (new_type, line.rstrip()))[2]
301 # get path if we're not changing it
302 if "path" not in new_attrs:
303 new_attrs["path"] = fname
304 a = actions.types[new_type](**new_attrs)
305 if show_debug:
306 print "Updating attributes on " + \
307 "'%s' in '%s' with '%s'" % \
308 (f.pathname, curpkg.name, a)
309 orig_action = self.file_to_action(f)
310
311 if not hasattr(f, "changed_attrs"):
312 f.changed_attrs = {}
313
314 # each chattr produces a dictionary of actions
315 # including a path=xxxx and whatever modifications
316 # are made. Note that the path value may be a list in
317 # the case of modifications to the path... since
318 # each chattr produces another path= entry and results
319 # from applying the changes to the original file spec,
320 # we need to ignore path if it hasn't changed... for
321 # generality, we ignore all unchanged attributes in
322 # the code below, adding into changed_attrs only those
323 # that are different from the original... this also
324 # insulates us from the possibility of actions.fromstr
325 # adding additional attributes in the constructor...
326
327 for key in a.attrs.keys():
328 if key not in orig_action.attrs or \
329 orig_action.attrs[key] != a.attrs[key]:
330 if key in f.changed_attrs:
331 print "Warning: overwriting " \
332 "changed attr %s on %s " \
333 "from %s to %s" % \
334 (key, f.pathname,
335 f.changed_attrs[key],
336 a.attrs[key])
337 f.changed_attrs[key] = a.attrs[key]
338
339
340 # apply a chattr to wildcarded files/dirs
341 # also allows package specification, wildcarding, regexp edit
342
343 def chattr_glob(self, glob, line):
344 args = line.split()
345 if args[0] == "from":
346 args.pop(0)
347 pkgglob = args.pop(0)
348 line = " ".join(args)
349 else:
350 pkgglob = "*"
351
352 if args[0] == "type": # we care about type
353 args.pop(0)
354 types = self.type_convert(args.pop(0))
355 line = " ".join(args)
356 else:
357 types = "dfevslx"
358
359 if args[0] == "edit": # we're doing regexp edit of attr
360 edit = True
361 args.pop(0)
362 target = args.pop(0)
363 regexp = re.compile(args.pop(0))
364 replace = args.pop(0)
365 line = " ".join(args)
366 else:
367 edit = False
368
369 o = [
370 f
371 for f in self.files
372 if fnmatch.fnmatchcase(f.pathname, glob) and
373 fnmatch.fnmatchcase(
374 usedlist[f.pathname][0], pkgglob) and
375 f.type in types
376 ]
377
378 chattr_line = line
379
380 for f in o:
381 fname = f.pathname
382 orig_action = self.file_to_action(f)
383 if edit:
384 if target in orig_action.attrs:
385 old_value = orig_action.attrs[target]
386 new_value = regexp.sub(replace, \
387 old_value)
388 if old_value == new_value:
389 continue
390 chattr_line = "%s=%s %s" % \
391 (target, new_value, line)
392 else:
393 continue
394 chattr_line = chattr_line.rstrip()
395 if show_debug:
396 print "Updating attributes on " + \
397 "'%s' in '%s' with '%s'" % \
398 (fname, curpkg.name, chattr_line)
399
400 # create attribute dictionary from line
401 new_type = self.convert_type(f.type)
402 new_attrs = actions._fromstr("%s %s" %
403 (new_type, chattr_line.rstrip()))[2]
404 # get path if we're not changing it
405 if "path" not in new_attrs:
406 new_attrs["path"] = fname
407 a = actions.types[new_type](**new_attrs)
408 # each chattr produces a dictionary of actions
409 # including a path=xxxx and whatever modifications
410 # are made. Note that the path value may be a list in
411 # the case of modifications to the path... since
412 # each chattr produces another path= entry and results
413 # from applying the changes to the original file spec,
414 # we need to ignore path if it hasn't changed... for
415 # generality, we ignore all unchanged attributes in
416 # the code below, adding into changed_attrs only those
417 # that are different from the original... this also
418 # insulates us from the possibility of actions.fromstr
419 # adding additional attributes in the constructor...
420
421 if not hasattr(f, "changed_attrs"):
422 f.changed_attrs = {}
423 for key in a.attrs.keys():
424 if key not in orig_action.attrs or \
425 orig_action.attrs[key] != a.attrs[key]:
426 if key in f.changed_attrs:
427 print "Warning: overwriting " \
428 "changed attr %s on %s " \
429 "from %s to %s" % \
430 (key, f.pathname,
431 f.changed_attrs[key],
432 a.attrs[key])
433 f.changed_attrs[key] = a.attrs[key]
434
435 pkgpaths = {}
436
437 def pkg_path(pkgname):
438 name = os.path.basename(pkgname)
439 if pkgname in pkgpaths:
440 return pkgpaths[name]
441 if "/" in pkgname:
442 pkgpaths[name] = os.path.realpath(pkgname)
443 return pkgname
444 else:
445 for each_path in wos_path:
446 if os.path.exists(each_path + "/" + pkgname):
447 pkgpaths[name] = each_path + "/" + pkgname
448 return pkgpaths[name]
449
450 raise RuntimeError("package %s not found" % pkgname)
451
452
453 def start_package(pkgname):
454 set_macro("PKGNAME", urllib.quote(pkgname, ""))
455 return package(pkgname)
456
457 def end_package(pkg):
458 pkg_branch = get_branch(pkg.name)
459 if not pkg.version:
460 pkg.version = "%s-%s" % (def_vers, pkg_branch)
461 elif "-" not in pkg.version:
462 pkg.version += "-%s" % pkg_branch
463
464 clear_macro("PKGNAME")
465 print "Package '%s'" % pkg.name
466 print " Version:", pkg.version
467 print " Description:", pkg.desc
468 print " Summary:", pkg.summary
469 print " Classification: ", ",".join(pkg.classification)
470
471 def publish_pkg(pkg):
472
473 new_pkg_name = "%s@%s" % (pkg.name, pkg.version)
474 t = trans.Transaction(def_repo, create_repo=create_repo,
475 pkg_name=new_pkg_name, noexecute=nopublish)
476
477 print " open %s" % new_pkg_name
478 transaction_id = t.open()
479
480 # Publish non-file objects first: they're easy.
481 for f in pkg.files:
482 if f.type in "dx":
483 action = actions.directory.DirectoryAction(
484 None, mode = f.mode, owner = f.owner,
485 group = f.group, path = f.pathname)
486 if hasattr(f, "changed_attrs"):
487 action.attrs.update(f.changed_attrs)
488 # chattr may have produced two path values
489 action.attrs["path"] = \
490 action.attrlist("path")[-1]
491 print " %s add dir %s %s %s %s" % (
492 pkg.name,
493 action.attrs["mode"],
494 action.attrs["owner"],
495 action.attrs["group"],
496 action.attrs["path"]
497 )
498 elif f.type == "s":
499 action = actions.link.LinkAction(None,
500 target = f.target, path = f.pathname)
501 if hasattr(f, "changed_attrs"):
502 action.attrs.update(f.changed_attrs)
503 # chattr may have produced two path values
504 action.attrs["path"] = \
505 action.attrlist("path")[-1]
506 print " %s add link %s %s" % (
507 pkg.name,
508 action.attrs["path"],
509 action.attrs["target"]
510 )
511 elif f.type == "l":
512 action = actions.hardlink.HardLinkAction(None,
513 target = f.target, path = f.pathname)
514 if hasattr(f, "changed_attrs"):
515 action.attrs.update(f.changed_attrs)
516 # chattr may have produced two path values
517 action.attrs["path"] = \
518 action.attrlist("path")[-1]
519 pkg.depend += process_link_dependencies(
520 action.attrs["path"], action.attrs["target"])
521 print " %s add hardlink %s %s" % (
522 pkg.name,
523 action.attrs["path"],
524 action.attrs["target"]
525 )
526 else:
527 continue
528
529 #
530 # If the originating package was hollow, tag this file
531 # as being global zone only.
532 #
533
534 if f.type not in "dx" and f.pathname in usedlist and \
535 usedlist[f.pathname][0] in hollow_pkgs:
536 action.attrs["opensolaris.zone"] = "global"
537 action.attrs["variant.opensolaris.zone"] = "global"
538
539 if f.type in "dx" and f.pathname in usedlist and \
540 usedlist[f.pathname][0] in hollow_pkgs and \
541 f.pathname not in pkg.nonhollow_dirs:
542 action.attrs["opensolaris.zone"] = "global"
543 action.attrs["variant.opensolaris.zone"] = "global"
544
545 # handle attribute deletion
546 if hasattr(f, "deleted_attrs"):
547 for d in f.deleted_attrs:
548 if d in action.attrs:
549 del action.attrs[d]
550
551 t.add(action)
552
553 # Group the files in a (new) package based on what (old) package they
554 # came from, so that we can iterate through all files in a single (old)
555 # package (and, therefore, in a single bzip2 archive) before moving on
556 # to the next. Because groupby() needs its input pre-sorted by group
557 # and we want to maintain the order that the files come out of the cpio
558 # archives, we coalesce the groups with the groups dictionary.
559 def fn(key):
560 return usedlist[key.pathname][0]
561 groups = {}
562 for k, g in groupby((f for f in pkg.files if f.type in "fevi"), fn):
563 if k in groups:
564 groups[k].extend(g)
565 else:
566 groups[k] = list(g)
567
568 def otherattrs(action):
569 s = " ".join(
570 "%s=%s" % (a, action.attrs[a])
571 for a in action.attrs
572 if a not in ("owner", "group", "mode", "path")
573 )
574 if s:
575 return " " + s
576 else:
577 return ""
578
579 # Maps class names to preserve attribute values.
580 preserve_dict = {
581 "renameold": "renameold",
582 "renamenew": "renamenew",
583 "preserve": "true",
584 "svmpreserve": "true"
585 }
586
587 undeps = set()
588 for g in groups.values():
589 pkgname = usedlist[g[0].pathname][0]
590 print "pulling files from archive in package", pkgname
591 bundle = SolarisPackageDirBundle(svr4pkgpaths[pkgname])
592 pathdict = dict((f.pathname, f) for f in g)
593 for f in bundle:
594 if f.name == "license":
595 if f.attrs["license"] in pkg.dropped_licenses:
596 continue
597 # add transaction id so that every version
598 # of a pkg will have a unique license to prevent
599 # license from disappearing on upgrade
600 f.attrs["transaction_id"] = transaction_id
601 # The "path" attribute is confusing and
602 # unnecessary for licenses.
603 del f.attrs["path"]
604 print " %s add license %s" % \
605 (pkg.name, f.attrs["license"])
606 t.add(f)
607 elif f.attrs["path"] in pathdict:
608 if pkgname in hollow_pkgs:
609 f.attrs["opensolaris.zone"] = "global"
610 f.attrs["variant.opensolaris.zone"] = \
611 "global"
612 path = f.attrs["path"]
613 if pathdict[path].type in "ev":
614 f.attrs["preserve"] = "true"
615 f.attrs["owner"] = pathdict[path].owner
616 f.attrs["group"] = pathdict[path].group
617 f.attrs["mode"] = pathdict[path].mode
618
619 # is this a file for which we need a timestamp?
620 basename = os.path.basename(path)
621 for file_pattern in timestamp_files:
622 if fnmatch.fnmatch(basename,
623 file_pattern):
624 break
625 else:
626 del f.attrs["timestamp"]
627 if pathdict[path].klass in preserve_dict.keys():
628 f.attrs["preserve"] = \
629 preserve_dict[pathdict[path].klass]
630 if hasattr(pathdict[path], "changed_attrs"):
631 f.attrs.update(
632 pathdict[path].changed_attrs)
633 # chattr may have produced two values
634 f.attrs["path"] = f.attrlist("path")[-1]
635
636 print " %s add file %s %s %s %s%s" % \
637 (pkg.name, f.attrs["mode"],
638 f.attrs["owner"], f.attrs["group"],
639 f.attrs["path"], otherattrs(f))
640
641 # handle attribute deletion
642 if hasattr(pathdict[path], "deleted_attrs"):
643 for d in pathdict[path].deleted_attrs:
644 if d in f.attrs:
645 print "removed %s from %s in pkg %s" % (d, path, new_pkg_name)
646 del f.attrs[d]
647
648 # Read the file in chunks to avoid a memory
649 # footprint blowout.
650 fo = f.data()
651 bufsz = 256 * 1024
652 sz = int(f.attrs["pkg.size"])
653 fd, tmp = mkstemp(prefix="pkg.")
654 while sz > 0:
655 d = fo.read(min(bufsz, sz))
656 os.write(fd, d)
657 sz -= len(d)
658 d = None
659 os.close(fd)
660
661 # Fool the action into pulling from a
662 # temporary file so that both add() and
663 # process_dependencies() can read() the
664 # data.
665 f.data = lambda: open(tmp, "rb")
666 t.add(f)
667
668 # Look for dependencies
669 deps, u = process_dependencies(tmp, path)
670 pkg.depend += deps
671 if u:
672 print \
673 "%s has missing dependencies: %s" \
674 % (path, u)
675 undeps |= set(u)
676 os.unlink(tmp)
677
678 # process any dependencies on files
679 for f in pkg.file_depend:
680 f = f.lstrip("/") # remove any leading /
681 if f in usedlist:
682 pkg.depend += [ "%s@%s" %
683 (usedlist[f][1].name,
684 usedlist[f][1].version)
685 ]
686 else:
687 print "Warning: pkg %s: depend_path %s not satisfied" \
688 % (pkg.name, f)
689 undeps.add(f)
690 # Publish dependencies
691
692 missing_cnt = 0
693
694 for p in set(pkg.idepend): # over set of svr4 deps, append ipkgs
695 if p in destpkgs:
696 pkg.depend.extend(destpkgs[p])
697 else:
698 print "pkg %s: SVR4 package %s not seen" % \
699 (pkg.name, p)
700 missing_cnt += 1
701 if missing_cnt > 0:
702 raise RuntimeError("missing packages!")
703
704 for p in set(pkg.depend) - set(pkg.undepend):
705 # Don't make a package depend on itself.
706 if p.split("@")[0] == pkg.name:
707 continue
708 # enhance unqualified dependencies to include current
709 # pkg version
710 if "@" not in p and p in pkgdict:
711 p = "%s@%s" % (p, pkgdict[p].version)
712
713 print " %s add depend require %s" % (pkg.name, p)
714 action = actions.depend.DependencyAction(None,
715 type = "require", fmri = p)
716 t.add(action)
717
718 for a in pkg.extra:
719 print " %s add %s" % (pkg.name, a)
720 action = actions.fromstr(a)
721 if hasattr(action, "hash"):
722 fname, fd = sourcehook(action.hash)
723 fd.close()
724 action.data = lambda: file(fname, "rb")
725 action.attrs["pkg.size"] = str(os.stat(fname).st_size)
726 if action.name == "license":
727 action.attrs["transaction_id"] = transaction_id
728 elif "path" in action.attrs:
729 path = action.attrs["path"]
730 deps, u = process_dependencies(fname, path)
731 pkg.depend += deps
732 if u:
733 print "%s has missing dependencies: " \
734 "%s" % (path, u)
735 undeps |= set(u)
736 #
737 # fmris may not be completely specified; enhance them to current
738 # version if this is the case
739 #
740 for attr in action.attrs:
741 if attr == "fmri" and \
742 "@" not in action.attrs[attr] and \
743 action.attrs[attr][5:] in pkgdict:
744 action.attrs[attr] += "@%s" % \
745 pkgdict[action.attrs[attr][5:]].version
746 t.add(action)
747
748 if pkg.desc:
749 print " %s add set pkg.description=%s" % (pkg.name, pkg.desc)
750 attrs = dict(name="pkg.description", value=pkg.desc)
751 action = actions.attribute.AttributeAction(None, **attrs)
752 t.add(action)
753
754 if pkg.summary:
755 print " %s add set pkg.summary=%s" % (pkg.name, pkg.summary)
756 attrs = dict(name="pkg.summary", value=pkg.summary)
757 action = actions.attribute.AttributeAction(None, **attrs)
758 t.add(action)
759
760 # Retain the description entry in the package manifest for
761 # now for backward compatibility.
762 print " %s add set description=%s" % (pkg.name, pkg.summary)
763 action = actions.attribute.AttributeAction(None,
764 description = pkg.summary)
765 t.add(action)
766
767 if pkg.classification:
768 print " %s add set info.classification=%s" % \
769 (pkg.name, pkg.classification)
770 attrs = dict(name="info.classification",
771 value=pkg.classification)
772 action = actions.attribute.AttributeAction(None, **attrs)
773 t.add(action)
774
775 if pkg.name != "SUNWipkg":
776 for p in pkg.srcpkgs:
777 try:
778 sp = svr4pkgsseen[p]
779 except KeyError:
780 continue
781
782 wanted_attrs = (
783 "PKG", "NAME", "ARCH", "VERSION", "CATEGORY",
784 "VENDOR", "DESC", "HOTLINE"
785 )
786 attrs = dict(
787 (k.lower(), v)
788 for k, v in sp.pkginfo.iteritems()
789 if k in wanted_attrs
790 )
791 attrs["pkg"] = sp.pkginfo["PKG.PLAT"]
792
793 action = actions.legacy.LegacyAction(None, **attrs)
794
795 print " %s add %s" % (pkg.name, action)
796 t.add(action)
797
798 if undeps:
799 print "Missing dependencies:", list(undeps)
800
801 print " close"
802 pkg_fmri, pkg_state = t.close(refresh_index=not defer_refresh)
803 print "%s: %s\n" % (pkg_fmri, pkg_state)
804
805 def process_link_dependencies(path, target):
806 orig_target = target
807 if target[0] != "/":
808 target = os.path.normpath(
809 os.path.join(os.path.split(path)[0], target))
810
811 if target in usedlist:
812 if show_debug:
813 print "hardlink %s -> %s makes %s depend on %s" % \
814 (
815 path, orig_target,
816 usedlist[path][1].name,
817 usedlist[target][1].name
818 )
819 return ["%s@%s" % (usedlist[target][1].name,
820 usedlist[target][1].version)]
821 else:
822 return []
823
824 def process_dependencies(fname, path):
825 if not elf.is_elf_object(fname):
826 return process_non_elf_dependencies(fname, path)
827
828 ei = elf.get_info(fname)
829 try:
830 ed = elf.get_dynamic(fname)
831 except elf.ElfError:
832 deps = []
833 rp = []
834 else:
835 deps = [
836 d[0]
837 for d in ed.get("deps", [])
838 ]
839 rp = ed.get("runpath", "").split(":")
840 if len(rp) == 1 and rp[0] == "":
841 rp = []
842
843 rp = [
844 os.path.normpath(p.replace("$ORIGIN", "/" + os.path.dirname(path)))
845 for p in rp
846 ]
847
848 kernel64 = None
849
850 # For kernel modules, default path resolution is /platform/<platform>,
851 # /kernel, /usr/kernel. But how do we know what <platform> would be for
852 # a given module? Does it do fallbacks to, say, sun4u?
853 if path.startswith("kernel") or path.startswith("usr/kernel") or \
854 (path.startswith("platform") and path.split("/")[2] == "kernel"):
855 if rp:
856 print "RUNPATH set for kernel module (%s): %s" % \
857 (path, rp)
858 # Default kernel search path
859 rp.extend(("/kernel", "/usr/kernel"))
860 # What subdirectory should we look in for 64-bit kernel modules?
861 if ei["bits"] == 64:
862 if ei["arch"] == "i386":
863 kernel64 = "amd64"
864 elif ei["arch"] == "sparc":
865 kernel64 = "sparcv9"
866 else:
867 print ei["arch"]
868 else:
869 if "/lib" not in rp:
870 rp.append("/lib")
871 if "/usr/lib" not in rp:
872 rp.append("/usr/lib")
873
874 # XXX Do we need to handle anything other than $ORIGIN? x86 images have
875 # a couple of $PLATFORM and $ISALIST instances.
876 for p in rp:
877 if "$" in p:
878 tok = p[p.find("$"):]
879 if "/" in tok:
880 tok = tok[:tok.find("/")]
881 print "%s has dynamic token %s in rpath" % (path, tok)
882
883 dep_pkgs = []
884 undeps = []
885 depend_list = []
886 for d in deps:
887 for p in rp:
888 # The instances of "[1:]" below are because usedlist
889 # stores paths without leading slash
890 if kernel64:
891 # Find 64-bit modules the way krtld does.
892 # XXX We don't resolve dependencies found in
893 # /platform, since we don't know where under
894 # /platform to look.
895 head, tail = os.path.split(d)
896 deppath = os.path.join(p,
897 head,
898 kernel64,
899 tail)[1:]
900 else:
901 # This is a hack for when a runpath uses the 64
902 # symlink to the actual 64-bit directory.
903 # Better would be to see if the runpath was a
904 # link, and if so, use its resolution, but
905 # extracting that information from used list is
906 # a pain, especially because you potentially
907 # have to resolve symlinks at all levels of the
908 # path.
909 if p.endswith("/64"):
910 if ei["arch"] == "i386":
911 p = p[:-2] + "amd64"
912 elif ei["arch"] == "sparc":
913 p = p[:-2] + "sparcv9"
914 deppath = os.path.join(p, d)[1:]
915 if deppath in usedlist:
916 dep_pkgs += [ "%s@%s" %
917 (usedlist[deppath][1].name,
918 usedlist[deppath][1].version) ]
919 depend_list.append(
920 (
921 deppath,
922 usedlist[deppath][1].name
923 )
924 )
925 break
926 else:
927 undeps += [ d ]
928
929 if show_debug:
930 print "%s makes %s depend on %s" % \
931 (path, usedlist[path][1].name, depend_list)
932
933 return dep_pkgs, undeps
934
935 def process_non_elf_dependencies(localpath, path):
936 # localpath is path to actual file
937 # path is path in installed image
938 # take 1
939 dep_pkgs = []
940 undeps = []
941
942 f = file(localpath)
943 l = f.readline()
944 f.close()
945
946 # add #!/ dependency
947 if l.startswith("#!/"):
948 # usedlist omits leading /
949 p = (l[2:].split()[0]) # first part of string is path (removes options)
950 # we don't handle dependencies through links, so fix up the common one
951 if p.startswith("/bin"):
952 p = "/usr" + p
953 if p[1:] in usedlist:
954 dep_pkgs += [ "%s@%s" % (
955 usedlist[p[1:]][1].name,
956 usedlist[p[1:]][1].version)
957 ]
958 print "Added dependency on %s because of %s" % (usedlist[p[1:]][1].name, p)
959 else:
960 undeps = [ p ]
961
962 return dep_pkgs, undeps
963
964 def zap_strings(instr, strings):
965 """takes an input string and a list of strings to be removed, ignoring
966 case"""
967 for s in strings:
968 ls = s.lower()
969 while True:
970 li = instr.lower()
971 i = li.find(ls)
972 if i < 0:
973 break
974 instr = instr[0:i] + instr[i + len(ls):]
975 return instr
976
977 def get_branch(name):
978 return branch_dict.get(name, def_branch)
979
980 def_vers = "0.5.11"
981 def_branch = ""
982 def_wos_path = ["/net/netinstall.eng/export/nv/x/latest/Solaris_11/Product"]
983 create_repo = False
984 nopublish = False
985 show_debug = False
986 print_pkg_names = False
987 def_repo = "http://localhost:10000"
988 wos_path = []
989 include_path = []
990 branch_dict = {}
991 timestamp_files = []
992
993 #
994 # files (by path) we always delete for bulk imports
995 # note that we ignore these if specifically included.
996 #
997 elided_files = {}
998 #
999 # if user uses -j, just_these_pkgs becomes list of pkgs to process
1000 # allowing other arguments to be read in as files...
1001 #
1002 just_these_pkgs = []
1003 #
1004 # strings to rip out of summaries (case insensitve)
1005 #
1006 summary_detritus = [", (usr)", ", (root)", " (usr)", " (root)",
1007 " (/usr)", " - / filesystem", ",root(/)"]
1008 #
1009 # list of global includes to add to every package
1010 #
1011 global_includes = []
1012 # list of macro substitutions
1013 macro_definitions = {}
1014
1015 def set_macro(key, value):
1016 macro_definitions.update([("$(%s)" % key, value)])
1017
1018 def clear_macro(key):
1019 del macro_definitions["$(%s)" % key]
1020
1021 try:
1022 _opts, _args = getopt.getopt(sys.argv[1:], "B:D:I:G:NT:b:dj:m:ns:v:w:p:")
1023 except getopt.GetoptError, _e:
1024 print "unknown option", _e.opt
1025 sys.exit(1)
1026
1027 g_proto_area = os.environ.get("ROOT", "")
1028
1029 for opt, arg in _opts:
1030 if opt == "-b":
1031 def_branch = arg.rstrip("abcdefghijklmnopqrstuvwxyz")
1032 elif opt == "-d":
1033 show_debug = True
1034 elif opt == "-j": # means we're using the new argument form...
1035 just_these_pkgs.append(arg)
1036 elif opt == "-m":
1037 _a = arg.split("=", 1)
1038 set_macro(_a[0], _a[1])
1039 elif opt == "-n":
1040 nopublish = True
1041 elif opt == "-p":
1042 if not os.path.exists(arg):
1043 raise RuntimeError("Invalid prototype area specified.")
1044 # Clean up relative ../../, etc. out of path to proto
1045 g_proto_area = os.path.realpath(arg)
1046 elif opt == "-s":
1047 def_repo = arg
1048 if def_repo.startswith("file://"):
1049 # When publishing to file:// repositories, automatically
1050 # create the target repository if needed.
1051 create_repo = True
1052 elif opt == "-v":
1053 def_vers = arg
1054 elif opt == "-w":
1055 wos_path.append(arg)
1056 elif opt == "-D":
1057 elided_files[arg] = True
1058 elif opt == "-I":
1059 include_path.extend(arg.split(":"))
1060 elif opt == "-B":
1061 branch_file = file(arg)
1062 for _line in branch_file:
1063 if not _line.startswith("#"):
1064 bfargs = _line.split()
1065 if len(bfargs) == 2:
1066 branch_dict[bfargs[0]] = bfargs[1]
1067 branch_file.close()
1068 elif opt == "-G": #another file of global includes
1069 global_includes.append(arg)
1070 elif opt == "-N":
1071 print_pkg_names = True
1072 elif opt == "-T":
1073 timestamp_files.append(arg)
1074
1075 if not def_branch:
1076 print "need a branch id (build number)"
1077 sys.exit(1)
1078 elif "." not in def_branch:
1079 print "branch id needs to be of the form 'x.y'"
1080 sys.exit(1)
1081
1082 if not _args:
1083 print "need argument!"
1084 sys.exit(1)
1085
1086 if not wos_path:
1087 wos_path = def_wos_path
1088
1089 if just_these_pkgs:
1090 filelist = _args
1091 else:
1092 filelist = _args[0:1]
1093 just_these_pkgs = _args[1:]
1094
1095
1096 in_multiline_import = False
1097
1098 # This maps what files we've seen to a tuple of what packages they came from and
1099 # what packages they went into, so we can prevent more than one package from
1100 # grabbing the same file.
1101 usedlist = {}
1102
1103 #
1104 # pkgdict contains ipkgs by name
1105 #
1106 pkgdict = {}
1107
1108 #
1109 # destpkgs contains the list of ipkgs generated from each svr4 pkg
1110 # this is needed to generate metaclusters
1111 #
1112 destpkgs = {}
1113
1114 #
1115 #svr4 pkgs seen - pkgs indexed by name
1116 #
1117 svr4pkgsseen = {}
1118
1119 #
1120 #paths where we found the packages we need
1121 #
1122 svr4pkgpaths = {}
1123
1124 #
1125 # editable files and where they're found
1126 #
1127 editable_files = {}
1128
1129 #
1130 # hollow svr4 packages processed
1131 #
1132 hollow_pkgs = {}
1133
1134
1135 reuse_err = \
1136 "Conflict in path %s: IPS %s SVR4 %s from %s with IPS %s SVR4 %s from %s"
1137
1138
1139 # First pass: don't actually publish anything, because we're not collecting
1140 # dependencies here.
1141 def read_full_line(lexer, continuation='\\'):
1142 """Read a complete line, allowing for the possibility of it being
1143 continued over multiple lines. Returns a single joined line, with
1144 continuation characters and leading and trailing spaces removed.
1145 """
1146
1147 lines = []
1148 while True:
1149 line = lexer.instream.readline().strip()
1150 lexer.lineno = lexer.lineno + 1
1151 if line[-1] in continuation:
1152 lines.append(line[:-1])
1153 else:
1154 lines.append(line)
1155 break
1156
1157 return apply_macros(' '.join(lines))
1158
1159 def apply_macros(s):
1160 """Apply macro subs defined on command line... keep applying
1161 macros until no translations are found. If macro translates
1162 to a comment, replace entire token text."""
1163 while s and "$(" in s:
1164 for key in macro_definitions.keys():
1165 if key in s:
1166 value = macro_definitions[key]
1167 if value == "#": # comment character
1168 s = "#" # affects whole token
1169 break
1170 s = s.replace(key, value)
1171 break # look for more substitutions
1172 else:
1173 break # no more substitutable tokens
1174 return s
1175
1176 def sourcehook(filename):
1177 """ implement include hierarchy """
1178 for i in include_path:
1179 f = os.path.join(i, filename)
1180 if os.path.exists(f):
1181 return (f, open(f))
1182
1183 return filename, open(filename)
1184
1185 class tokenlexer(shlex.shlex):
1186 def read_token(self):
1187 """ simple replacement of $(ARCH) with a non-special
1188 value defined on the command line is trivial. Since
1189 shlex's read_token routine also strips comments and
1190 white space, this read_token cannot return either
1191 one so any macros that translate to either spaces or
1192 # (comment) need to be removed from the token stream."""
1193
1194 while True:
1195 s = apply_macros(shlex.shlex.read_token(self))
1196 if s == "#": # discard line if comment; try again
1197 self.instream.readline()
1198 self.lineno = self.lineno + 1
1199 # bail on EOF or not space; loop on space
1200 elif s == None or (s != "" and not s.isspace()):
1201 break
1202 return s
1203
1204 curpkg = None
1205 def SolarisParse(mf):
1206 global curpkg
1207 global in_multiline_import
1208
1209 lexer = tokenlexer(file(mf), mf, True)
1210 lexer.whitespace_split = True
1211 lexer.source = "include"
1212 lexer.sourcehook = sourcehook
1213
1214 while True:
1215 token = lexer.get_token()
1216
1217 if not token:
1218 break
1219
1220 if token == "package":
1221 curpkg = start_package(lexer.get_token())
1222
1223 if print_pkg_names:
1224 print "-j %s" % curpkg.name
1225
1226 elif token == "end":
1227 endarg = lexer.get_token()
1228 if endarg == "package":
1229 if print_pkg_names:
1230 curpkg = None
1231 continue
1232
1233 for filename in global_includes:
1234 for i in include_path:
1235 f = os.path.join(i, filename)
1236 if os.path.exists(f):
1237 SolarisParse(f)
1238 break
1239 else:
1240 raise RuntimeError("File not "
1241 "found: %s" % filename)
1242 try:
1243 end_package(curpkg)
1244 except Exception, e:
1245 print "ERROR(end_pkg):", e
1246
1247 curpkg = None
1248 if endarg == "import":
1249 in_multiline_import = False
1250 curpkg.imppkg = None
1251
1252 elif token == "version":
1253 curpkg.version = lexer.get_token()
1254
1255 elif token == "import":
1256 package_name = lexer.get_token()
1257 next = lexer.get_token()
1258 if next != "exclude":
1259 line = ""
1260 lexer.push_token(next)
1261 else:
1262 line = read_full_line(lexer)
1263
1264 if not print_pkg_names:
1265 curpkg.import_pkg(package_name, line)
1266
1267 elif token == "from":
1268 pkgspec = lexer.get_token()
1269 if not print_pkg_names:
1270 p = SolarisPackage(pkg_path(pkgspec))
1271 curpkg.imppkg = p
1272 spkgname = p.pkginfo["PKG.PLAT"]
1273 svr4pkgpaths[spkgname] = pkg_path(pkgspec)
1274 svr4pkgsseen[spkgname] = p
1275 curpkg.add_svr4_src(spkgname)
1276
1277 junk = lexer.get_token()
1278 assert junk == "import"
1279 in_multiline_import = True
1280
1281 elif token == "classification":
1282 cat_subcat = lexer.get_token()
1283 curpkg.classification.append(
1284 "org.opensolaris.category.2008:%s" % cat_subcat)
1285
1286 elif token == "description":
1287 curpkg.desc = lexer.get_token()
1288
1289 elif token == "summary":
1290 curpkg.summary = lexer.get_token()
1291
1292 elif token == "depend":
1293 curpkg.depend.append(lexer.get_token())
1294
1295 elif token == "depend_path":
1296 curpkg.file_depend.append(lexer.get_token())
1297
1298 elif token == "cluster":
1299 curpkg.add_svr4_src(lexer.get_token())
1300
1301 elif token == "idepend":
1302 curpkg.idepend.append(lexer.get_token())
1303
1304 elif token == "undepend":
1305 curpkg.undepend.append(lexer.get_token())
1306
1307 elif token == "add":
1308 curpkg.extra.append(read_full_line(lexer))
1309
1310 elif token == "drop":
1311 f = lexer.get_token()
1312 if print_pkg_names:
1313 continue
1314 l = [o for o in curpkg.files if o.pathname == f]
1315 if not l:
1316 print "Cannot drop '%s' from '%s': not " \
1317 "found" % (f, curpkg.name)
1318 else:
1319 del curpkg.files[curpkg.files.index(l[0])]
1320 # XXX The problem here is that if we do this on
1321 # a shared file (directory, etc), then it's
1322 # missing from usedlist entirely, since we don't
1323 # keep around *all* packages delivering a shared
1324 # file, just the last seen. This probably
1325 # doesn't matter much.
1326 del usedlist[f]
1327
1328 elif token == "drop_license":
1329 curpkg.dropped_licenses.append(lexer.get_token())
1330
1331 elif token == "chattr":
1332 fname = lexer.get_token()
1333 line = read_full_line(lexer)
1334 if print_pkg_names:
1335 continue
1336 try:
1337 curpkg.chattr(fname, line)
1338 except Exception, e:
1339 print "Can't change attributes on " + \
1340 "'%s': not in the package" % fname, e
1341 raise
1342
1343 elif token == "chattr_glob":
1344 glob = lexer.get_token()
1345 line = read_full_line(lexer)
1346 if print_pkg_names:
1347 continue
1348 try:
1349 curpkg.chattr_glob(glob, line)
1350 except Exception, e:
1351 print "Can't change attributes on " + \
1352 "'%s': no matches in the package" % \
1353 glob, e
1354 raise
1355
1356 elif in_multiline_import:
1357 next = lexer.get_token()
1358 if next == "with":
1359 # I can't imagine this is supported, but there's
1360 # no other way to read the rest of the line
1361 # without a whole lot more pain.
1362 line = read_full_line(lexer)
1363 else:
1364 lexer.push_token(next)
1365 line = ""
1366
1367 try:
1368 curpkg.import_file(token, line)
1369 except Exception, e:
1370 print "ERROR(import_file):", e
1371 raise
1372 else:
1373 raise RuntimeError("Error: unknown token '%s' "
1374 "(%s:%s)" % (token, lexer.infile, lexer.lineno))
1375
1376 if print_pkg_names:
1377 for _mf in filelist:
1378 SolarisParse(_mf)
1379 sys.exit(0)
1380
1381
1382 print "First pass:", datetime.now()
1383
1384 for _mf in filelist:
1385 SolarisParse(_mf)
1386
1387 seenpkgs = set(i[0] for i in usedlist.values())
1388
1389 print "Files you seem to have forgotten:\n " + "\n ".join(
1390 "%s %s" % (f.type, f.pathname)
1391 for pkg in seenpkgs
1392 for f in svr4pkgsseen[pkg].manifest
1393 if f.type != "i" and f.pathname not in usedlist)
1394
1395 print "\n\nDuplicate Editables files list:\n"
1396
1397 if editable_files:
1398 length = 2 + max(len(p) for p in editable_files)
1399 for paths in editable_files:
1400 if len(editable_files[paths]) > 1:
1401 print ("%s:" % paths).ljust(length - 1) + \
1402 ("\n".ljust(length)).join("%s (from %s)" % \
1403 (l[1].name, l[0]) for l in editable_files[paths])
1404
1405
1406 # Second pass: iterate over the existing package objects, gathering dependencies
1407 # and publish!
1408
1409 print "Second pass:", datetime.now()
1410
1411 print "New packages:\n"
1412 # XXX Sort these. Preferably topologically, if possible, alphabetically
1413 # otherwise (for a rough progress gauge).
1414 if just_these_pkgs:
1415 newpkgs = set(pkgdict[name]
1416 for name in pkgdict.keys()
1417 if name in just_these_pkgs
1418 )
1419 else:
1420 newpkgs = set(pkgdict.values())
1421
1422 # Indicates whether search indices refresh will be deferred until the end.
1423 defer_refresh = False
1424 # Indicates whether local publishing is active.
1425 local_publish = False
1426 if def_repo.startswith("file:"):
1427 # If publishing to disk, the search indices should be refreshed at
1428 # the end of the publishing process and the feed cache will have to be
1429 # generated by starting the depot server using the provided path and
1430 # then accessing it.
1431 defer_refresh = True
1432 local_publish = True
1433
1434 processed = 0
1435 total = len(newpkgs)
1436 for _p in sorted(newpkgs):
1437 print "Package '%s'" % _p.name
1438 print " Version:", _p.version
1439 print " Description:", _p.desc
1440 print " Summary:", _p.summary
1441 print " Classification:", ",".join(_p.classification)
1442 try:
1443 publish_pkg(_p)
1444 except trans.TransactionError, _e:
1445 print "%s: FAILED: %s\n" % (_p.name, _e)
1446 processed += 1
1447 print "%d/%d packages processed; %.2f%% complete" % (processed, total,
1448 processed * 100.0 / total)
1449
1450 if not nopublish and defer_refresh:
1451 # This has to be done at the end for some publishing modes.
1452 print "Updating search indices..."
1453 _t = trans.Transaction(def_repo)
1454 _t.refresh_index()
1455
1456 # Ensure that the feed is updated and cached to reflect changes.
1457 if not nopublish:
1458 print "Caching RSS/Atom feed..."
1459 dc = None
1460 durl = def_repo
1461 if local_publish:
1462 # The depot server isn't already running, so will have to be
1463 # temporarily started to allow proper feed cache generation.
1464 dc = depotcontroller.DepotController()
1465 dc.set_depotd_path(g_proto_area + "/usr/lib/pkg.depotd")
1466 dc.set_depotd_content_root(g_proto_area + "/usr/share/lib/pkg")
1467
1468 _scheme, _netloc, _path, _params, _query, _fragment = \
1469 urlparse.urlparse(def_repo, "file", allow_fragments=0)
1470
1471 dc.set_repodir(_path)
1472
1473 # XXX There must be a better way...
1474 dc.set_port(29083)
1475
1476 # Start the depot
1477 dc.start()
1478
1479 durl = "http://localhost:29083"
1480
1481 _f = urllib.urlopen("%s/feed" % durl)
1482 _f.close()
1483
1484 if dc:
1485 dc.stop()
1486 dc = None
1487
1488 print "Done:", datetime.now()