1 #!/usr/bin/python2.4
2 #
3 # CDDL HEADER START
4 #
5 # The contents of this file are subject to the terms of the
6 # Common Development and Distribution License (the "License").
7 # You may not use this file except in compliance with the License.
8 #
9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 # or http://www.opensolaris.org/os/licensing.
11 # See the License for the specific language governing permissions
12 # and limitations under the License.
13 #
14 # When distributing Covered Code, include this CDDL HEADER in each
15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 # If applicable, add the following below this CDDL HEADER, with the
17 # fields enclosed by brackets "[]" replaced with your own identifying
18 # information: Portions Copyright [yyyy] [name of copyright owner]
19 #
20 # CDDL HEADER END
21 #
22 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 # Use is subject to license terms.
24
25
26 import fnmatch
27 import getopt
28 import gettext
29 import os
30 import pkg.depotcontroller as depotcontroller
31 import pkg.publish.transaction as trans
32 import re
33 import shlex
34 import sys
35 import urllib
36 import urlparse
37
38 from datetime import datetime
39 from itertools import groupby
40 from pkg import actions, elf
41 from pkg.bundle.SolarisPackageDirBundle import SolarisPackageDirBundle
42 from pkg.sysvpkg import SolarisPackage
43 from tempfile import mkstemp
44
45 gettext.install("import", "/usr/lib/locale")
46
47 class package(object):
48 def __init__(self, name):
49 self.name = name
50 self.files = []
51 self.depend = []
52 self.file_depend = []
53 self.idepend = [] #svr4 pkg deps, if any
54 self.undepend = []
55 self.extra = []
56 self.dropped_licenses = []
57 self.nonhollow_dirs = {}
58 self.srcpkgs = []
59 self.classification = []
60 self.desc = ""
61 self.summary = ""
62 self.version = ""
63 self.imppkg = None
64 pkgdict[name] = self
65
66 def import_pkg(self, imppkg, line):
67 try:
68 p = SolarisPackage(pkg_path(imppkg))
69 except:
70 raise RuntimeError("No such package: '%s'" % imppkg)
71
72 self.imppkg = p
73
74 svr4pkgpaths[p.pkginfo["PKG.PLAT"]] = pkg_path(imppkg)
75
76 # filename NOT always same as pkgname
77 imppkg = p.pkginfo["PKG.PLAT"]
78 svr4pkgsseen[imppkg] = p
79
80 if "SUNW_PKG_HOLLOW" in p.pkginfo and \
81 p.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true":
82 hollow_pkgs[imppkg] = True
83
84 excludes = dict((f, True) for f in line.split())
85
86 # XXX This isn't thread-safe. We want a dict method that adds
87 # the key/value pair, but throws an exception if the key is
88 # already present.
89 for o in p.manifest:
90 if o.pathname in excludes:
91 print "excluding %s from %s" % \
92 (o.pathname, imppkg)
93 continue
94
95 if o.pathname in elided_files:
96 print "ignoring %s in %s" % (o.pathname, imppkg)
97 continue
98
99 if o.type == "e":
100 if o.pathname not in editable_files:
101 editable_files[o.pathname] = \
102 [(imppkg, self)]
103 else:
104 editable_files[o.pathname].append(
105 (imppkg, self))
106
107 # XXX This decidedly ignores "e"-type files.
108
109 if o.type in "fv" and o.pathname in usedlist:
110 s = reuse_err % (
111 o.pathname,
112 self.name,
113 imppkg,
114 svr4pkgpaths[imppkg],
115 usedlist[o.pathname][1].name,
116 usedlist[o.pathname][0],
117 svr4pkgpaths[usedlist[o.pathname][0]])
118 print s
119 raise RuntimeError(s)
120 elif o.type == "i" and o.pathname == "copyright":
121 # Fake up a unique path for each license.
122 o.pathname = "//license/%s" % imppkg
123 usedlist[o.pathname] = (imppkg, self)
124 self.files.append(o)
125 elif o.type != "i":
126 if o.type in "dx" and imppkg not in hollow_pkgs:
127 self.nonhollow_dirs[o.pathname] = True
128
129 usedlist[o.pathname] = (imppkg, self)
130 self.check_perms(o)
131 self.files.append(o)
132
133 if not self.version:
134 self.version = "%s-%s" % (def_vers,
135 get_branch(self.name))
136 if not self.desc:
137 try:
138 self.desc = p.pkginfo["DESC"]
139 except KeyError:
140 self.desc = None
141 if not self.summary:
142 self.summary = zap_strings(p.pkginfo["NAME"],
143 summary_detritus)
144
145 # This is how we'd import dependencies, but we'll use
146 # file-specific dependencies only, since these tend to be
147 # broken.
148 # self.depend.extend(
149 # d.req_pkg_fmri
150 # for d in p.deps
151 # )
152
153 self.add_svr4_src(imppkg)
154
155 def add_svr4_src(self, imppkg):
156 if imppkg in destpkgs:
157 destpkgs[imppkg].append(self.name)
158 else:
159 destpkgs[imppkg] = [self.name]
160 self.srcpkgs.append(imppkg)
161
162 def import_file(self, fname, line):
163 imppkgname = self.imppkg.pkginfo["PKG.PLAT"]
164
165 if "SUNW_PKG_HOLLOW" in self.imppkg.pkginfo and \
166 self.imppkg.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true":
167 hollow_pkgs[imppkgname] = True
168
169 if fname in usedlist:
170 t = [
171 f for f in usedlist[fname][1].files
172 if f.pathname == fname
173 ][0].type
174 if t in "fv":
175 assert imppkgname == usedlist[fname][0]
176 raise RuntimeError(reuse_err % (
177 fname,
178 self.name,
179 self.imppkg,
180 svr4pkgpaths[self.imppkg],
181 usedlist[fname][1].name,
182 usedlist[fname][0],
183 svr4pkgpaths[usedlist[fname][0]]))
184
185 usedlist[fname] = (imppkgname, self)
186 o = [
187 o
188 for o in self.imppkg.manifest
189 if o.pathname == fname
190 ]
191 # There should be only one file with a given pathname in a
192 # single package.
193 if len(o) != 1:
194 print "ERROR: %s %s" % (imppkgname, fname)
195 assert len(o) == 1
196
197 if line:
198 a = actions.fromstr(
199 "%s path=%s %s" % \
200 (
201 self.convert_type(o[0].type),
202 o[0].pathname,
203 line
204 )
205 )
206 for attr in a.attrs:
207 if attr == "owner":
208 o[0].owner = a.attrs[attr]
209 elif attr == "group":
210 o[0].group = a.attrs[attr]
211 elif attr == "mode":
212 o[0].mode = a.attrs[attr]
213 self.check_perms(o[0])
214 self.files.extend(o)
215
216 def convert_type(self, svrtype):
217 """ given sv4r type, return IPS type"""
218 return {
219 "f": "file", "e": "file", "v": "file",
220 "d": "dir", "x": "dir",
221 "s": "link",
222 "l": "hardlink"
223 }[svrtype]
224
225 def type_convert(self, ipstype):
226 """ given IPS type, return svr4 type(s)"""
227 return {
228 "file": "fev", "dir": "dx", "link": "s",
229 "hardlink": "l"
230 }[ipstype]
231
232 def file_to_action(self, f):
233
234 if f.type in "dx":
235 action = actions.directory.DirectoryAction(
236 None, mode = f.mode, owner = f.owner,
237 group = f.group, path = f.pathname)
238 elif f.type in "efv":
239 action = actions.file.FileAction(
240 None, mode = f.mode, owner = f.owner,
241 group = f.group, path = f.pathname)
242 elif f.type == "s":
243 action = actions.link.LinkAction(None,
244 target = f.target, path = f.pathname)
245 elif f.type == "l":
246 action = actions.hardlink.HardLinkAction(None,
247 target = f.target, path = f.pathname)
248 else:
249 print "unknown type %s - path %s" % \
250 ( f.type, f.pathname)
251
252 return action
253
254 def check_perms(self, manifest):
255 if manifest.type not in "fevdxbc":
256 return
257
258 if manifest.owner == "?":
259 manifest.owner = "root"
260 print "File %s in pkg %s owned by '?': mapping to %s" \
261 % (manifest.pathname, self.name, manifest.owner)
262
263 if manifest.group == "?":
264 manifest.group = "bin"
265 print "File %s in pkg %s of group '?': mapping to %s" \
266 % (manifest.pathname, self.name, manifest.group)
267 if manifest.mode == "?":
268 if manifest.type in "dx":
269 manifest.mode = "0755"
270 else:
271 manifest.mode = "0444"
272 print "File %s in pkg %s mode '?': mapping to %s" % \
273 (manifest.pathname, self.name, manifest.mode)
274
275
276 def chattr(self, fname, line):
277 o = [f for f in self.files if f.pathname == fname]
278 if not o:
279 raise RuntimeError("No file '%s' in package '%s'" % \
280 (fname, curpkg.name))
281
282 line = line.rstrip()
283
284 # is this a deletion?
285 if line.startswith("drop"):
286 for f in o:
287 # deletion of existing attribute
288 if not hasattr(f, "deleted_attrs"):
289 f.deleted_attrs = []
290 print "Adding drop on %s of %s" % \
291 (fname, line.split()[1:])
292 f.deleted_attrs.extend(line.split()[1:])
293 return
294
295 # handle insertion/modification case
296 for f in o:
297 # create attribute dictionary from line
298 new_type = self.convert_type(f.type)
299 new_attrs = actions._fromstr("%s %s" %
300 (new_type, line.rstrip()))[2]
301 # get path if we're not changing it
302 if "path" not in new_attrs:
303 new_attrs["path"] = fname
304 a = actions.types[new_type](**new_attrs)
305 if show_debug:
306 print "Updating attributes on " + \
307 "'%s' in '%s' with '%s'" % \
308 (f.pathname, curpkg.name, a)
309 orig_action = self.file_to_action(f)
310
311 if not hasattr(f, "changed_attrs"):
312 f.changed_attrs = {}
313
314 # each chattr produces a dictionary of actions
315 # including a path=xxxx and whatever modifications
316 # are made. Note that the path value may be a list in
317 # the case of modifications to the path... since
318 # each chattr produces another path= entry and results
319 # from applying the changes to the original file spec,
320 # we need to ignore path if it hasn't changed... for
321 # generality, we ignore all unchanged attributes in
322 # the code below, adding into changed_attrs only those
323 # that are different from the original... this also
324 # insulates us from the possibility of actions.fromstr
325 # adding additional attributes in the constructor...
326
327 for key in a.attrs.keys():
328 if key not in orig_action.attrs or \
329 orig_action.attrs[key] != a.attrs[key]:
330 if key in f.changed_attrs:
331 print "Warning: overwriting " \
332 "changed attr %s on %s " \
333 "from %s to %s" % \
334 (key, f.pathname,
335 f.changed_attrs[key],
336 a.attrs[key])
337 f.changed_attrs[key] = a.attrs[key]
338
339
340 # apply a chattr to wildcarded files/dirs
341 # also allows package specification, wildcarding, regexp edit
342
343 def chattr_glob(self, glob, line):
344 args = line.split()
345 if args[0] == "from":
346 args.pop(0)
347 pkgglob = args.pop(0)
348 line = " ".join(args)
349 else:
350 pkgglob = "*"
351
352 if args[0] == "type": # we care about type
353 args.pop(0)
354 types = self.type_convert(args.pop(0))
355 line = " ".join(args)
356 else:
357 types = "dfevslx"
358
359 if args[0] == "edit": # we're doing regexp edit of attr
360 edit = True
361 args.pop(0)
362 target = args.pop(0)
363 regexp = re.compile(args.pop(0))
364 replace = args.pop(0)
365 line = " ".join(args)
366 else:
367 edit = False
368
369 o = [
370 f
371 for f in self.files
372 if fnmatch.fnmatchcase(f.pathname, glob) and
373 fnmatch.fnmatchcase(
374 usedlist[f.pathname][0], pkgglob) and
375 f.type in types
376 ]
377
378 chattr_line = line
379
380 for f in o:
381 fname = f.pathname
382 orig_action = self.file_to_action(f)
383 if edit:
384 if target in orig_action.attrs:
385 old_value = orig_action.attrs[target]
386 new_value = regexp.sub(replace, \
387 old_value)
388 if old_value == new_value:
389 continue
390 chattr_line = "%s=%s %s" % \
391 (target, new_value, line)
392 else:
393 continue
394 chattr_line = chattr_line.rstrip()
395 if show_debug:
396 print "Updating attributes on " + \
397 "'%s' in '%s' with '%s'" % \
398 (fname, curpkg.name, chattr_line)
399
400 # create attribute dictionary from line
401 new_type = self.convert_type(f.type)
402 new_attrs = actions._fromstr("%s %s" %
403 (new_type, chattr_line.rstrip()))[2]
404 # get path if we're not changing it
405 if "path" not in new_attrs:
406 new_attrs["path"] = fname
407 a = actions.types[new_type](**new_attrs)
408 # each chattr produces a dictionary of actions
409 # including a path=xxxx and whatever modifications
410 # are made. Note that the path value may be a list in
411 # the case of modifications to the path... since
412 # each chattr produces another path= entry and results
413 # from applying the changes to the original file spec,
414 # we need to ignore path if it hasn't changed... for
415 # generality, we ignore all unchanged attributes in
416 # the code below, adding into changed_attrs only those
417 # that are different from the original... this also
418 # insulates us from the possibility of actions.fromstr
419 # adding additional attributes in the constructor...
420
421 if not hasattr(f, "changed_attrs"):
422 f.changed_attrs = {}
423 for key in a.attrs.keys():
424 if key not in orig_action.attrs or \
425 orig_action.attrs[key] != a.attrs[key]:
426 if key in f.changed_attrs:
427 print "Warning: overwriting " \
428 "changed attr %s on %s " \
429 "from %s to %s" % \
430 (key, f.pathname,
431 f.changed_attrs[key],
432 a.attrs[key])
433 f.changed_attrs[key] = a.attrs[key]
434
435 pkgpaths = {}
436
437 def pkg_path(pkgname):
438 name = os.path.basename(pkgname)
439 if pkgname in pkgpaths:
440 return pkgpaths[name]
441 if "/" in pkgname:
442 pkgpaths[name] = os.path.realpath(pkgname)
443 return pkgname
444 else:
445 for each_path in wos_path:
446 if os.path.exists(each_path + "/" + pkgname):
447 pkgpaths[name] = each_path + "/" + pkgname
448 return pkgpaths[name]
449
450 raise RuntimeError("package %s not found" % pkgname)
451
452
453 def start_package(pkgname):
454 return package(pkgname)
455
456 def end_package(pkg):
457 pkg_branch = get_branch(pkg.name)
458 if not pkg.version:
459 pkg.version = "%s-%s" % (def_vers, pkg_branch)
460 elif "-" not in pkg.version:
461 pkg.version += "-%s" % pkg_branch
462
463 print "Package '%s'" % pkg.name
464 print " Version:", pkg.version
465 print " Description:", pkg.desc
466 print " Summary:", pkg.summary
467 print " Classification: ", ",".join(pkg.classification)
468
469 def publish_pkg(pkg):
470
471 new_pkg_name = "%s@%s" % (pkg.name, pkg.version)
472 t = trans.Transaction(def_repo, create_repo=create_repo,
473 pkg_name=new_pkg_name, noexecute=nopublish)
474
475 print " open %s" % new_pkg_name
476 transaction_id = t.open()
477
478 # Publish non-file objects first: they're easy.
479 for f in pkg.files:
480 if f.type in "dx":
481 action = actions.directory.DirectoryAction(
482 None, mode = f.mode, owner = f.owner,
483 group = f.group, path = f.pathname)
484 if hasattr(f, "changed_attrs"):
485 action.attrs.update(f.changed_attrs)
486 # chattr may have produced two path values
487 action.attrs["path"] = \
488 action.attrlist("path")[-1]
489 print " %s add dir %s %s %s %s" % (
490 pkg.name,
491 action.attrs["mode"],
492 action.attrs["owner"],
493 action.attrs["group"],
494 action.attrs["path"]
495 )
496 elif f.type == "s":
497 action = actions.link.LinkAction(None,
498 target = f.target, path = f.pathname)
499 if hasattr(f, "changed_attrs"):
500 action.attrs.update(f.changed_attrs)
501 # chattr may have produced two path values
502 action.attrs["path"] = \
503 action.attrlist("path")[-1]
504 print " %s add link %s %s" % (
505 pkg.name,
506 action.attrs["path"],
507 action.attrs["target"]
508 )
509 elif f.type == "l":
510 action = actions.hardlink.HardLinkAction(None,
511 target = f.target, path = f.pathname)
512 if hasattr(f, "changed_attrs"):
513 action.attrs.update(f.changed_attrs)
514 # chattr may have produced two path values
515 action.attrs["path"] = \
516 action.attrlist("path")[-1]
517 pkg.depend += process_link_dependencies(
518 action.attrs["path"], action.attrs["target"])
519 print " %s add hardlink %s %s" % (
520 pkg.name,
521 action.attrs["path"],
522 action.attrs["target"]
523 )
524 else:
525 continue
526
527 #
528 # If the originating package was hollow, tag this file
529 # as being global zone only.
530 #
531
532 if f.type not in "dx" and f.pathname in usedlist and \
533 usedlist[f.pathname][0] in hollow_pkgs:
534 action.attrs["opensolaris.zone"] = "global"
535 action.attrs["variant.opensolaris.zone"] = "global"
536
537 if f.type in "dx" and f.pathname in usedlist and \
538 usedlist[f.pathname][0] in hollow_pkgs and \
539 f.pathname not in pkg.nonhollow_dirs:
540 action.attrs["opensolaris.zone"] = "global"
541 action.attrs["variant.opensolaris.zone"] = "global"
542
543 # handle attribute deletion
544 if hasattr(f, "deleted_attrs"):
545 for d in f.deleted_attrs:
546 if d in action.attrs:
547 del action.attrs[d]
548
549 t.add(action)
550
551 # Group the files in a (new) package based on what (old) package they
552 # came from, so that we can iterate through all files in a single (old)
553 # package (and, therefore, in a single bzip2 archive) before moving on
554 # to the next. Because groupby() needs its input pre-sorted by group
555 # and we want to maintain the order that the files come out of the cpio
556 # archives, we coalesce the groups with the groups dictionary.
557 def fn(key):
558 return usedlist[key.pathname][0]
559 groups = {}
560 for k, g in groupby((f for f in pkg.files if f.type in "fevi"), fn):
561 if k in groups:
562 groups[k].extend(g)
563 else:
564 groups[k] = list(g)
565
566 def otherattrs(action):
567 s = " ".join(
568 "%s=%s" % (a, action.attrs[a])
569 for a in action.attrs
570 if a not in ("owner", "group", "mode", "path")
571 )
572 if s:
573 return " " + s
574 else:
575 return ""
576
577 # Maps class names to preserve attribute values.
578 preserve_dict = {
579 "renameold": "renameold",
580 "renamenew": "renamenew",
581 "preserve": "true",
582 "svmpreserve": "true"
583 }
584
585 undeps = set()
586 for g in groups.values():
587 pkgname = usedlist[g[0].pathname][0]
588 print "pulling files from archive in package", pkgname
589 bundle = SolarisPackageDirBundle(svr4pkgpaths[pkgname])
590 pathdict = dict((f.pathname, f) for f in g)
591 for f in bundle:
592 if f.name == "license":
593 if f.attrs["license"] in pkg.dropped_licenses:
594 continue
595 # add transaction id so that every version
596 # of a pkg will have a unique license to prevent
597 # license from disappearing on upgrade
598 f.attrs["transaction_id"] = transaction_id
599 # The "path" attribute is confusing and
600 # unnecessary for licenses.
601 del f.attrs["path"]
602 print " %s add license %s" % \
603 (pkg.name, f.attrs["license"])
604 t.add(f)
605 elif f.attrs["path"] in pathdict:
606 if pkgname in hollow_pkgs:
607 f.attrs["opensolaris.zone"] = "global"
608 f.attrs["variant.opensolaris.zone"] = \
609 "global"
610 path = f.attrs["path"]
611 if pathdict[path].type in "ev":
612 f.attrs["preserve"] = "true"
613 f.attrs["owner"] = pathdict[path].owner
614 f.attrs["group"] = pathdict[path].group
615 f.attrs["mode"] = pathdict[path].mode
616
617 # is this a file for which we need a timestamp?
618 basename = os.path.basename(path)
619 for file_pattern in timestamp_files:
620 if fnmatch.fnmatch(basename,
621 file_pattern):
622 break
623 else:
624 del f.attrs["timestamp"]
625 if pathdict[path].klass in preserve_dict.keys():
626 f.attrs["preserve"] = \
627 preserve_dict[pathdict[path].klass]
628 if hasattr(pathdict[path], "changed_attrs"):
629 f.attrs.update(
630 pathdict[path].changed_attrs)
631 # chattr may have produced two values
632 f.attrs["path"] = f.attrlist("path")[-1]
633
634 print " %s add file %s %s %s %s%s" % \
635 (pkg.name, f.attrs["mode"],
636 f.attrs["owner"], f.attrs["group"],
637 f.attrs["path"], otherattrs(f))
638
639 # handle attribute deletion
640 if hasattr(pathdict[path], "deleted_attrs"):
641 for d in pathdict[path].deleted_attrs:
642 if d in f.attrs:
643 print "removed %s from %s in pkg %s" % (d, path, new_pkg_name)
644 del f.attrs[d]
645
646 # Read the file in chunks to avoid a memory
647 # footprint blowout.
648 fo = f.data()
649 bufsz = 256 * 1024
650 sz = int(f.attrs["pkg.size"])
651 fd, tmp = mkstemp(prefix="pkg.")
652 while sz > 0:
653 d = fo.read(min(bufsz, sz))
654 os.write(fd, d)
655 sz -= len(d)
656 d = None
657 os.close(fd)
658
659 # Fool the action into pulling from a
660 # temporary file so that both add() and
661 # process_dependencies() can read() the
662 # data.
663 f.data = lambda: open(tmp, "rb")
664 t.add(f)
665
666 # Look for dependencies
667 deps, u = process_dependencies(tmp, path)
668 pkg.depend += deps
669 if u:
670 print \
671 "%s has missing dependencies: %s" \
672 % (path, u)
673 undeps |= set(u)
674 os.unlink(tmp)
675
676 # process any dependencies on files
677 for f in pkg.file_depend:
678 f = f.lstrip("/") # remove any leading /
679 if f in usedlist:
680 pkg.depend += [ "%s@%s" %
681 (usedlist[f][1].name,
682 usedlist[f][1].version)
683 ]
684 else:
685 print "Warning: pkg %s: depend_path %s not satisfied" \
686 % (pkg.name, f)
687 undeps.add(f)
688 # Publish dependencies
689
690 missing_cnt = 0
691
692 for p in set(pkg.idepend): # over set of svr4 deps, append ipkgs
693 if p in destpkgs:
694 pkg.depend.extend(destpkgs[p])
695 else:
696 print "pkg %s: SVR4 package %s not seen" % \
697 (pkg.name, p)
698 missing_cnt += 1
699 if missing_cnt > 0:
700 raise RuntimeError("missing packages!")
701
702 for p in set(pkg.depend) - set(pkg.undepend):
703 # Don't make a package depend on itself.
704 if p.split("@")[0] == pkg.name:
705 continue
706 # enhance unqualified dependencies to include current
707 # pkg version
708 if "@" not in p and p in pkgdict:
709 p = "%s@%s" % (p, pkgdict[p].version)
710
711 print " %s add depend require %s" % (pkg.name, p)
712 action = actions.depend.DependencyAction(None,
713 type = "require", fmri = p)
714 t.add(action)
715
716 for a in pkg.extra:
717 print " %s add %s" % (pkg.name, a)
718 action = actions.fromstr(a)
719 if hasattr(action, "hash"):
720 fname, fd = sourcehook(action.hash)
721 fd.close()
722 action.data = lambda: file(fname, "rb")
723 action.attrs["pkg.size"] = str(os.stat(fname).st_size)
724 if action.name == "license":
725 action.attrs["transaction_id"] = transaction_id
726 elif "path" in action.attrs:
727 path = action.attrs["path"]
728 deps, u = process_dependencies(fname, path)
729 pkg.depend += deps
730 if u:
731 print "%s has missing dependencies: " \
732 "%s" % (path, u)
733 undeps |= set(u)
734 #
735 # fmris may not be completely specified; enhance them to current
736 # version if this is the case
737 #
738 for attr in action.attrs:
739 if attr == "fmri" and \
740 "@" not in action.attrs[attr] and \
741 action.attrs[attr][5:] in pkgdict:
742 action.attrs[attr] += "@%s" % \
743 pkgdict[action.attrs[attr][5:]].version
744 t.add(action)
745
746 if pkg.desc:
747 print " %s add set pkg.description=%s" % (pkg.name, pkg.desc)
748 attrs = dict(name="pkg.description", value=pkg.desc)
749 action = actions.attribute.AttributeAction(None, **attrs)
750 t.add(action)
751
752 if pkg.summary:
753 print " %s add set pkg.summary=%s" % (pkg.name, pkg.summary)
754 attrs = dict(name="pkg.summary", value=pkg.summary)
755 action = actions.attribute.AttributeAction(None, **attrs)
756 t.add(action)
757
758 # Retain the description entry in the package manifest for
759 # now for backward compatibility.
760 print " %s add set description=%s" % (pkg.name, pkg.summary)
761 action = actions.attribute.AttributeAction(None,
762 description = pkg.summary)
763 t.add(action)
764
765 if pkg.classification:
766 print " %s add set info.classification=%s" % \
767 (pkg.name, pkg.classification)
768 attrs = dict(name="info.classification",
769 value=pkg.classification)
770 action = actions.attribute.AttributeAction(None, **attrs)
771 t.add(action)
772
773 if pkg.name != "SUNWipkg":
774 for p in pkg.srcpkgs:
775 try:
776 sp = svr4pkgsseen[p]
777 except KeyError:
778 continue
779
780 wanted_attrs = (
781 "PKG", "NAME", "ARCH", "VERSION", "CATEGORY",
782 "VENDOR", "DESC", "HOTLINE"
783 )
784 attrs = dict(
785 (k.lower(), v)
786 for k, v in sp.pkginfo.iteritems()
787 if k in wanted_attrs
788 )
789 attrs["pkg"] = sp.pkginfo["PKG.PLAT"]
790
791 action = actions.legacy.LegacyAction(None, **attrs)
792
793 print " %s add %s" % (pkg.name, action)
794 t.add(action)
795
796 if undeps:
797 print "Missing dependencies:", list(undeps)
798
799 print " close"
800 pkg_fmri, pkg_state = t.close(refresh_index=not defer_refresh)
801 print "%s: %s\n" % (pkg_fmri, pkg_state)
802
803 def process_link_dependencies(path, target):
804 orig_target = target
805 if target[0] != "/":
806 target = os.path.normpath(
807 os.path.join(os.path.split(path)[0], target))
808
809 if target in usedlist:
810 if show_debug:
811 print "hardlink %s -> %s makes %s depend on %s" % \
812 (
813 path, orig_target,
814 usedlist[path][1].name,
815 usedlist[target][1].name
816 )
817 return ["%s@%s" % (usedlist[target][1].name,
818 usedlist[target][1].version)]
819 else:
820 return []
821
822 def process_dependencies(fname, path):
823 if not elf.is_elf_object(fname):
824 return process_non_elf_dependencies(fname, path)
825
826 ei = elf.get_info(fname)
827 try:
828 ed = elf.get_dynamic(fname)
829 except elf.ElfError:
830 deps = []
831 rp = []
832 else:
833 deps = [
834 d[0]
835 for d in ed.get("deps", [])
836 ]
837 rp = ed.get("runpath", "").split(":")
838 if len(rp) == 1 and rp[0] == "":
839 rp = []
840
841 rp = [
842 os.path.normpath(p.replace("$ORIGIN", "/" + os.path.dirname(path)))
843 for p in rp
844 ]
845
846 kernel64 = None
847
848 # For kernel modules, default path resolution is /platform/<platform>,
849 # /kernel, /usr/kernel. But how do we know what <platform> would be for
850 # a given module? Does it do fallbacks to, say, sun4u?
851 if path.startswith("kernel") or path.startswith("usr/kernel") or \
852 (path.startswith("platform") and path.split("/")[2] == "kernel"):
853 if rp:
854 print "RUNPATH set for kernel module (%s): %s" % \
855 (path, rp)
856 # Default kernel search path
857 rp.extend(("/kernel", "/usr/kernel"))
858 # What subdirectory should we look in for 64-bit kernel modules?
859 if ei["bits"] == 64:
860 if ei["arch"] == "i386":
861 kernel64 = "amd64"
862 elif ei["arch"] == "sparc":
863 kernel64 = "sparcv9"
864 else:
865 print ei["arch"]
866 else:
867 if "/lib" not in rp:
868 rp.append("/lib")
869 if "/usr/lib" not in rp:
870 rp.append("/usr/lib")
871
872 # XXX Do we need to handle anything other than $ORIGIN? x86 images have
873 # a couple of $PLATFORM and $ISALIST instances.
874 for p in rp:
875 if "$" in p:
876 tok = p[p.find("$"):]
877 if "/" in tok:
878 tok = tok[:tok.find("/")]
879 print "%s has dynamic token %s in rpath" % (path, tok)
880
881 dep_pkgs = []
882 undeps = []
883 depend_list = []
884 for d in deps:
885 for p in rp:
886 # The instances of "[1:]" below are because usedlist
887 # stores paths without leading slash
888 if kernel64:
889 # Find 64-bit modules the way krtld does.
890 # XXX We don't resolve dependencies found in
891 # /platform, since we don't know where under
892 # /platform to look.
893 head, tail = os.path.split(d)
894 deppath = os.path.join(p,
895 head,
896 kernel64,
897 tail)[1:]
898 else:
899 # This is a hack for when a runpath uses the 64
900 # symlink to the actual 64-bit directory.
901 # Better would be to see if the runpath was a
902 # link, and if so, use its resolution, but
903 # extracting that information from used list is
904 # a pain, especially because you potentially
905 # have to resolve symlinks at all levels of the
906 # path.
907 if p.endswith("/64"):
908 if ei["arch"] == "i386":
909 p = p[:-2] + "amd64"
910 elif ei["arch"] == "sparc":
911 p = p[:-2] + "sparcv9"
912 deppath = os.path.join(p, d)[1:]
913 if deppath in usedlist:
914 dep_pkgs += [ "%s@%s" %
915 (usedlist[deppath][1].name,
916 usedlist[deppath][1].version) ]
917 depend_list.append(
918 (
919 deppath,
920 usedlist[deppath][1].name
921 )
922 )
923 break
924 else:
925 undeps += [ d ]
926
927 if show_debug:
928 print "%s makes %s depend on %s" % \
929 (path, usedlist[path][1].name, depend_list)
930
931 return dep_pkgs, undeps
932
933 def process_non_elf_dependencies(localpath, path):
934 # localpath is path to actual file
935 # path is path in installed image
936 # take 1
937 dep_pkgs = []
938 undeps = []
939
940 f = file(localpath)
941 l = f.readline()
942 f.close()
943
944 # add #!/ dependency
945 if l.startswith("#!/"):
946 # usedlist omits leading /
947 p = (l[2:].split()[0]) # first part of string is path (removes options)
948 # we don't handle dependencies through links, so fix up the common one
949 if p.startswith("/bin"):
950 p = "/usr" + p
951 if p[1:] in usedlist:
952 dep_pkgs += [ "%s@%s" % (
953 usedlist[p[1:]][1].name,
954 usedlist[p[1:]][1].version)
955 ]
956 print "Added dependency on %s because of %s" % (usedlist[p[1:]][1].name, p)
957 else:
958 undeps = [ p ]
959
960 return dep_pkgs, undeps
961
962 def zap_strings(instr, strings):
963 """takes an input string and a list of strings to be removed, ignoring
964 case"""
965 for s in strings:
966 ls = s.lower()
967 while True:
968 li = instr.lower()
969 i = li.find(ls)
970 if i < 0:
971 break
972 instr = instr[0:i] + instr[i + len(ls):]
973 return instr
974
975 def get_branch(name):
976 return branch_dict.get(name, def_branch)
977
978 def_vers = "0.5.11"
979 def_branch = ""
980 def_wos_path = ["/net/netinstall.eng/export/nv/x/latest/Solaris_11/Product"]
981 create_repo = False
982 nopublish = False
983 show_debug = False
984 print_pkg_names = False
985 def_repo = "http://localhost:10000"
986 wos_path = []
987 include_path = []
988 branch_dict = {}
989 timestamp_files = []
990
991 #
992 # files (by path) we always delete for bulk imports
993 # note that we ignore these if specifically included.
994 #
995 elided_files = {}
996 #
997 # if user uses -j, just_these_pkgs becomes list of pkgs to process
998 # allowing other arguments to be read in as files...
999 #
1000 just_these_pkgs = []
1001 #
1002 # strings to rip out of summaries (case insensitve)
1003 #
1004 summary_detritus = [", (usr)", ", (root)", " (usr)", " (root)",
1005 " (/usr)", " - / filesystem", ",root(/)"]
1006 #
1007 # list of global includes to add to every package
1008 #
1009 global_includes = []
1010 # list of macro substitutions
1011 macro_definitions = {}
1012
1013 try:
1014 _opts, _args = getopt.getopt(sys.argv[1:], "B:D:I:G:NT:b:dj:m:ns:v:w:p:")
1015 except getopt.GetoptError, _e:
1016 print "unknown option", _e.opt
1017 sys.exit(1)
1018
1019 g_proto_area = os.environ.get("ROOT", "")
1020
1021 for opt, arg in _opts:
1022 if opt == "-b":
1023 def_branch = arg.rstrip("abcdefghijklmnopqrstuvwxyz")
1024 elif opt == "-d":
1025 show_debug = True
1026 elif opt == "-j": # means we're using the new argument form...
1027 just_these_pkgs.append(arg)
1028 elif opt == "-m":
1029 _a = arg.split("=", 1)
1030 macro_definitions.update([("$(%s)" % _a[0], _a[1])])
1031 elif opt == "-n":
1032 nopublish = True
1033 elif opt == "-p":
1034 if not os.path.exists(arg):
1035 raise RuntimeError("Invalid prototype area specified.")
1036 # Clean up relative ../../, etc. out of path to proto
1037 g_proto_area = os.path.realpath(arg)
1038 elif opt == "-s":
1039 def_repo = arg
1040 if def_repo.startswith("file://"):
1041 # When publishing to file:// repositories, automatically
1042 # create the target repository if needed.
1043 create_repo = True
1044 elif opt == "-v":
1045 def_vers = arg
1046 elif opt == "-w":
1047 wos_path.append(arg)
1048 elif opt == "-D":
1049 elided_files[arg] = True
1050 elif opt == "-I":
1051 include_path.extend(arg.split(":"))
1052 elif opt == "-B":
1053 branch_file = file(arg)
1054 for _line in branch_file:
1055 if not _line.startswith("#"):
1056 bfargs = _line.split()
1057 if len(bfargs) == 2:
1058 branch_dict[bfargs[0]] = bfargs[1]
1059 branch_file.close()
1060 elif opt == "-G": #another file of global includes
1061 global_includes.append(arg)
1062 elif opt == "-N":
1063 print_pkg_names = True
1064 elif opt == "-T":
1065 timestamp_files.append(arg)
1066
1067 if not def_branch:
1068 print "need a branch id (build number)"
1069 sys.exit(1)
1070 elif "." not in def_branch:
1071 print "branch id needs to be of the form 'x.y'"
1072 sys.exit(1)
1073
1074 if not _args:
1075 print "need argument!"
1076 sys.exit(1)
1077
1078 if not wos_path:
1079 wos_path = def_wos_path
1080
1081 if just_these_pkgs:
1082 filelist = _args
1083 else:
1084 filelist = _args[0:1]
1085 just_these_pkgs = _args[1:]
1086
1087
1088 in_multiline_import = False
1089
1090 # This maps what files we've seen to a tuple of what packages they came from and
1091 # what packages they went into, so we can prevent more than one package from
1092 # grabbing the same file.
1093 usedlist = {}
1094
1095 #
1096 # pkgdict contains ipkgs by name
1097 #
1098 pkgdict = {}
1099
1100 #
1101 # destpkgs contains the list of ipkgs generated from each svr4 pkg
1102 # this is needed to generate metaclusters
1103 #
1104 destpkgs = {}
1105
1106 #
1107 #svr4 pkgs seen - pkgs indexed by name
1108 #
1109 svr4pkgsseen = {}
1110
1111 #
1112 #paths where we found the packages we need
1113 #
1114 svr4pkgpaths = {}
1115
1116 #
1117 # editable files and where they're found
1118 #
1119 editable_files = {}
1120
1121 #
1122 # hollow svr4 packages processed
1123 #
1124 hollow_pkgs = {}
1125
1126
1127 reuse_err = \
1128 "Conflict in path %s: IPS %s SVR4 %s from %s with IPS %s SVR4 %s from %s"
1129
1130
1131 # First pass: don't actually publish anything, because we're not collecting
1132 # dependencies here.
1133 def read_full_line(lexer, continuation='\\'):
1134 """Read a complete line, allowing for the possibility of it being
1135 continued over multiple lines. Returns a single joined line, with
1136 continuation characters and leading and trailing spaces removed.
1137 """
1138
1139 lines = []
1140 while True:
1141 line = lexer.instream.readline().strip()
1142 lexer.lineno = lexer.lineno + 1
1143 if line[-1] in continuation:
1144 lines.append(line[:-1])
1145 else:
1146 lines.append(line)
1147 break
1148
1149 return apply_macros(' '.join(lines))
1150
1151 def apply_macros(s):
1152 """Apply macro subs defined on command line... keep applying
1153 macros until no translations are found. If macro translates
1154 to a comment, replace entire token text."""
1155 while s and "$(" in s:
1156 for key in macro_definitions.keys():
1157 if key in s:
1158 value = macro_definitions[key]
1159 if value == "#": # comment character
1160 s = "#" # affects whole token
1161 break
1162 s = s.replace(key, value)
1163 break # look for more substitutions
1164 else:
1165 break # no more substitutable tokens
1166 return s
1167
1168 def sourcehook(filename):
1169 """ implement include hierarchy """
1170 for i in include_path:
1171 f = os.path.join(i, filename)
1172 if os.path.exists(f):
1173 return (f, open(f))
1174
1175 return filename, open(filename)
1176
1177 class tokenlexer(shlex.shlex):
1178 def read_token(self):
1179 """ simple replacement of $(ARCH) with a non-special
1180 value defined on the command line is trivial. Since
1181 shlex's read_token routine also strips comments and
1182 white space, this read_token cannot return either
1183 one so any macros that translate to either spaces or
1184 # (comment) need to be removed from the token stream."""
1185
1186 while True:
1187 s = apply_macros(shlex.shlex.read_token(self))
1188 if s == "#": # discard line if comment; try again
1189 self.instream.readline()
1190 self.lineno = self.lineno + 1
1191 # bail on EOF or not space; loop on space
1192 elif s == None or (s != "" and not s.isspace()):
1193 break
1194 return s
1195
1196 curpkg = None
1197 def SolarisParse(mf):
1198 global curpkg
1199 global in_multiline_import
1200
1201 lexer = tokenlexer(file(mf), mf, True)
1202 lexer.whitespace_split = True
1203 lexer.source = "include"
1204 lexer.sourcehook = sourcehook
1205
1206 while True:
1207 token = lexer.get_token()
1208
1209 if not token:
1210 break
1211
1212 if token == "package":
1213 curpkg = start_package(lexer.get_token())
1214
1215 if print_pkg_names:
1216 print "-j %s" % curpkg.name
1217
1218 elif token == "end":
1219 endarg = lexer.get_token()
1220 if endarg == "package":
1221 if print_pkg_names:
1222 curpkg = None
1223 continue
1224
1225 for filename in global_includes:
1226 for i in include_path:
1227 f = os.path.join(i, filename)
1228 if os.path.exists(f):
1229 SolarisParse(f)
1230 break
1231 else:
1232 raise RuntimeError("File not "
1233 "found: %s" % filename)
1234 try:
1235 end_package(curpkg)
1236 except Exception, e:
1237 print "ERROR(end_pkg):", e
1238
1239 curpkg = None
1240 if endarg == "import":
1241 in_multiline_import = False
1242 curpkg.imppkg = None
1243
1244 elif token == "version":
1245 curpkg.version = lexer.get_token()
1246
1247 elif token == "import":
1248 package_name = lexer.get_token()
1249 next = lexer.get_token()
1250 if next != "exclude":
1251 line = ""
1252 lexer.push_token(next)
1253 else:
1254 line = read_full_line(lexer)
1255
1256 if not print_pkg_names:
1257 curpkg.import_pkg(package_name, line)
1258
1259 elif token == "from":
1260 pkgspec = lexer.get_token()
1261 if not print_pkg_names:
1262 p = SolarisPackage(pkg_path(pkgspec))
1263 curpkg.imppkg = p
1264 spkgname = p.pkginfo["PKG.PLAT"]
1265 svr4pkgpaths[spkgname] = pkg_path(pkgspec)
1266 svr4pkgsseen[spkgname] = p
1267 curpkg.add_svr4_src(spkgname)
1268
1269 junk = lexer.get_token()
1270 assert junk == "import"
1271 in_multiline_import = True
1272
1273 elif token == "classification":
1274 cat_subcat = lexer.get_token()
1275 curpkg.classification.append(
1276 "org.opensolaris.category.2008:%s" % cat_subcat)
1277
1278 elif token == "description":
1279 curpkg.desc = lexer.get_token()
1280
1281 elif token == "summary":
1282 curpkg.summary = lexer.get_token()
1283
1284 elif token == "depend":
1285 curpkg.depend.append(lexer.get_token())
1286
1287 elif token == "depend_path":
1288 curpkg.file_depend.append(lexer.get_token())
1289
1290 elif token == "cluster":
1291 curpkg.add_svr4_src(lexer.get_token())
1292
1293 elif token == "idepend":
1294 curpkg.idepend.append(lexer.get_token())
1295
1296 elif token == "undepend":
1297 curpkg.undepend.append(lexer.get_token())
1298
1299 elif token == "add":
1300 curpkg.extra.append(read_full_line(lexer))
1301
1302 elif token == "drop":
1303 f = lexer.get_token()
1304 if print_pkg_names:
1305 continue
1306 l = [o for o in curpkg.files if o.pathname == f]
1307 if not l:
1308 print "Cannot drop '%s' from '%s': not " \
1309 "found" % (f, curpkg.name)
1310 else:
1311 del curpkg.files[curpkg.files.index(l[0])]
1312 # XXX The problem here is that if we do this on
1313 # a shared file (directory, etc), then it's
1314 # missing from usedlist entirely, since we don't
1315 # keep around *all* packages delivering a shared
1316 # file, just the last seen. This probably
1317 # doesn't matter much.
1318 del usedlist[f]
1319
1320 elif token == "drop_license":
1321 curpkg.dropped_licenses.append(lexer.get_token())
1322
1323 elif token == "chattr":
1324 fname = lexer.get_token()
1325 line = read_full_line(lexer)
1326 if print_pkg_names:
1327 continue
1328 try:
1329 curpkg.chattr(fname, line)
1330 except Exception, e:
1331 print "Can't change attributes on " + \
1332 "'%s': not in the package" % fname, e
1333 raise
1334
1335 elif token == "chattr_glob":
1336 glob = lexer.get_token()
1337 line = read_full_line(lexer)
1338 if print_pkg_names:
1339 continue
1340 try:
1341 curpkg.chattr_glob(glob, line)
1342 except Exception, e:
1343 print "Can't change attributes on " + \
1344 "'%s': no matches in the package" % \
1345 glob, e
1346 raise
1347
1348 elif in_multiline_import:
1349 next = lexer.get_token()
1350 if next == "with":
1351 # I can't imagine this is supported, but there's
1352 # no other way to read the rest of the line
1353 # without a whole lot more pain.
1354 line = read_full_line(lexer)
1355 else:
1356 lexer.push_token(next)
1357 line = ""
1358
1359 try:
1360 curpkg.import_file(token, line)
1361 except Exception, e:
1362 print "ERROR(import_file):", e
1363 raise
1364 else:
1365 raise RuntimeError("Error: unknown token '%s' "
1366 "(%s:%s)" % (token, lexer.infile, lexer.lineno))
1367
1368 if print_pkg_names:
1369 for _mf in filelist:
1370 SolarisParse(_mf)
1371 sys.exit(0)
1372
1373
1374 print "First pass:", datetime.now()
1375
1376 for _mf in filelist:
1377 SolarisParse(_mf)
1378
1379 seenpkgs = set(i[0] for i in usedlist.values())
1380
1381 print "Files you seem to have forgotten:\n " + "\n ".join(
1382 "%s %s" % (f.type, f.pathname)
1383 for pkg in seenpkgs
1384 for f in svr4pkgsseen[pkg].manifest
1385 if f.type != "i" and f.pathname not in usedlist)
1386
1387 print "\n\nDuplicate Editables files list:\n"
1388
1389 if editable_files:
1390 length = 2 + max(len(p) for p in editable_files)
1391 for paths in editable_files:
1392 if len(editable_files[paths]) > 1:
1393 print ("%s:" % paths).ljust(length - 1) + \
1394 ("\n".ljust(length)).join("%s (from %s)" % \
1395 (l[1].name, l[0]) for l in editable_files[paths])
1396
1397
1398 # Second pass: iterate over the existing package objects, gathering dependencies
1399 # and publish!
1400
1401 print "Second pass:", datetime.now()
1402
1403 print "New packages:\n"
1404 # XXX Sort these. Preferably topologically, if possible, alphabetically
1405 # otherwise (for a rough progress gauge).
1406 if just_these_pkgs:
1407 newpkgs = set(pkgdict[name]
1408 for name in pkgdict.keys()
1409 if name in just_these_pkgs
1410 )
1411 else:
1412 newpkgs = set(pkgdict.values())
1413
1414 # Indicates whether search indices refresh will be deferred until the end.
1415 defer_refresh = False
1416 # Indicates whether local publishing is active.
1417 local_publish = False
1418 if def_repo.startswith("file:"):
1419 # If publishing to disk, the search indices should be refreshed at
1420 # the end of the publishing process and the feed cache will have to be
1421 # generated by starting the depot server using the provided path and
1422 # then accessing it.
1423 defer_refresh = True
1424 local_publish = True
1425
1426 processed = 0
1427 total = len(newpkgs)
1428 for _p in sorted(newpkgs):
1429 print "Package '%s'" % _p.name
1430 print " Version:", _p.version
1431 print " Description:", _p.desc
1432 print " Summary:", _p.summary
1433 print " Classification:", ",".join(_p.classification)
1434 try:
1435 publish_pkg(_p)
1436 except trans.TransactionError, _e:
1437 print "%s: FAILED: %s\n" % (_p.name, _e)
1438 processed += 1
1439 print "%d/%d packages processed; %.2f%% complete" % (processed, total,
1440 processed * 100.0 / total)
1441
1442 if not nopublish and defer_refresh:
1443 # This has to be done at the end for some publishing modes.
1444 print "Updating search indices..."
1445 _t = trans.Transaction(def_repo)
1446 _t.refresh_index()
1447
1448 # Ensure that the feed is updated and cached to reflect changes.
1449 if not nopublish:
1450 print "Caching RSS/Atom feed..."
1451 dc = None
1452 durl = def_repo
1453 if local_publish:
1454 # The depot server isn't already running, so will have to be
1455 # temporarily started to allow proper feed cache generation.
1456 dc = depotcontroller.DepotController()
1457 dc.set_depotd_path(g_proto_area + "/usr/lib/pkg.depotd")
1458 dc.set_depotd_content_root(g_proto_area + "/usr/share/lib/pkg")
1459
1460 _scheme, _netloc, _path, _params, _query, _fragment = \
1461 urlparse.urlparse(def_repo, "file", allow_fragments=0)
1462
1463 dc.set_repodir(_path)
1464
1465 # XXX There must be a better way...
1466 dc.set_port(29083)
1467
1468 # Start the depot
1469 dc.start()
1470
1471 durl = "http://localhost:29083"
1472
1473 _f = urllib.urlopen("%s/feed" % durl)
1474 _f.close()
1475
1476 if dc:
1477 dc.stop()
1478 dc = None
1479
1480 print "Done:", datetime.now()