--- /dev/null Wed Jul 1 14:56:24 2009 +++ new/src/cacert/Verisign_Class_3_Public_Primary_Certification_Authority-G2.pem Wed Jul 1 14:56:24 2009 @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4 +pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0 +13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk +U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i +F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY +oJ2daZH9 +-----END CERTIFICATE----- --- old/src/client.py Wed Jul 1 14:56:24 2009 +++ new/src/client.py Wed Jul 1 14:56:24 2009 @@ -75,9 +75,6 @@ from pkg.client.history import (RESULT_CANCELED, RESULT_FAILED_BAD_REQUEST, RESULT_FAILED_CONFIGURATION, RESULT_FAILED_TRANSPORT, RESULT_FAILED_UNKNOWN, RESULT_FAILED_OUTOFMEMORY) -from pkg.client.filelist import FileListRetrievalError -from pkg.client.retrieve import (CatalogRetrievalError, - DatastreamRetrievalError, ManifestRetrievalError) from pkg.misc import EmptyI, msg, emsg, PipeError CLIENT_API_VERSION = 15 @@ -574,7 +571,6 @@ return 1 except (api_errors.CertificateError, api_errors.PlanCreationException, - api_errors.NetworkUnavailableException, api_errors.PermissionsException), e: # Prepend a newline because otherwise the exception will # be printed on the same line as the spinner. @@ -599,10 +595,12 @@ # XXX would be nice to kick the progress tracker. try: api_inst.prepare() - except misc.TransportException: + except api_errors.TransportError, e: # move past the progress tracker line. msg("\n") - raise + if verbose: + e.verbose = True + raise e except KeyboardInterrupt: raise except api_errors.PermissionsException, e: @@ -736,7 +734,6 @@ return 1 except (api_errors.CertificateError, api_errors.PlanCreationException, - api_errors.NetworkUnavailableException, api_errors.PermissionsException), e: # Prepend a newline because otherwise the exception will # be printed on the same line as the spinner. @@ -758,10 +755,12 @@ # XXX would be nice to kick the progress tracker. try: api_inst.prepare() - except misc.TransportException: + except api_errors.TransportError, e: # move past the progress tracker line. msg("\n") - raise + if verbose: + e.verbose = True + raise e except KeyboardInterrupt: raise except api_errors.PermissionsException, e: @@ -879,10 +878,12 @@ # XXX would be nice to kick the progress tracker. try: api_inst.prepare() - except misc.TransportException: + except api_errors.TransportError, e: # move past the progress tracker line. msg("\n") - raise + if verbose: + e.verbose = True + raise e except api_errors.FileInUseException, e: error("\n" + str(e)) return 1 @@ -933,30 +934,6 @@ state.""" return 0 -def process_v_0_search(tup, first): - """Transforms the tuples returned by search v1 into the four column - output format. - - The "tup" parameter is a four tuple with the each entry corresponding - to a column of the output. - - The "first" parameter is a boolean stating whether this is the first - time this function has been called. This controls the printing of the - header information.""" - - try: - index, mfmri, action, value = tup - except ValueError: - error(_("The server returned a malformed result.\n" - "The problematic structure: %r") % (tup,)) - return False - if first: - msg("%-10s %-9s %-25s %s" % - ("INDEX", "ACTION", "VALUE", "PACKAGE")) - msg("%-10s %-9s %-25s %s" % (index, action, value, - fmri.PkgFmri(str(mfmri)).get_short_fmri())) - return True - def __convert_output(a_str, match): """Converts a string to a three tuple with the information to fill the INDEX, ACTION, and VALUE columns. @@ -1020,8 +997,13 @@ if first: msg("%s" % ("PACKAGE")) pub_name = '' - if pub is not None and "prefix" in pub: + # If pub is not None, it's either a RepositoryURI or a Publisher + # object. If it's a Publisher, it has a prefix. Otherwise, + # use the uri. + if pub is not None and hasattr(pub, "prefix"): pub_name = " (%s)" % pub.prefix + elif pub is not None and hasattr(pub, "uri"): + pub_name = " (%s)" % pub.uri msg("%s%s" % (fmri.PkgFmri(str(pfmri)).get_short_fmri(), pub_name)) return True @@ -1100,11 +1082,8 @@ "result:%r") % (raw_value,)) bad_res = True continue - if v == 0: - ret = process_v_0_search(tmp, first) - else: - ret = process_v_1_search(tmp, first, - return_type, pub) + ret = process_v_1_search(tmp, first, + return_type, pub) good_res |= ret bad_res |= not ret first = False @@ -1613,20 +1592,6 @@ else: emsg(" %s: %s" % \ (pub["origin"], err.args[0][1])) - elif isinstance(err, CatalogRetrievalError) and \ - isinstance(err.exc, EnvironmentError) and \ - err.exc.errno == errno.EACCES: - if err.prefix: - emsg(" ", _("Could not update catalog " - "for '%s' due to insufficient " - "permissions.") % err.prefix) - else: - emsg(" ", _("Could not update a catalog " - "due to insufficient permissions.")) - - emsg(" ", _("Please try the command again " - "using pfexec, or otherwise increase \n your " - "permissions.")) else: emsg(" ", err) @@ -1661,8 +1626,7 @@ error(e) error(_("'pkg publisher' will show a list of publishers.")) return 1 - except (api_errors.PermissionsException, - api_errors.NetworkUnavailableException), e: + except (api_errors.PermissionsException), e: # Prepend a newline because otherwise the exception will # be printed on the same line as the spinner. error("\n" + str(e)) @@ -2289,6 +2253,7 @@ "\nAdditional details:\n\n%(error)s") % { "pub_url": pub_url, "error": e }, cmd="image-create") + print_proxy_config() return 1 except api_errors.CatalogRefreshException, cre: # Ensure messages are displayed after the spinner. @@ -2428,6 +2393,25 @@ return 0 +def print_proxy_config(): + """If the user has configured http_proxy or https_proxy in the + environment, print out the values. Some transport errors are + not debuggable without this information handy.""" + + http_proxy = os.environ.get("http_proxy", None) + https_proxy = os.environ.get("https_proxy", None) + + if not http_proxy and not https_proxy: + return + + emsg(_("\nThe following proxy configuration is set in the" + " environment:\n")) + if http_proxy: + emsg(_("http_proxy: %s\n") % http_proxy) + if https_proxy: + emsg(_("https_proxy: %s\n") % https_proxy) + + # To allow exception handler access to the image. __img = None @@ -2472,14 +2456,18 @@ elif not subcommand: usage() - socket.setdefaulttimeout( - int(os.environ.get("PKG_CLIENT_TIMEOUT", "30"))) # in seconds - - # Override default PKG_TIMEOUT_MAX if a value has been specified - # in the environment. + # Override default PKG_TIMEOUT_MAX and PKG_CLIENT_TIMEOUT + # if a value has been specified in the environment. global_settings.PKG_TIMEOUT_MAX = int(os.environ.get("PKG_TIMEOUT_MAX", global_settings.PKG_TIMEOUT_MAX)) + global_settings.PKG_CLIENT_TIMEOUT = int(os.environ.get( + "PKG_CLIENT_TIMEOUT", global_settings.PKG_CLIENT_TIMEOUT)) + + # This call only affects sockets created by Python. The transport + # framework must set the timeout value internally. + socket.setdefaulttimeout(global_settings.PKG_TIMEOUT_MAX) # in seconds + if subcommand == "image-create": if "mydir" in locals(): usage(_("-R not allowed for %s subcommand") % @@ -2637,29 +2625,24 @@ __img.history.abort(RESULT_FAILED_BAD_REQUEST) error(__e) __ret = 1 - except misc.TransportException, __e: + except api_errors.TransportError, __e: if __img: __img.history.abort(RESULT_FAILED_TRANSPORT) - error(_("\nMaximum number of network retries exceeded during " - "download. Details follow:\n%s") % __e) + emsg(_("\nErrors were encountered while attempting to retrieve" + " package or file data for\nthe requested operation.")) + emsg(_("Details follow:\n\n%s") % __e) + print_proxy_config() __ret = 1 - except (ManifestRetrievalError, - DatastreamRetrievalError, FileListRetrievalError), __e: - if __img: - __img.history.abort(RESULT_FAILED_TRANSPORT) - error(_("An error was encountered while attempting to retrieve" - " package or file data for the requested operation.")) - error(__e) - __ret = 1 except api_errors.InvalidDepotResponseException, __e: if __img: __img.history.abort(RESULT_FAILED_TRANSPORT) - error(_("\nUnable to contact a valid package depot. " + emsg(_("\nUnable to contact a valid package depot. " "This may be due to a problem with the server, " "network misconfiguration, or an incorrect pkg client " "configuration. Please check your network settings and " "attempt to contact the server using a web browser.")) - error(_("\nAdditional details:\n\n%s") % __e) + emsg(_("\nAdditional details:\n\n%s") % __e) + print_proxy_config() __ret = 1 except history.HistoryLoadException, __e: # Since a history related error occurred, discard all --- old/src/depot.py Wed Jul 1 14:56:26 2009 +++ new/src/depot.py Wed Jul 1 14:56:26 2009 @@ -128,9 +128,9 @@ print """\ Usage: /usr/lib/pkg.depotd [-d repo_dir] [-p port] [-s threads] [-t socket_timeout] [--cfg-file] [--content-root] [--debug] - [--log-access dest] [--log-errors dest] [--mirror] [--proxy-base url] - [--readonly] [--rebuild] [--ssl-cert-file] [--ssl-dialog] - [--ssl-key-file] [--writable-root dir] + [--log-access dest] [--log-errors dest] [--mirror] [--nasty] + [--proxy-base url] [--readonly] [--rebuild] [--ssl-cert-file] + [--ssl-dialog] [--ssl-key-file] [--writable-root dir] --cfg-file The pathname of the file from which to read and to write configuration information. @@ -153,6 +153,11 @@ --mirror Package mirror mode; publishing and metadata operations disallowed. Cannot be used with --readonly or --rebuild. + --nasty Instruct the server to misbehave. At random intervals + it will time-out, send bad responses, hang up on + clients, and generally be hostile. The option + takes a value (1 to 100) for how nasty the server + should be. --proxy-base The url to use as the base for generating internal redirects and content. --readonly Read-only operation; modifying operations disallowed. @@ -202,6 +207,8 @@ reindex = REINDEX_DEFAULT proxy_base = None mirror = MIRROR_DEFAULT + nasty = False + nasty_value = 0 repo_config_file = None ssl_cert_file = None ssl_key_file = None @@ -238,9 +245,9 @@ opt = None try: long_opts = ["cfg-file=", "content-root=", "debug=", "mirror", - "proxy-base=", "readonly", "rebuild", "refresh-index", - "ssl-cert-file=", "ssl-dialog=", "ssl-key-file=", - "writable-root="] + "nasty=", "proxy-base=", "readonly", "rebuild", + "refresh-index", "ssl-cert-file=", "ssl-dialog=", + "ssl-key-file=", "writable-root="] for opt in log_opts: long_opts.append("%s=" % opt.lstrip('--')) opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:", @@ -296,6 +303,19 @@ log_routes[opt.lstrip("--log-")] = arg elif opt == "--mirror": mirror = True + elif opt == "--nasty": + value_err = None + try: + nasty_value = int(arg) + except ValueError, e: + value_err = e + + if value_err or (nasty_value > 100 or + nasty_value < 1): + raise OptionError, "Invalid value " \ + "for nasty option.\n Please " \ + "choose a value between 1 and 100." + nasty = True elif opt == "--proxy-base": # Attempt to decompose the url provided into # its base parts. This is done so we can @@ -443,9 +463,15 @@ fork_allowed = not reindex - scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT, - auto_create=not readonly, fork_allowed=fork_allowed, - writable_root=writable_root) + if nasty: + scfg = config.NastySvrConfig(repo_path, content_root, + AUTH_DEFAULT, auto_create=not readonly, + fork_allowed=fork_allowed, writable_root=writable_root) + scfg.set_nasty(nasty_value) + else: + scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT, + auto_create=not readonly, fork_allowed=fork_allowed, + writable_root=writable_root) if readonly: scfg.set_read_only() @@ -453,6 +479,7 @@ if mirror: scfg.set_mirror() + try: scfg.init_dirs() except (errors.SvrConfigError, EnvironmentError), _e: @@ -628,8 +655,12 @@ sys.exit(1) try: - root = cherrypy.Application(depot.DepotHTTP(scfg, - repo_config_file)) + if nasty: + root = cherrypy.Application(depot.NastyDepotHTTP(scfg, + repo_config_file)) + else: + root = cherrypy.Application(depot.DepotHTTP(scfg, + repo_config_file)) except rc.InvalidAttributeValueError, _e: emsg("pkg.depotd: repository.conf error: %s" % _e) sys.exit(1) --- old/src/gui/modules/installupdate.py Wed Jul 1 14:56:27 2009 +++ new/src/gui/modules/installupdate.py Wed Jul 1 14:56:27 2009 @@ -36,7 +36,6 @@ import traceback import string from threading import Thread -from urllib2 import URLError try: import gobject import gtk @@ -52,11 +51,7 @@ nobe = True import pkg.client.progress as progress import pkg.misc -from pkg.client.retrieve import ManifestRetrievalError -from pkg.client.retrieve import DatastreamRetrievalError -from pkg.client.filelist import FileListRetrievalError import pkg.client.api_errors as api_errors -from pkg.misc import TransferTimedOutException, TransportException import pkg.gui.beadmin as beadm import pkg.gui.misc as gui_misc import pkg.gui.enumerations as enumerations @@ -434,10 +429,7 @@ msg = e.message self.__g_error_stage(msg) return - except (api_errors.NetworkUnavailableException, - TransferTimedOutException, TransportException, URLError, - ManifestRetrievalError, DatastreamRetrievalError, - FileListRetrievalError), ex: + except api_errors.TransportError, ex: msg = _("Please check the network " "connection.\nIs the repository accessible?\n\n" "%s") % str(ex) --- old/src/modules/actions/generic.py Wed Jul 1 14:56:28 2009 +++ new/src/modules/actions/generic.py Wed Jul 1 14:56:28 2009 @@ -39,7 +39,6 @@ except AttributeError: os.SEEK_SET, os.SEEK_CUR, os.SEEK_END = range(3) import pkg.actions -import pkg.client.retrieve as retrieve import pkg.portable as portable class Action(object): @@ -399,7 +398,7 @@ return None def opener(): - return retrieve.get_datastream(img, fmri, self.hash) + return img.transport.get_datastream(fmri, self.hash) return opener --- old/src/modules/catalog.py Wed Jul 1 14:56:29 2009 +++ new/src/modules/catalog.py Wed Jul 1 14:56:29 2009 @@ -36,7 +36,6 @@ import bisect import pkg.fmri as fmri -from pkg.misc import TruncatedTransferException import pkg.portable as portable import pkg.version as version @@ -683,15 +682,12 @@ return self.attrs.get("origin", None) @classmethod - def recv(cls, filep, path, pub=None, content_size=-1): + def recv(cls, filep, path, pub=None): """A static method that takes a file-like object and a path. This is the other half of catalog.send(). It reads a stream as an incoming catalog and lays it down - on disk. Content_size is the size in bytes, if known, - of the transfer that is being received. The default - value of -1 means that the size is not known.""" + on disk.""" - size = 0 bad_fmri = None if not os.path.exists(path): @@ -706,50 +702,46 @@ attrpath_final = os.path.normpath(os.path.join(path, "attrs")) catpath_final = os.path.normpath(os.path.join(path, "catalog")) - for s in filep: - slen = len(s) - size += slen + try: + for s in filep: + slen = len(s) - # If line is too short, process the next one - if slen < 2: - continue - # check that line is in the proper format - elif not s[1].isspace(): - continue - elif not s[0] in known_prefixes: - catf.write(s) - elif s.startswith("S "): - attrf.write(s) - elif s.startswith("R "): - catf.write(s) - else: - # XXX Need to be able to handle old and new - # format catalogs. - try: - f = fmri.PkgFmri(s[2:]) - except fmri.IllegalFmri, e: - bad_fmri = e + # If line is too short, process the next one + if slen < 2: continue + # check that line is in the proper format + elif not s[1].isspace(): + continue + elif not s[0] in known_prefixes: + catf.write(s) + elif s.startswith("S "): + attrf.write(s) + elif s.startswith("R "): + catf.write(s) + else: + # XXX Need to be able to handle old and + # new format catalogs. + try: + f = fmri.PkgFmri(s[2:]) + except fmri.IllegalFmri, e: + bad_fmri = e + continue - catf.write("%s %s %s %s\n" % - (s[0], "pkg", f.pkg_name, f.version)) - - # Check that content was properly received before - # modifying any files. - if content_size > -1 and size != content_size: - url = None - if hasattr(filep, "geturl") and callable(filep.geturl): - url = filep.geturl() + catf.write("%s %s %s %s\n" % + (s[0], "pkg", f.pkg_name, + f.version)) + except: + # Re-raise all uncaught exceptions after performing + # cleanup. attrf.close() catf.close() os.remove(attrpath) os.remove(catpath) - raise TruncatedTransferException(url, size, - content_size) + raise # If we got a parse error on FMRIs and transfer # wasn't truncated, raise a FmriFailures error. - elif bad_fmri: + if bad_fmri: attrf.close() catf.close() os.remove(attrpath) --- old/src/modules/client/__init__.py Wed Jul 1 14:56:29 2009 +++ new/src/modules/client/__init__.py Wed Jul 1 14:56:29 2009 @@ -20,7 +20,7 @@ # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. __all__ = ["global_settings"] @@ -32,6 +32,7 @@ def __init__(self): object.__init__(self) self.PKG_TIMEOUT_MAX = 4 + self.PKG_CLIENT_TIMEOUT = 30 self.client_name = None global_settings = GlobalSettings() --- old/src/modules/client/api.py Wed Jul 1 14:56:30 2009 +++ new/src/modules/client/api.py Wed Jul 1 14:56:30 2009 @@ -26,9 +26,7 @@ # import copy -import httplib import os -import socket import simplejson as json import StringIO import sys @@ -51,7 +49,6 @@ from pkg.client.imageplan import EXECUTED_OK from pkg.client import global_settings -from pkg.misc import versioned_urlopen CURRENT_API_VERSION = 15 CURRENT_P5I_VERSION = 1 @@ -136,10 +133,10 @@ two things. The first is a boolean which tells the client whether there is anything to do. The third is either None, or an exception which indicates partial success. It can raise - PlanCreationException, NetworkUnavailableException, - PermissionsException and InventoryException. The noexecute - argument is included for compatibility with operational - history. The hope is it can be removed in the future.""" + PlanCreationException, PermissionsException and + InventoryException. The noexecute argument is included + for compatibility with operational history. + The hope is it can be removed in the future.""" self.__activity_lock.acquire() try: @@ -163,6 +160,8 @@ progtrack=self.progresstracker) except KeyboardInterrupt: raise + except api_errors.InvalidDepotResponseException: + raise except: # Since this is not a refresh # that was explicitly requested, @@ -286,8 +285,8 @@ either None, or an exception which indicates partial success. This is currently used to indicate a failure in refreshing catalogs. It can raise CatalogRefreshException, - IpkgOutOfDateException, NetworkUnavailableException, - PlanCreationException and PermissionsException.""" + IpkgOutOfDateException, PlanCreationException and + PermissionsException.""" self.__activity_lock.acquire() try: @@ -314,6 +313,8 @@ progtrack=self.progresstracker) except KeyboardInterrupt: raise + except api_errors.InvalidDepotResponseException: + raise except: # Since this is not a refresh # that was explicitly requested, @@ -444,6 +445,7 @@ raise if self.__canceling: + self.img.transport.reset() self.img.cleanup_downloads() raise api_errors.CanceledException() self.prepared = True @@ -601,6 +603,13 @@ if self.img.history.operation_name: self.log_operation_end() self.executed = True + try: + if int(os.environ.get("PKG_DUMP_STATS", 0)) > 0: + self.img.transport.stats.dump() + except ValueError: + # Don't generate stats if an invalid value + # is supplied. + pass finally: self.__activity_lock.release() @@ -671,6 +680,19 @@ the constants specified in the class definition. The values are lists of PackageInfo objects or strings.""" + # Currently, this is mostly a wapper for activity locking. + self.__activity_lock.acquire() + try: + i = self._info_op(fmri_strings, local, info_needed) + finally: + self.__activity_lock.release() + + return i + + def _info_op(self, fmri_strings, local, info_needed): + """Performs the actual info operation. The external + interface to the API's consumers is defined in info().""" + bad_opts = info_needed - PackageInfo.ALL_OPTIONS if bad_opts: raise api_errors.UnrecognizedOptionsToInfo(bad_opts) @@ -996,127 +1018,85 @@ a list of servers to search against. It performs each query against each server and yields the results in turn. If no servers are provided, the search is conducted against all - active servers known by the image.""" + active servers known by the image. + The servers argument is a list of servers in two possible + forms: the old deprecated form of a publisher, in a + dictionary, or a Publisher object. """ + failed = [] invalid = [] + unsupported = [] if not servers: servers = self.img.gen_publishers() - single = True - - if not len(query_str_and_args_lst) == 1: - single = False - - allow_version_zero = single - - version_list = [1] - - if single: - method = "GET" - q = query_str_and_args_lst[0] - qs = [urllib.quote(str(q), safe='')] - version_1_data = None - l = query_p.QueryLexer() - l.build() - qp = query_p.QueryParser(l) - # Parse the query to determine whether it can be - # represented in search/0 syntax. - try: - query = qp.parse(q.encoded_text()) - except query_p.BooleanQueryException, e: - raise api_errors.BooleanQueryException(e) - except query_p.ParseError, e: - raise api_errors.ParseError(e) - if query.allow_version(0): - version_list.append(0) - qs.append(urllib.quote(q.ver_0(), safe='')) - - else: - method = "POST" - qs = None - version_1_data = urllib.urlencode( - [(i, str(q)) - for i, q in enumerate(query_str_and_args_lst)]) - for pub in servers: - prefix = None - uuid = None - if not isinstance(pub, publisher.Publisher): + descriptive_name = None + + if isinstance(pub, dict): origin = pub["origin"] try: pub = self.img.get_publisher( origin=origin) except api_errors.UnknownPublisher: - pass - # This cannot be an else statment to the previous if - # clause because it needs to work on the value of pub - # after it has been set by self.img.get_publisher. - if isinstance(pub, publisher.Publisher): - repo = pub.selected_repository - origin = pub.selected_repository.origins[0] - prefix = pub.prefix - uuid = pub.client_uuid + pub = publisher.RepositoryURI(origin) + descriptive_name = origin - ssl_key = None - ssl_cert = None - if isinstance(origin, publisher.RepositoryURI): - ssl_key = origin.ssl_key - ssl_cert = origin.ssl_cert - origin = origin.uri - if ssl_cert: - try: - misc.validate_ssl_cert(ssl_cert, - prefix=prefix, uri=origin) - except api_errors.CertificateError, e: - failed.append((pub, e)) - continue - ssl_tuple = (ssl_key, ssl_cert) + if not descriptive_name: + descriptive_name = pub.prefix + try: - res, v = versioned_urlopen(origin, - "search", version_list, tail=qs, - data=version_1_data, - ssl_creds=ssl_tuple, imgtype=self.img.type, - method=method, uuid=uuid) - except urllib2.HTTPError, e: - if e.code != httplib.NOT_FOUND and \ - e.code != httplib.NO_CONTENT: - failed.append((pub, e)) + res = self.img.transport.do_search(pub, + query_str_and_args_lst) + except api_errors.NegativeSearchResult: continue - except (urllib2.URLError, httplib.BadStatusLine, - httplib.IncompleteRead, RuntimeError, - ValueError), e: - failed.append((pub, e)) + except api_errors.TransportError, e: + failed.append((descriptive_name, e)) continue - except KeyboardInterrupt: - raise - except Exception, e: - failed.append((pub, "Could not perform search." - "\nException: str:%s repr:%r" % (e, e))) + except api_errors.UnsupportedSearchError, e: + unsupported.append((descriptive_name, e)) continue + except api_errors.MalformedSearchRequest, e: + ex = self._validate_search( + query_str_and_args_lst) + if ex: + raise ex + failed.append((descriptive_name, e)) + continue try: - if v == 0: - for line in res: - yield self.__parse_v_0(line, - pub, v) - else: - if not self.validate_response(res, v): - invalid.append(pub) - continue - for line in res: - yield self.__parse_v_1(line, - pub, v) - except (socket.timeout, socket.error, ValueError, - httplib.IncompleteRead, - api_errors.ServerReturnError), e: - failed.append((pub, e)) + if not self.validate_response(res, 1): + invalid.append(descriptive_name) + continue + for line in res: + yield self.__parse_v_1(line, pub, 1) + except api_errors.TransportError, e: + failed.append((descriptive_name, e)) continue - if failed or invalid: + + if failed or invalid or unsupported: raise api_errors.ProblematicSearchServers(failed, - invalid) + invalid, unsupported) + def _validate_search(self, query_str_lst): + """Called by remote search if server responds that the + request was invalid. In this case, parse the query on + the client-side and determine what went wrong.""" + + for q in query_str_lst: + l = query_p.QueryLexer() + l.build() + qp = query_p.QueryParser(l) + try: + query = qp.parse(q.encoded_text()) + except query_p.BooleanQueryException, e: + return api_errors.BooleanQueryException(e) + except query_p.ParseError, e: + return api_errors.ParseError(e) + + return None + def rebuild_search_index(self): """Rebuilds the search indexes. Removes all existing indexes and replaces them from scratch rather than @@ -1352,7 +1332,7 @@ # One of the publisher's repository # origins may have changed, so the # publisher needs to be revalidated. - self.img.valid_publisher_test(pub) + self.img.transport.valid_publisher_test(pub) # Because the more strict test above # was performed, there is no point in --- old/src/modules/client/api_errors.py Wed Jul 1 14:56:31 2009 +++ new/src/modules/client/api_errors.py Wed Jul 1 14:56:31 2009 @@ -25,10 +25,7 @@ # Use is subject to license terms. # -import httplib import os -import socket -import urllib2 import urlparse # EmptyI for argument defaults; can't import from misc due to circular @@ -46,14 +43,6 @@ self.user_dir = user_dir self.root_dir = root_dir -class NetworkUnavailableException(ApiException): - def __init__(self, caught_exception): - ApiException.__init__(self) - self.ex = caught_exception - - def __str__(self): - return str(self.ex) - class VersionException(ApiException): def __init__(self, expected_version, received_version): ApiException.__init__(self) @@ -288,11 +277,111 @@ return outstr +# SearchExceptions + class SearchException(ApiException): """Based class used for all search-related api exceptions.""" pass +class MainDictParsingException(SearchException): + """This is used when the main dictionary could not parse a line.""" + def __init__(self, e): + SearchException.__init__(self) + self.e = e + + def __str__(self): + return str(self.e) + + +class MalformedSearchRequest(SearchException): + """Raised when the server cannot understand the format of the + search request.""" + + def __init__(self, url): + SearchException.__init__(self) + self.url = url + + def __str__(self): + return str(self.url) + + +class NegativeSearchResult(SearchException): + """Returned when the search cannot find any matches.""" + + def __init__(self, url): + SearchException.__init__(self) + self.url = url + + def __str__(self): + return _("The search at url %s returned no results.") % self.url + + +class ProblematicSearchServers(SearchException): + """This class wraps exceptions which could appear while trying to + do a search request.""" + + def __init__(self, failed=EmptyI, invalid=EmptyI, unsupported=EmptyI): + SearchException.__init__(self) + self.failed_servers = failed + self.invalid_servers = invalid + self.unsupported_servers = unsupported + + def __str__(self): + s = _("Some servers failed to respond appropriately:\n") + for pub, err in self.failed_servers: + s += _("%(o)s:\n%(msg)s\n") % \ + { "o": pub, "msg": err} + for pub in self.invalid_servers: + s += _("%s did not return a valid response.\n" \ + % pub) + if len(self.unsupported_servers) > 0: + s += _("Some servers don't support requested search" + " operation:\n") + for pub, err in self.unsupported_servers: + s += _("%(o)s:\n%(msg)s\n") % \ + { "o": pub, "msg": err} + + return s + + +class SlowSearchUsed(SearchException): + """This exception is thrown when a local search is performed without + an index. It's raised after all results have been yielded.""" + + def __str__(self): + return _("Search performance is degraded.\n" + "Run 'pkg rebuild-index' to improve search speed.") + + +class UnsupportedSearchError(SearchException): + """Returned when a search protocol is not supported by the + remote server.""" + + def __init__(self, url=None, proto=None): + SearchException.__init__(self) + self.url = url + self.proto = proto + + def __str__(self): + s = _("Search server does not support the requested protocol:") + if self.url: + s += "\nServer URL: %s" % self.url + if self.proto: + s += "\nRequested operation: %s" % self.proto + return s + + def __cmp__(self, other): + if not isinstance(other, UnsupportedSearchError): + return -1 + r = cmp(self.url, other.url) + if r != 0: + return r + return cmp(self.proto, other.proto) + + +# IndexingExceptions. + class IndexingException(SearchException): """ The base class for all exceptions that can occur while indexing. """ @@ -306,6 +395,17 @@ pass +class InconsistentIndexException(IndexingException): + """This is used when the existing index is found to have inconsistent + versions.""" + def __init__(self, e): + IndexingException.__init__(self, e) + self.exception = e + + def __str__(self): + return str(self.exception) + + class ProblematicPermissionsIndexException(IndexingException): """ This is used when the indexer is unable to create, move, or remove files or directories it should be able to. """ @@ -315,11 +415,14 @@ "permissions. Please correct this issue then " \ "rebuild the index." % self.cause +# Query Parsing Exceptions +class BooleanQueryException(ApiException): + """This exception is used when the children of a boolean operation + have different return types. The command 'pkg search foo AND ' + is the simplest example of this.""" -class MainDictParsingException(SearchException): - """This is used when the main dictionary could not parse a line.""" def __init__(self, e): - SearchException.__init__(self) + ApiException.__init__(self) self.e = e def __str__(self): @@ -326,6 +429,15 @@ return str(self.e) +class ParseError(ApiException): + def __init__(self, e): + ApiException.__init__(self) + self.e = e + + def __str__(self): + return str(self.e) + + class NonLeafPackageException(ApiException): """Removal of a package which satisfies dependencies has been attempted. @@ -393,36 +505,40 @@ class TransportError(ApiException): - """Base exception class for all transfer exceptions.""" + """Abstract exception class for all transport exceptions. + Specific transport exceptions should be implemented in the + transport code. Callers wishing to catch transport exceptions + should use this class. Subclasses must implement all methods + defined here that raise NotImplementedError.""" - def __init__(self, *args, **kwargs): - ApiException.__init__(self, *args) - if args: - self.data = args[0] - else: - self.data = None - self.args = kwargs - def __str__(self): - return str(self.data) + raise NotImplementedError -class RetrievalError(TransportError): +class RetrievalError(ApiException): """Used to indicate that a a requested resource could not be retrieved.""" + def __init__(self, data, location=None): + ApiException.__init__(self) + self.data = data + self.location = location + def __str__(self): - location = self.args.get("location", None) - if location: + if self.location: return _("Error encountered while retrieving data from " - "'%s':\n%s") % (location, self.data) + "'%s':\n%s") % (self.location, self.data) return _("Error encountered while retrieving data from: %s") % \ self.data -class InvalidResourceLocation(TransportError): +class InvalidResourceLocation(ApiException): """Used to indicate that an invalid transport location was provided.""" + def __init__(self, data): + ApiException.__init__(self) + self.data = data + def __str__(self): return _("'%s' is not a valid location.") % self.data @@ -512,63 +628,6 @@ s += _(" '") + str(o) + _("'") return s - -class ProblematicSearchServers(SearchException): - """This class wraps exceptions which could appear while trying to - do a search request.""" - - def __init__(self, failed, invalid): - self.failed_servers = failed - self.invalid_servers = invalid - - def __str__(self): - s = _("Some servers failed to respond appropriately:\n") - for pub, err in self.failed_servers: - # The messages and structure for these error - # messages was often lifted from retrieve.py. - if isinstance(err, urllib2.HTTPError): - s += _(" %(o)s: %(msg)s (%(code)d)\n") % \ - { "o": pub["origin"], "msg": err.msg, - "code": err.code } - elif isinstance(err, urllib2.URLError): - if isinstance(err.args[0], socket.timeout): - s += _(" %s: timeout\n") % \ - (pub["origin"],) - else: - s += _(" %(o)s: %(other)s\n") % \ - { "o": pub["origin"], - "other": err.args[0][1] } - elif isinstance(err, httplib.BadStatusLine): - s += _(" %(o)s: Unable to read status of " - "HTTP response:%(l)s\n This is " - "most likely not a pkg(5) depot. Please " - "check the URL and the \n port " - "number.") % \ - { "o": pub["origin"], "l": err.line} - elif isinstance(err, - (httplib.IncompleteRead, ValueError)): - s += _(" %s: Incomplete read from " - "host") % pub["origin"] - # RunetimeErrors arise when no supported version - # of the operation request is found. - elif isinstance(err, RuntimeError): - s += _(" %(o)s: %(msg)s\n") % \ - { "o": pub["origin"], "msg": err} - elif isinstance(err, socket.timeout): - s += _(" %s: Socket timeout") % pub["origin"] - elif isinstance(err, socket.error): - s += _(" %(o)s: Socket error, reason: " - "%(msg)s") % { "o": pub["origin"], - "msg": err } - else: - s += _(" %(o)s: %(msg)s") % \ - { "o": pub["origin"], "msg": err} - for pub in self.invalid_servers: - s += _("%s appears not to be a valid package depot.\n" \ - % pub['origin']) - return s - - class IncorrectIndexFileHash(ApiException): """This is used when the index hash value doesn't match the hash of the packages installed in the image.""" @@ -575,47 +634,6 @@ pass -class InconsistentIndexException(IndexingException): - """This is used when the existing index is found to have inconsistent - versions.""" - def __init__(self, e): - self.exception = e - - def __str__(self): - return str(self.exception) - - -class SlowSearchUsed(SearchException): - """This exception is thrown when a local search is performed without - an index. It's raised after all results have been yielded.""" - - def __str__(self): - return _("Search performance is degraded.\n" - "Run 'pkg rebuild-index' to improve search speed.") - - -class BooleanQueryException(ApiException): - """This exception is used when the children of a boolean operation - have different return types. The command 'pkg search foo AND ' - is the simplest example of this.""" - - def __init__(self, e): - ApiException.__init__(self) - self.e = e - - def __str__(self): - return str(self.e) - - -class ParseError(ApiException): - def __init__(self, e): - ApiException.__init__(self) - self.e = e - - def __str__(self): - return str(self.e) - - class PublisherError(ApiException): """Base exception class for all publisher exceptions.""" --- old/src/modules/client/filelist.py Wed Jul 1 14:56:32 2009 +++ /dev/null Wed Jul 1 14:56:32 2009 @@ -1,557 +0,0 @@ -#!/usr/bin/python2.4 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# - -# -# Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -import os -import urllib -import urllib2 -import httplib -import socket -import time -import zlib -from tarfile import ReadError - -import pkg.pkgtarfile as ptf -import pkg.portable as portable -import pkg.client.api_errors as api_errors -import pkg.misc as misc -from pkg.client import global_settings -from pkg.misc import versioned_urlopen -from pkg.misc import hash_file_name -from pkg.misc import get_pkg_otw_size -from pkg.misc import TransportException -from pkg.misc import TransportFailures -from pkg.misc import TransferTimedOutException -from pkg.misc import TransferIOException -from pkg.misc import TransferContentException -from pkg.misc import InvalidContentException -from pkg.misc import retryable_http_errors -from pkg.misc import retryable_socket_errors - -class FileList(object): - """A FileList maintains mappings between files and Actions. - The list is built with knowledge of the Image and the PackagePlan's - associated actions. - - The FileList is responsible for downloading the files needed by the - PkgPlan from the repository. Once downloaded, the FileList generates - the appropriate opener and closer for the actions that it processed. By - downloading files in a group, it is possible to achieve better - performance. This is because the FileList asks for the files to be - sent in groups, instead of individual HTTP GET's.""" - - # - # This value should be left at the lowest value that provides "good - # enough" performance; tuning beyond 1MB has not in our experiments - # yielded more than a token speedup-- at the expense of interactivity. - # - maxbytes_default = 1024 * 1024 - - # - # A limit is placed on the maximum number of files to prevent - # the request size from growing too large. Some HTTP implementations - # choke on POST requests that are 128k or larger. By setting - # maxfiles to 1,500, this will prevent requests using sha1 or sha256 - # from exceeeding the size limit. - # - # XXX This will need to be adjusted for the sha512 hash, or any hash - # with equivalent length. - # - maxnfiles_default = 1500 - - def __init__(self, image, fmri, progtrack, check_cancelation): - """ - Create a FileList object for the specified image and pkgplan. - """ - - self.image = image - self.fmri = fmri - self.progtrack = progtrack - self.fhash = { } - - self.maxbytes = FileList.maxbytes_default - self.maxnfiles = FileList.maxnfiles_default - - self.actual_bytes = 0 - self.actual_nfiles = 0 - self.effective_bytes = 0 - self.effective_nfiles = 0 - - if fmri: - pub = self.fmri.get_publisher() - - self.publisher = pub - self.ssl_tuple = self.image.get_ssl_credentials(pub) - self.uuid = self.image.get_uuid(self.publisher) - else: - self.publisher = None - self.ssl_tuple = None - self.uuid = None - - self.ds = None - self.url = None - self.check_cancelation = check_cancelation - - def add_action(self, action): - """Add the specified action to the filelist. The action - must name a file that can be retrieved from the repository. - - This method will pull cached content from the download - directory, if it's available.""" - - # Check if we've got a cached version of the file before - # trying to add it to the list. If a cached version is present, - # create the opener and return. - - hashval = action.hash - cache_path = os.path.normpath(os.path.join( - self.image.cached_download_dir(), - hash_file_name(hashval))) - - try: - if os.path.exists(cache_path): - action.data = self._make_opener(cache_path) - bytes = get_pkg_otw_size(action) - - self._verify_content(action, cache_path) - self.progtrack.download_add_progress(1, bytes) - - return - except InvalidContentException: - # If the content in the cache doesn't match the hash of - # the action, verify will have already purged the item - # from the cache. Reset action.data to None and have - # _add_action download the file. - action.data = None - - while self._is_full(): - self._do_get_files() - - self._add_action(action) - - def _add_action(self, action): - """Add the specified action to the filelist. The action - must name a file that can be retrieved from the repository. - - This method gets invoked when we must go over the network - to retrieve file content. - - This is a private method which performs the majority of the - work for add_content().""" - - if not hasattr(action, "hash"): - raise FileListException, "Invalid action type" - - if self._is_full(): - raise FileListFullException - - hashval = action.hash - - # Each fhash key accesses a list of one or more actions. If we - # already have a key in the dictionary, get the list and append - # the action to it. Otherwise, create a new list with the first - # action. - if hashval in self.fhash: - l = self.fhash[hashval] - l.append(action) - else: - self.fhash[hashval] = [ action ] - self.actual_nfiles += 1 - self.actual_bytes += get_pkg_otw_size(action) - - # Regardless of whether files map to the same hash, we - # also track the total (effective) size and number of entries - # in the flist, for reporting purposes. - self.effective_nfiles += 1 - self.effective_bytes += get_pkg_otw_size(action) - - def _clear_mirror(self): - """Clear any selected DepotStatus and URL assocated with - a mirror selection.""" - - self.ds = None - self.url = None - - def _del_hash(self, fhash): - """Given the supplied content hash, remove the entry - from the flist's dictionary and adjust the counters - accordingly.""" - - try: - act_list = self.fhash[fhash] - except KeyError: - return - - pkgsz = get_pkg_otw_size(act_list[0]) - nactions = len(act_list) - - # Update the actual counts by subtracting the first - # item in the list - self.actual_nfiles -= 1 - self.actual_bytes -= pkgsz - - # Now update effective count - self.effective_nfiles -= nactions - self.effective_bytes -= nactions * pkgsz - - # Now delete the entry out of the dictionary - del self.fhash[fhash] - - # XXX detect missing size and warn - - def _do_get_files(self): - """A wrapper around _get_files. This handles exceptions - that might occur and deals with timeouts.""" - - num_mirrors = self.image.num_mirrors(self.publisher) - max_timeout = global_settings.PKG_TIMEOUT_MAX - if num_mirrors > 0: - retry_count = max_timeout * (num_mirrors + 1) - else: - retry_count = max_timeout - - files_extracted = 0 - nfiles = self._get_nfiles() - nbytes = self._get_nbytes() - chosen_mirrors = set() - ts = 0 - failures = TransportFailures() - - while files_extracted == 0: - try: - self._pick_mirror(chosen_mirrors) - ts = time.time() - - fe = self._get_files() - files_extracted += fe - - except TransportException, e: - retry_count -= 1 - self.ds.record_error() - self._clear_mirror() - - failures.append(e) - if retry_count <= 0: - raise failures - else: - ts = time.time() - ts - self.ds.record_success(ts) - - nfiles -= self._get_nfiles() - nbytes -= self._get_nbytes() - self.progtrack.download_add_progress(nfiles, nbytes) - - def _extract_file(self, tarinfo, tar_stream, download_dir): - """Given a tarinfo object, extract that onto the filesystem - so it can be installed.""" - - completed_dir = self.image.cached_download_dir() - - hashval = tarinfo.name - - # Set the perms of the temporary file. The file must - # be writable so that the mod time can be changed on Windows - tarinfo.mode = 0600 - tarinfo.uname = "root" - tarinfo.gname = "root" - - if self.check_cancelation(): - raise api_errors.CanceledException - - # XXX catch IOError if tar stream closes inadvertently? - tar_stream.extract_to(tarinfo, download_dir, hashval) - - # Now that the file has been successfully extracted, move - # it to the cached content directory. - dl_path = os.path.join(download_dir, hashval) - final_path = os.path.normpath(os.path.join(completed_dir, - hash_file_name(hashval))) - - # Check that hashval is in the list of files we requested - try: - l = self.fhash[hashval] - except KeyError: - # If the key isn't in the dictionary, the server sent us - # a file we didn't ask for. In this case, we can't - # create an opener for it, nor should we hold onto it. - os.remove(dl_path) - return - - # Verify downloaded content - self._verify_content(l[0], dl_path) - - if not os.path.exists(os.path.dirname(final_path)): - os.makedirs(os.path.dirname(final_path)) - - # Content has been verified and was requested from server. - # Move into content cache - portable.rename(dl_path, final_path) - - # assign opener to actions in the list - for action in l: - action.data = self._make_opener(final_path) - - # Remove successfully extracted items from the hash - # and adjust bean counters - self._del_hash(hashval) - - - def flush(self): - """Ensure that the actions added to the filelist have had - their data retrieved from the depot.""" - while self._list_size() > 0: - self._do_get_files() - - def _get_files(self): - """Instruct the FileList object to download the files - for the actions that have been associated with this object. - - This routine will raise a FileListException if the server - does not support filelist. Callers of get_files should - consider catching this exception.""" - - req_dict = { } - tar_stream = None - files_extracted = 0 - - url_prefix = self.url - - download_dir = self.image.incoming_download_dir() - # Make sure the download directory is there before we start - # retrieving and extracting files. - try: - if not os.path.exists(download_dir): - os.makedirs(download_dir) - except EnvironmentError, e: - raise FileListRetrievalError("Unable to create " \ - "download directory %s: %s" % - (download_dir, e.strerror)) - - for i, k in enumerate(self.fhash.keys()): - fstr = "File-Name-%s" % i - req_dict[fstr] = k - - req_str = urllib.urlencode(req_dict) - - try: - f, v = versioned_urlopen(url_prefix, "filelist", [0], - data=req_str, ssl_creds=self.ssl_tuple, - imgtype=self.image.type, uuid=self.uuid) - except RuntimeError: - raise FileListRetrievalError, "No server-side support" - except urllib2.HTTPError, e: - # Must check for HTTPError before URLError - self.image.cleanup_downloads() - if e.code in retryable_http_errors: - raise TransferTimedOutException(url_prefix, - "%d - %s" % (e.code, e.msg)) - - raise FileListRetrievalError("Could not retrieve" - " filelist from '%s'\nHTTPError code: %d - %s" % - (url_prefix, e.code, e.msg)) - except urllib2.URLError, e: - self.image.cleanup_downloads() - if isinstance(e.args[0], socket.timeout): - raise TransferTimedOutException(url_prefix, - e.reason) - elif isinstance(e.args[0], socket.error): - sockerr = e.args[0] - if isinstance(sockerr.args, tuple) and \ - sockerr.args[0] in retryable_socket_errors: - raise TransferIOException( - url_prefix, - "Retryable socket error: %s" % - e.reason) - - raise FileListRetrievalError("Could not retrieve" - " filelist from '%s'\nURLError reason: %s" % - (url_prefix, e.reason)) - except (ValueError, httplib.IncompleteRead): - self.image.cleanup_downloads() - raise TransferContentException(url_prefix, - "Incomplete Read from remote host") - except httplib.BadStatusLine: - self.image.cleanup_downloads() - raise TransferContentException(url_prefix, - "Unable to read status of HTTP response") - except KeyboardInterrupt: - self.image.cleanup_downloads() - raise - except Exception, e: - self.image.cleanup_downloads() - raise FileListRetrievalError("Could not retrieve" - " filelist from '%s'\nException: str:%s repr:%s" % - (url_prefix, e, repr(e))) - - - # Exception handling here is a bit complicated. The finally - # block makes sure we always close our file objects. If we get - # a socket.timeout we may have gotten an error in the middle of - # downloading a file. In that case, delete the incoming files we - # were processing. They were not successfully retrieved. - try: - try: - tar_stream = ptf.PkgTarFile.open(mode = "r|", - fileobj = f) - for info in tar_stream: - self._extract_file(info, tar_stream, - download_dir) - files_extracted += 1 - except socket.timeout, e: - self.image.cleanup_downloads() - raise TransferTimedOutException(url_prefix) - except socket.error, e: - self.image.cleanup_downloads() - if isinstance(e.args, tuple) and \ - e.args[0] in retryable_socket_errors: - raise TransferIOException( - url_prefix, - "Retryable socket error: %s" % e) - - raise FileListRetrievalError( - "Could not retrieve filelist from" - " '%s'\nsocket error, reason: %s" % - (url_prefix, e)) - except (ValueError, httplib.IncompleteRead): - self.image.cleanup_downloads() - raise TransferContentException(url_prefix, - "Incomplete Read from remote host") - except ReadError: - self.image.cleanup_downloads() - raise TransferContentException(url_prefix, - "Read error on tar stream") - except EnvironmentError, e: - self.image.cleanup_downloads() - raise FileListRetrievalError( - "Could not retrieve filelist from '%s'\n" - "Exception: str:%s repr:%s" % - (url_prefix, e, repr(e))) - finally: - if tar_stream: - tar_stream.close() - f.close() - - return files_extracted - - def _get_nbytes(self): - return self.effective_bytes - - def _get_nfiles(self): - return self.effective_nfiles - - def _is_full(self): - """Returns true if the FileList object has filled its - allocated slots and can no longer accept new actions.""" - - if self.actual_bytes >= self.maxbytes or \ - self.actual_nfiles >= self.maxnfiles: - return True - - return False - - def _list_size(self): - """Returns the current number of files in the filelist.""" - - return len(self.fhash) - - @staticmethod - def _make_opener(filepath): - def opener(): - f = open(filepath, "rb") - return f - return opener - - def _pick_mirror(self, chosen_set=None): - """If we don't already have a DepotStatus or a URL, - select a mirror, populate the DepotStatus, and choose a URL.""" - - if self.ds and self.url: - return - elif self.ds: - self.url = self.ds.url - else: - self.ds = self.image.select_mirror(self.publisher, - chosen_set) - self.url = self.ds.url - chosen_set.add(self.ds) - - @staticmethod - def _verify_content(action, filepath): - """If action contains an attribute that has the compressed - hash, read the file specified in filepath and verify - that the hash values match. If the values do not match, - remove the file and raise an InvalidContentException.""" - - chash = action.attrs.get("chash", None) - path = action.attrs.get("path", None) - if not chash: - # Compressed hash doesn't exist. Decompress and - # generate hash of uncompressed content. - ifile = open(filepath, "rb") - ofile = open(os.devnull, "wb") - - try: - fhash = misc.gunzip_from_stream(ifile, ofile) - except zlib.error, e: - os.remove(filepath) - raise InvalidContentException(path, - "zlib.error:%s" % - (" ".join([str(a) for a in e.args]))) - - ifile.close() - ofile.close() - - if action.hash != fhash: - os.remove(filepath) - raise InvalidContentException(action.path, - "hash failure: expected: %s" - "computed: %s" % (action.hash, fhash)) - return - - newhash = misc.get_data_digest(filepath)[0] - if chash != newhash: - os.remove(filepath) - raise InvalidContentException(path, - "chash failure: expected: %s computed: %s" % \ - (chash, newhash)) - - -class FileListException(Exception): - def __init__(self, args=None): - Exception.__init__(self) - self.args = args - -class FileListFullException(FileListException): - pass - -class FileListRetrievalError(FileListException): - """Used when filelist retrieval fails""" - def __init__(self, data): - FileListException.__init__(self) - self.data = data - - def __str__(self): - return str(self.data) --- old/src/modules/client/image.py Wed Jul 1 14:56:32 2009 +++ new/src/modules/client/image.py Wed Jul 1 14:56:32 2009 @@ -35,33 +35,29 @@ import urllib import pkg.Uuid25 -import pkg.catalog as catalog -import pkg.client.api_errors as api_errors -import pkg.client.constraint as constraint -import pkg.client.history as history -import pkg.client.imageconfig as imageconfig -import pkg.client.imageplan as imageplan -import pkg.client.imagestate as imagestate -import pkg.client.pkgplan as pkgplan -import pkg.client.progress as progress -import pkg.client.publisher as publisher -import pkg.client.retrieve as retrieve -import pkg.client.variant as variant +import pkg.catalog as catalog +import pkg.client.api_errors as api_errors +import pkg.client.constraint as constraint +import pkg.client.history as history +import pkg.client.imageconfig as imageconfig +import pkg.client.imageplan as imageplan +import pkg.client.imagestate as imagestate +import pkg.client.pkgplan as pkgplan +import pkg.client.progress as progress +import pkg.client.publisher as publisher +import pkg.client.transport.transport as transport +import pkg.client.variant as variant import pkg.fmri -import pkg.manifest as manifest -import pkg.misc as misc -import pkg.portable as portable +import pkg.manifest as manifest +import pkg.misc as misc +import pkg.portable as portable import pkg.version -from pkg.actions import MalformedActionError from pkg.client import global_settings -from pkg.client.api_errors import InvalidDepotResponseException from pkg.client.imagetypes import IMG_USER, IMG_ENTIRE from pkg.misc import CfgCacheError from pkg.misc import EmptyI, EmptyDict from pkg.misc import msg, emsg -from pkg.misc import TransportException -from pkg.misc import TransportFailures CATALOG_CACHE_FILE = "catalog_cache" img_user_prefix = ".org.opensolaris,pkg" @@ -162,7 +158,7 @@ self.dl_cache_dir = None self.dl_cache_incoming = None self.is_user_cache_dir = False - self.state = imagestate.ImageState() + self.state = imagestate.ImageState(self) self.attrs = { "Policy-Require-Optional": False, "Policy-Pursue-Latest": True @@ -173,13 +169,12 @@ self.constraints = constraint.ConstraintSet() + # Transport operations for this image + self.transport = transport.Transport(self) + # a place to keep info about saved_files; needed by file action self.saved_files = {} - # A place to keep track of which manifests (based on fmri and - # operation) have already provided intent information. - self.__touched_manifests = {} - self.__manifest_cache = {} # right now we don't explicitly set dir/file modes everywhere; @@ -420,149 +415,6 @@ if inc_disabled or not pub.disabled: yield self.cfg_cache.publishers[p] - def get_url_by_publisher(self, prefix=None): - """Return the URL prefix associated with the given prefix. - For the undefined case, represented by None, return the - preferred publisher.""" - - # XXX This function is a possible location to insert one or more - # policies regarding use of mirror responses, etc. - - if prefix is None: - prefix = self.cfg_cache.preferred_publisher - - try: - o = self.cfg_cache.publishers[prefix]["origin"] - except KeyError: - # If the publisher that we're trying to get no longer - # exists, fall back to preferred publisher. - prefix = self.cfg_cache.preferred_publisher - o = self.cfg_cache.publishers[prefix]["origin"] - - return o.rstrip("/") - - def gen_depot_status(self): - """Walk all publishers and return all depot status - objects for both mirrors and primary publishers.""" - - pubs = self.cfg_cache.publishers - # return depot status objects in publisher order - for pub in pubs.keys(): - # first yield publisher origin - yield self.cfg_cache.publisher_status[pub] - # then return mirrors - for ds in self.cfg_cache.mirror_status[pub]: - yield ds - - def num_mirrors(self, pub): - """Return the number of mirrors configured for the - given publisher.""" - - if pub == None: - pub = self.cfg_cache.preferred_publisher - - try: - num = len(self.cfg_cache.mirror_status[pub]) - except KeyError: - # pub isn't in the list of mirrors, return 0 - num = 0 - - return num - - def select_mirror(self, pub = None, chosen_set = None): - """For the given publisher, look through the status of - the mirrors. Pick the best one. This method returns - a DepotStatus object or None. The chosen_set argument - contains a set object that lists the mirrors that were - previously chosen. This allows us to choose both - by depot status statistics and ensures we don't - always pick the same depot.""" - - if pub == None: - pub = self.cfg_cache.preferred_publisher - try: - slst = self.cfg_cache.mirror_status[pub] - except KeyError: - # If the publisher that we're trying to get no longer - # exists, fall back to preferred publisher. - pub = self.cfg_cache.preferred_publisher - slst = self.cfg_cache.mirror_status[pub] - - if len(slst) == 0: - if pub in self.cfg_cache.publisher_status: - return self.cfg_cache.publisher_status[pub] - else: - return None - - # Choose mirror with fewest errors. - # If mirrors have same number of errors, choose mirror - # with smaller number of good transactions. Assume it's - # being underused, not high-latency. - # - # XXX Will need to revisit the above assumption. - def cmp_depotstatus(a, b): - res = cmp(a.errors, b.errors) - if res == 0: - return cmp(a.good_tx, b.good_tx) - return res - - slst.sort(cmp = cmp_depotstatus) - - # All mirrors in the chosen_set have already been - # selected. Try the publisher origin instead. - # Empty chosen_set, next time we start over. - if chosen_set and len(chosen_set) == len(slst): - chosen_set.clear() - return self.cfg_cache.publisher_status[pub] - - if chosen_set and slst[0] in chosen_set: - for ds in slst: - if ds not in chosen_set: - return ds - - return slst[0] - - def get_ssl_credentials(self, prefix=None, origin=None, - pubent=None): - """Deprecated; this function will be removed in a future - release. This information should be retrieved directly from a - repository origin or mirror object. - - Return a tuple containing (ssl_key, ssl_cert) for the - specified publisher prefix. If the publisher isn't specified, - attempt to determine the publisher by the given origin. If - neither is specified, use the preferred publisher. pubent - is a dictionary argument that contains the publisher - information.""" - - if not pubent and prefix is None: - if origin is None: - prefix = self.cfg_cache.preferred_publisher - else: - pubs = self.cfg_cache.publishers - for pfx, pub in pubs.iteritems(): - repo = pub.selected_repository - if repo.has_origin(origin): - prefix = pfx - break - else: - return None - - # One of these should be defined at this point unless the - # caller didn't provide anything. - assert prefix or origin or pubent - - if not pubent: - try: - pubent = self.cfg_cache.publishers[prefix] - except KeyError: - prefix = self.cfg_cache.preferred_publisher - pubent = self.cfg_cache.publishers[prefix] - - repo = pubent.selected_repository - origin = repo.origins[0] - return (origin.ssl_key, origin.ssl_cert) - def check_cert_validity(self): """Look through the publishers defined for the image. Print a message and exit with an error if one of the certificates @@ -578,22 +430,6 @@ prefix=p.prefix, uri=uri) return True - def get_uuid(self, prefix): - """Deprecated; this function will be removed in a future - release. This information should be retrieved directly from a - publisher object. - - Return the UUID for the specified publisher prefix. If the - policy for sending the UUID is set to false, return None.""" - - if not self.cfg_cache.get_policy(imageconfig.SEND_UUID): - return None - - try: - return self.cfg_cache.publishers[prefix].client_uuid - except KeyError: - return None - def has_publisher(self, prefix=None, alias=None): for pub in self.gen_publishers(): if prefix == pub.prefix or (alias and @@ -611,7 +447,7 @@ alias=alias) except api_errors.ApiException, e: self.history.log_operation_end(e) - raise e + raise if pub.prefix == self.cfg_cache.preferred_publisher: e = api_errors.RemovePreferredPublisher() @@ -703,7 +539,7 @@ alias=alias) except api_errors.UnknownPublisher, e: self.history.log_operation_end(error=e) - raise e + raise if pub.disabled: e = api_errors.SetPreferredPublisherDisabled(pub) @@ -762,7 +598,7 @@ try: # First, verify that the publisher has a valid # pkg(5) repository. - self.valid_publisher_test(pub) + self.transport.valid_publisher_test(pub) self.__retrieve_catalogs(full_refresh=True, pubs=[pub], progtrack=progtrack) @@ -832,108 +668,33 @@ return False - def __fetch_manifest_with_retries(self, fmri, excludes=EmptyI): - """go get the manifest we want - retry if needed""" + def __fetch_manifest(self, fmri, excludes=EmptyI): + """A wrapper call for getting manifests. This invokes + the transport method, gets the manifest, and performs + any additional image-related processing.""" - retry_count = global_settings.PKG_TIMEOUT_MAX - failures = TransportFailures() + m = self.transport.get_manifest(fmri, excludes, + self.state.get_intent_str(fmri)) - while retry_count > 0: - try: - mcontent = retrieve.get_manifest(self, fmri) + # What is the client currently processing? + targets = self.state.get_targets() + + intent = None + for entry in targets: + target, reason = entry - m = manifest.CachedManifest(fmri, self.pkgdir, - self.cfg_cache.preferred_publisher, - excludes, mcontent) + # Ignore the publisher for comparison. + np_target = target.get_fmri(anarchy=True) + np_fmri = fmri.get_fmri(anarchy=True) + if np_target == np_fmri: + intent = reason - # What is the client currently processing? - targets = self.state.get_targets() + # If no intent could be found, assume INTENT_INFO. + self.state.set_touched_manifest(fmri, + max(intent, imagestate.INTENT_INFO)) - intent = None - for entry in targets: - target, reason = entry + return m - # Ignore the publisher for comparison. - np_target = target.get_fmri( - anarchy=True) - np_fmri = fmri.get_fmri(anarchy=True) - if np_target == np_fmri: - intent = reason - - # If no intent could be found, assume - # INTENT_INFO. - self.__set_touched_manifest(fmri, - max(intent, imagestate.INTENT_INFO)) - - return m - - except TransportException, e: - retry_count -= 1 - failures.append(e) - - except MalformedActionError, e: - retry_count -= 1 - pub = fmri.get_publisher() - url = self.cfg_cache.publishers[pub]["origin"] - te = misc.TransferContentException(url=url, - reason=str(e)) - failures.append(te) - - raise failures - - def __get_touched_manifest(self, fmri, intent): - """Returns whether intent information has been provided for the - given fmri.""" - - op = self.history.operation_name - if not op: - # The client may not have provided the name of the - # operation it is performing. - op = "unknown" - - if op not in self.__touched_manifests: - # No intent information has been provided for fmris - # for the current operation. - return False - - f = str(fmri) - if f not in self.__touched_manifests[op]: - # No intent information has been provided for this - # fmri for the current operation. - return False - - if intent not in self.__touched_manifests[op][f]: - # No intent information has been provided for this - # fmri for the current operation and reason. - return False - - return True - - def __set_touched_manifest(self, fmri, intent): - """Records that intent information has been provided for the - given fmri's manifest.""" - - op = self.history.operation_name - if not op: - # The client may not have provided the name of the - # operation it is performing. - op = "unknown" - - if op not in self.__touched_manifests: - # No intent information has yet been provided for fmris - # for the current operation. - self.__touched_manifests[op] = {} - - f = str(fmri) - if f not in self.__touched_manifests[op]: - # No intent information has yet been provided for this - # fmri for the current operation. - self.__touched_manifests[op][f] = { intent: None } - else: - # No intent information has yet been provided for this - # fmri for the current operation and reason. - self.__touched_manifests[op][f][intent] = None - def __touch_manifest(self, fmri): """Perform steps necessary to 'touch' a manifest to provide intent information. Ignores most exceptions as this operation @@ -947,7 +708,7 @@ if not target or intent == imagestate.INTENT_EVALUATE: return - if not self.__get_touched_manifest(fmri, intent): + if not self.state.get_touched_manifest(fmri, intent): # If the manifest for this fmri hasn't been "seen" # before, determine if intent information needs to be # provided. @@ -959,8 +720,15 @@ # If the client is currently processing # the given fmri (for an install, etc.) # then intent information is needed. - retrieve.touch_manifest(self, fmri) - self.__set_touched_manifest(fmri, intent) + try: + self.transport.touch_manifest(fmri, + self.state.get_intent_str(fmri)) + except (api_errors.UnknownPublisher, + api_errors.TransportError), e: + # It's not fatal if we can't find + # or reach the publisher. + pass + self.state.set_touched_manifest(fmri, intent) def get_manifest_path(self, fmri): """Return path to on-disk manifest""" @@ -977,8 +745,7 @@ self.cfg_cache.preferred_publisher, excludes) except KeyError: - return self.__fetch_manifest_with_retries(fmri, - excludes) + return self.__fetch_manifest(fmri, excludes) def get_manifest(self, fmri, add_to_cache=True, all_arch=False): """return manifest; uses cached version if available. @@ -1379,140 +1146,6 @@ dependents.extend(self.__req_dependents[f]) return dependents - def __do_get_versions(self, pub): - """An internal method that is a wrapper around get_catalog. - This handles retryable exceptions and timeouts.""" - - retry_count = global_settings.PKG_TIMEOUT_MAX - failures = TransportFailures() - versdict = None - - while not versdict: - try: - versdict = retrieve.get_versions(self, pub) - except TransportException, e: - retry_count -= 1 - failures.append(e) - - if retry_count <= 0: - raise failures - - return versdict - - def valid_publisher_test(self, pub): - """Test that the publisher supplied in pub actually - points to a valid packaging server.""" - - try: - vd = self.__do_get_versions(pub) - except (retrieve.VersionRetrievalError, - TransportFailures), e: - # Failure when contacting server. Report - # this as an error. - raise InvalidDepotResponseException(pub["origin"], - "Transport errors encountered when trying to " - "contact depot server. Reported the following " - "errors:\n%s" % e) - - if not self._valid_versions_test(vd): - raise InvalidDepotResponseException(pub["origin"], - "Invalid or unparseable version information.") - - def captive_portal_test(self, pubs=None): - """A captive portal forces a HTTP client on a network to see a - special web page, usually for pubentication purposes - (http://en.wikipedia.org/wiki/Captive_portal). - - 'pubs' is an optional list of publisher objects to be used for - the check. If not provided, any publishers available for - packaging operations will be used for the test instead.""" - - if not pubs: - pubs = list(self.gen_publishers()) - - vd = None - for pub in pubs: - try: - vd = self.__do_get_versions(pub) - except (retrieve.VersionRetrievalError, - TransportFailures): - # Encountered a transport error while - # trying to contact this publisher. - # Pick another publisher instead. - continue - - if self._valid_versions_test(vd): - return - else: - raise InvalidDepotResponseException( - pub["origin"], _("This server is not a " - "valid package depot.")) - if not vd: - # We got all the way through the list of puborites but - # encountered transport errors in every case. This is - # likely a network configuration problem. Report our - # inability to contact a server. - raise InvalidDepotResponseException(None, - "Unable to contact any configured publishers. " - "This is likely a network configuration problem.") - - @staticmethod - def _valid_versions_test(versdict): - """Check that the versions information contained in - versdict contains valid version specifications. - - In order to test for this condition, pick a publisher - from the list of active publishers. Check to see if - we can connect to it. If so, test to see if it supports - the versions/0 operations. If versions/0 is not found, - we get an unparseable response, or the response does - not contain pkg-server, or versions 0 then we're not - talking to a depot. Return an error in these cases.""" - - if "pkg-server" in versdict: - # success! - return True - elif "versions" in versdict: - try: - versids = [ - int(v) - for v in versdict["versions"].split() - ] - except ValueError: - # Unable to determine version number. Fail. - return False - - if 0 not in versids: - # Paranoia. versions 0 should be in the - # output for versions/0. If we're here, - # something has gone very wrong. EPIC FAIL! - return False - - # found versions/0, success! - return True - - # Some other error encountered. Fail - return False - - def _do_get_catalog(self, pub, hdr, ts): - """An internal method that is a wrapper around get_catalog. - This handles retryable exceptions and timeouts.""" - - retry_count = global_settings.PKG_TIMEOUT_MAX - failures = TransportFailures() - success = False - - while not success: - try: - success = retrieve.get_catalog(self, pub, - hdr, ts) - except TransportException, e: - retry_count -= 1 - failures.append(e) - - if retry_count <= 0: - raise failures - def refresh_publishers(self, full_refresh=False, immediate=False, pubs=None, progtrack=None, validate=True): """Refreshes the metadata (e.g. catalog) for one or more @@ -1582,7 +1215,7 @@ # from the publisher repositories, a check needs # to be done to ensure that the client isn't # stuck behind a captive portal. - self.captive_portal_test() + self.transport.captive_portal_test() self.__retrieve_catalogs(full_refresh=full_refresh, pubs=pubs_to_refresh, progtrack=progtrack) @@ -1656,7 +1289,7 @@ full_refresh_this_pub = False cat = None - ts = 0 + ts = None size = 0 if pub.prefix in self.__catalogs: cat = self.__catalogs[pub.prefix] @@ -1671,18 +1304,15 @@ if cat.origin() not in repo.origins: full_refresh_this_pub = True - if ts and not full_refresh and \ - not full_refresh_this_pub: - hdr = {"If-Modified-Since": ts} - else: - hdr = {} + if full_refresh or full_refresh_this_pub: + # Set timestamp to None in order + # to perform full refresh. + ts = None try: - self._do_get_catalog(pub, hdr, ts) - except retrieve.CatalogRetrievalError, e: + self.transport.get_catalog(pub, ts) + except api_errors.TransportError, e: failed.append((pub, e)) - except TransportFailures, e: - failed.append((pub, e)) else: if catalog_changed(pub.prefix, ts, size): updated += 1 --- old/src/modules/client/imageconfig.py Wed Jul 1 14:56:33 2009 +++ new/src/modules/client/imageconfig.py Wed Jul 1 14:56:33 2009 @@ -34,41 +34,6 @@ import pkg.portable as portable import re -class DepotStatus(object): - """An object that encapsulates status about a depot server. - This includes things like observed performance, availability, - successful and unsuccessful transaction rates, etc.""" - - def __init__(self, prefix, url): - """prefix is the publisher prefix for this depot. url is the - URL that names the server or mirror itself.""" - - self.prefix = prefix - self.url = url.rstrip("/") - self.available = True - - self.errors = 0 - self.good_tx = 0 - - self.last_tx_time = None - - def record_error(self): - - self.errors += 1 - - def record_success(self, tx_time): - - self.good_tx += 1 - self.last_tx_time = tx_time - - def set_available(self, avail): - - if avail: - self.available = True - else: - self.available = False - - # The default_policies dictionary defines the policies that are supported by # pkg(5) and their default values. Calls to the ImageConfig.get_policy method # should use the constants defined here. @@ -110,8 +75,6 @@ def __init__(self, imgroot): self.__imgroot = imgroot self.publishers = {} - self.publisher_status = {} - self.mirror_status = {} self.properties = dict(( (p, str(v)) for p, v in default_policies.iteritems() @@ -163,17 +126,9 @@ for s in cp.sections(): if re.match("authority_.*", s): k, a = self.read_publisher(pmroot, cp, s) - ms = [] self.publishers[k] = a - self.publisher_status[k] = DepotStatus(k, - a["origin"]) - - for mirror in a["mirrors"]: - ms.append(DepotStatus(k, mirror)) - - self.mirror_status[k] = ms - + if self.preferred_publisher == None: self.preferred_publisher = k @@ -221,8 +176,6 @@ k, a = self.read_publisher(pmroot, cp, s) self.publishers[k] = a - # status objects are not created for - # disabled publishers def write(self, path): """Write the configuration to the given directory""" --- old/src/modules/client/imageplan.py Wed Jul 1 14:56:34 2009 +++ new/src/modules/client/imageplan.py Wed Jul 1 14:56:34 2009 @@ -40,9 +40,6 @@ from pkg.client.filter import compile_filter from pkg.misc import msg -from pkg.client.retrieve import ManifestRetrievalError -from pkg.client.retrieve import DatastreamRetrievalError - UNEVALUATED = 0 # nothing done yet EVALUATED_PKGS = 1 # established fmri changes EVALUATED_OK = 2 # ready to execute @@ -437,11 +434,6 @@ except KeyError, e: outstring += "Attempting to install %s " \ "causes:\n\t%s\n" % (f.get_name(), e) - except (ManifestRetrievalError, - DatastreamRetrievalError), e: - raise api_errors.NetworkUnavailableException( - str(e)) - if outstring: raise RuntimeError("No packages were installed because " "package dependencies could not be satisfied\n" + --- old/src/modules/client/imagestate.py Wed Jul 1 14:56:35 2009 +++ new/src/modules/client/imagestate.py Wed Jul 1 14:56:35 2009 @@ -20,7 +20,7 @@ # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # Indicates that the fmri is being used strictly for information. @@ -40,9 +40,14 @@ packages that are being installed, uninstalled, etc.). """ - def __init__(self): + def __init__(self, image): self.__fmri_intent_stack = [] + self.__image = image + # A place to keep track of which manifests (based on fmri and + # operation) have already provided intent information. + self.__touched_manifests = {} + def __str__(self): return "%s" % self.__fmri_intent_stack @@ -75,3 +80,189 @@ """ return self.__fmri_intent_stack[:] + def get_touched_manifest(self, fmri, intent): + """Returns whether intent information has been provided for the + given fmri.""" + + op = self.__image.history.operation_name + if not op: + # The client may not have provided the name of the + # operation it is performing. + op = "unknown" + + if op not in self.__touched_manifests: + # No intent information has been provided for fmris + # for the current operation. + return False + + f = str(fmri) + if f not in self.__touched_manifests[op]: + # No intent information has been provided for this + # fmri for the current operation. + return False + + if intent not in self.__touched_manifests[op][f]: + # No intent information has been provided for this + # fmri for the current operation and reason. + return False + + return True + + def set_touched_manifest(self, fmri, intent): + """Records that intent information has been provided for the + given fmri's manifest.""" + + op = self.__image.history.operation_name + if not op: + # The client may not have provided the name of the + # operation it is performing. + op = "unknown" + + if op not in self.__touched_manifests: + # No intent information has yet been provided for fmris + # for the current operation. + self.__touched_manifests[op] = {} + + f = str(fmri) + if f not in self.__touched_manifests[op]: + # No intent information has yet been provided for this + # fmri for the current operation. + self.__touched_manifests[op][f] = { intent: None } + else: + # No intent information has yet been provided for this + # fmri for the current operation and reason. + self.__touched_manifests[op][f][intent] = None + + def get_intent_str(self, fmri): + """Returns a string representing the intent of the client + in retrieving information based on the operation information + provided by the image history object. + """ + + op = self.__image.history.operation_name + if not op: + # The client hasn't indicated what operation + # is executing. + op = "unknown" + + reason = INTENT_INFO + target_pkg = None + initial_pkg = None + needed_by_pkg = None + current_pub = fmri.get_publisher() + + targets = self.get_targets() + if targets: + # Attempt to determine why the client is retrieving the + # manifest for this fmri and what its current target is. + target, reason = targets[-1] + + # Compare the FMRIs with no publisher information + # embedded. + na_current = fmri.get_fmri(anarchy=True) + na_target = target.get_fmri(anarchy=True) + + if na_target == na_current: + # Only provide this information if the fmri for + # the manifest being retrieved matches the fmri + # of the target. If they do not match, then the + # target fmri is being retrieved for information + # purposes only (e.g. dependency calculation, + # etc.). + target_pub = target.get_publisher() + if target_pub == current_pub: + # Prevent providing information across + # publishers. + target_pkg = na_target[len("pkg:/"):] + else: + target_pkg = "unknown" + + # The very first fmri should be the initial + # target that caused the current and needed_by + # fmris to be retrieved. + initial = targets[0][0] + initial_pub = initial.get_publisher() + if initial_pub == current_pub: + # Prevent providing information across + # publishers. + initial_pkg = initial.get_fmri( + anarchy=True)[len("pkg:/"):] + + if target_pkg == initial_pkg: + # Don't bother sending the + # target information if it is + # the same as the initial target + # (i.e. the manifest for foo@1.0 + # is being retrieved because the + # user is installing foo@1.0). + target_pkg = None + + else: + # If they didn't match, indicate that + # the needed_by_pkg was a dependency of + # another, but not which one. + initial_pkg = "unknown" + + if len(targets) > 1: + # The fmri responsible for the current + # one being processed should immediately + # precede the current one in the target + # list. + needed_by = targets[-2][0] + + needed_by_pub = \ + needed_by.get_publisher() + if needed_by_pub == current_pub: + # To prevent dependency + # information being shared + # across publisher boundaries, + # publishers must match. + needed_by_pkg = \ + needed_by.get_fmri( + anarchy=True)[len("pkg:/"):] + else: + # If they didn't match, indicate + # that the package is needed by + # another, but not which one. + needed_by_pkg = "unknown" + else: + # An operation is being performed that has not provided + # any target information and is likely for informational + # purposes only. Assume the "initial target" is what is + # being retrieved. + initial_pkg = str(fmri)[len("pkg:/"):] + + prior_version = None + if reason != INTENT_INFO: + # Only provide version information for non-informational + # operations. + prior = self.__image.get_version_installed(fmri) + + try: + prior_version = prior.version + except AttributeError: + # We didn't get a match back, drive on. + pass + else: + prior_pub = prior.get_publisher() + if prior_pub != current_pub: + # Prevent providing information across + # publishers by indicating that a prior + # version was installed, but not which + # one. + prior_version = "unknown" + + info = { + "operation": op, + "prior_version": prior_version, + "reason": reason, + "target": target_pkg, + "initial_target": initial_pkg, + "needed_by": needed_by_pkg, + } + + # op/prior_version/reason/initial_target/needed_by/ + return "(%s)" % ";".join([ + "%s=%s" % (key, info[key]) for key in info + if info[key] is not None + ]) --- old/src/modules/client/pkgplan.py Wed Jul 1 14:56:36 2009 +++ new/src/modules/client/pkgplan.py Wed Jul 1 14:56:36 2009 @@ -20,7 +20,7 @@ # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. import errno @@ -28,7 +28,6 @@ import os import pkg.manifest as manifest -import pkg.client.filelist as filelist import pkg.actions.directory as directory from pkg.misc import msg from pkg.misc import get_pkg_otw_size @@ -232,15 +231,19 @@ def download(self): """Download data for any actions that need it.""" - flist = filelist.FileList(self.image, self.destination_fmri, - self.__progtrack, self.check_cancelation) self.__progtrack.download_start_pkg(self.get_xfername()) + mfile = self.image.transport.multi_file(self.destination_fmri, + self.__progtrack, self.check_cancelation) + + if not mfile: + self.__progtrack.download_end_pkg() + return + for src, dest in itertools.chain(*self.actions): - if dest: - if dest.needsdata(src): - flist.add_action(dest) + if dest and dest.needsdata(src): + mfile.add_action(dest) - flist.flush() + mfile.wait_files() self.__progtrack.download_end_pkg() def gen_install_actions(self): --- old/src/modules/client/retrieve.py Wed Jul 1 14:56:36 2009 +++ /dev/null Wed Jul 1 14:56:36 2009 @@ -1,508 +0,0 @@ -#!/usr/bin/python2.4 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -import socket -import httplib -import urllib2 - -import pkg.fmri -import pkg.client.imagestate as imagestate -import pkg.updatelog as updatelog -from pkg.misc import versioned_urlopen -from pkg.misc import TransferTimedOutException -from pkg.misc import TransferIOException -from pkg.misc import TruncatedTransferException -from pkg.misc import TransferContentException -from pkg.misc import retryable_http_errors -from pkg.misc import retryable_socket_errors -from pkg.client.api_errors import InvalidDepotResponseException - -class CatalogRetrievalError(Exception): - """Used when catalog retrieval fails""" - def __init__(self, data, exc=None, prefix=None): - Exception.__init__(self) - self.data = data - self.exc = exc - self.prefix = prefix - - def __str__(self): - return str(self.data) - -class VersionRetrievalError(Exception): - """Used when catalog retrieval fails""" - def __init__(self, data, exc=None, prefix=None): - Exception.__init__(self) - self.data = data - self.exc = exc - self.prefix = prefix - - def __str__(self): - return str(self.data) - -class ManifestRetrievalError(Exception): - """Used when manifest retrieval fails""" - def __init__(self, data): - Exception.__init__(self) - self.data = data - - def __str__(self): - return str(self.data) - -class DatastreamRetrievalError(Exception): - """Used when datastream retrieval fails""" - def __init__(self, data): - Exception.__init__(self) - self.data = data - - def __str__(self): - return str(self.data) - -# client/retrieve.py - collected methods for retrieval of pkg components -# from repositories - -def get_catalog(img, pub, hdr, ts): - """Get a catalog from a remote host. Img is the image object - that we're updating. pub is the publisher from which the - catalog will be retrieved. Additional headers are contained - in hdr. Ts is the timestamp if we're performing an incremental - catalog operation.""" - - prefix = pub["prefix"] - ssl_tuple = img.get_ssl_credentials(pubent=pub) - - try: - c, v = versioned_urlopen(pub["origin"], - "catalog", [0], ssl_creds=ssl_tuple, - headers=hdr, imgtype=img.type, - uuid=img.get_uuid(prefix)) - except urllib2.HTTPError, e: - # Server returns NOT_MODIFIED if catalog is up - # to date - if e.code == httplib.NOT_MODIFIED: - # success - return True - elif e.code in retryable_http_errors: - raise TransferTimedOutException(prefix, "%d - %s" % - (e.code, e.msg)) - - raise CatalogRetrievalError("Could not retrieve catalog from" - " '%s'\nHTTPError code: %d - %s" % (prefix, e.code, e.msg)) - except urllib2.URLError, e: - if isinstance(e.args[0], socket.timeout): - raise TransferTimedOutException(prefix, e.reason) - elif isinstance(e.args[0], socket.error): - sockerr = e.args[0] - if isinstance(sockerr.args, tuple) and \ - sockerr.args[0] in retryable_socket_errors: - raise TransferIOException(prefix, - "Retryable socket error: %s" % e.reason) - - raise CatalogRetrievalError("Could not retrieve catalog from" - " '%s'\nURLError, reason: %s" % (prefix, e.reason)) - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(prefix, - "Incomplete Read from remote host") - except httplib.BadStatusLine: - raise TransferContentException(prefix, - "Unable to read status of HTTP response") - except KeyboardInterrupt: - raise - except Exception, e: - raise CatalogRetrievalError("Could not retrieve catalog " - "from '%s'\nException: str:%s repr:%r" % (prefix, - e, e), e, prefix) - - # root for this catalog - croot = "%s/catalog/%s" % (img.imgdir, prefix) - - try: - updatelog.recv(c, croot, ts, pub) - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(prefix, - "Incomplete Read from remote host") - except socket.timeout, e: - raise TransferTimedOutException(prefix) - except socket.error, e: - if isinstance(e.args, tuple) \ - and e.args[0] in retryable_socket_errors: - raise TransferIOException(prefix, - "Retryable socket error: %s" % e) - - raise CatalogRetrievalError("Could not retrieve catalog" - " from '%s'\nsocket error, reason: %s" % (prefix, e)) - except pkg.fmri.IllegalFmri, e: - raise CatalogRetrievalError("Could not retrieve catalog" - " from '%s'\nUnable to parse FMRI. Details follow:\n%s" - % (prefix, e)) - except updatelog.UpdateLogException, e: - raise CatalogRetrievalError("Could not retrieve catalog" - " from '%s'\nUnable to process update log." - " Details follow:\n%s" % (prefix, e.args)) - except EnvironmentError, e: - raise CatalogRetrievalError("Could not retrieve catalog " - "from '%s'\nException: str:%s repr:%r" % (prefix, - e, e), e, prefix) - - return True - -def __get_intent_str(img, fmri): - """Returns a string representing the intent of the client in retrieving - information based on the operation information provided by the image - history object. - """ - - op = img.history.operation_name - if not op: - # The client hasn't indicated what operation is executing. - op = "unknown" - - reason = imagestate.INTENT_INFO - target_pkg = None - initial_pkg = None - needed_by_pkg = None - current_pub = fmri.get_publisher() - - targets = img.state.get_targets() - if targets: - # Attempt to determine why the client is retrieving the - # manifest for this fmri and what its current target is. - target, reason = targets[-1] - - # Compare the FMRIs with no publisher information embedded. - na_current = fmri.get_fmri(anarchy=True) - na_target = target.get_fmri(anarchy=True) - - if na_target == na_current: - # Only provide this information if the fmri for the - # manifest being retrieved matches the fmri of the - # target. If they do not match, then the target fmri is - # being retrieved for information purposes only (e.g. - # dependency calculation, etc.). - target_pub = target.get_publisher() - if target_pub == current_pub: - # Prevent providing information across - # publishers. - target_pkg = na_target[len("pkg:/"):] - else: - target_pkg = "unknown" - - # The very first fmri should be the initial target that - # caused the current and needed_by fmris to be - # retrieved. - initial = targets[0][0] - initial_pub = initial.get_publisher() - if initial_pub == current_pub: - # Prevent providing information across - # publishers. - initial_pkg = initial.get_fmri( - anarchy=True)[len("pkg:/"):] - - if target_pkg == initial_pkg: - # Don't bother sending the target - # information if it is the same - # as the initial target (i.e. the - # manifest for foo@1.0 is being - # retrieved because the user is - # installing foo@1.0). - target_pkg = None - - else: - # If they didn't match, indicate that - # the needed_by_pkg was a dependency of - # another, but not which one. - initial_pkg = "unknown" - - if len(targets) > 1: - # The fmri responsible for the current one being - # processed should immediately precede the - # current one in the target list. - needed_by = targets[-2][0] - - needed_by_pub = needed_by.get_publisher() - if needed_by_pub == current_pub: - # To prevent dependency information - # being shared across publisher - # boundaries, publishers must match. - needed_by_pkg = needed_by.get_fmri( - anarchy=True)[len("pkg:/"):] - else: - # If they didn't match, indicate that - # the package is needed by another, but - # not which one. - needed_by_pkg = "unknown" - else: - # An operation is being performed that has not provided any - # target information and is likely for informational purposes - # only. Assume the "initial target" is what is being retrieved. - initial_pkg = str(fmri)[len("pkg:/"):] - - prior_version = None - if reason != imagestate.INTENT_INFO: - # Only provide version information for non-informational - # operations. - prior = img.get_version_installed(fmri) - - try: - prior_version = prior.version - except AttributeError: - # We didn't get a match back, drive on. - pass - else: - prior_pub = prior.get_publisher() - if prior_pub != current_pub: - # Prevent providing information across - # publishers by indicating that a prior - # version was installed, but not which one. - prior_version = "unknown" - - info = { - "operation": op, - "prior_version": prior_version, - "reason": reason, - "target": target_pkg, - "initial_target": initial_pkg, - "needed_by": needed_by_pkg, - } - - # op/prior_version/reason/initial_target/needed_by/ - return "(%s)" % ";".join([ - "%s=%s" % (key, info[key]) for key in info - if info[key] is not None - ]) - -def get_datastream(img, fmri, fhash): - """Retrieve a file handle based on a package fmri and a file hash. - """ - - publisher = fmri.get_publisher_str() - publisher = pkg.fmri.strip_pub_pfx(publisher) - url_prefix = img.get_url_by_publisher(publisher) - ssl_tuple = img.get_ssl_credentials(publisher) - uuid = img.get_uuid(publisher) - - try: - f = versioned_urlopen(url_prefix, "file", [0], fhash, - ssl_creds=ssl_tuple, imgtype=img.type, uuid=uuid)[0] - except urllib2.HTTPError, e: - raise DatastreamRetrievalError("Could not retrieve file '%s'\n" - "from '%s'\nHTTPError, code: %d" % - (fhash, url_prefix, e.code)) - except urllib2.URLError, e: - raise DatastreamRetrievalError("Could not retrieve file '%s'\n" - "from '%s'\nURLError args:%s" % (fhash, url_prefix, - " ".join([str(a) for a in e.args]))) - except KeyboardInterrupt: - raise - except Exception, e: - raise DatastreamRetrievalError("Could not retrieve file '%s'\n" - "from '%s'\nException: str:%s repr:%r" % - (fmri.get_url_path(), url_prefix, e, e)) - - return f - -def __get_manifest(img, fmri, method): - """Given an image object, fmri, and http method; return a file object - for the related manifest and send intent information. - """ - - publisher = fmri.get_publisher_str() - publisher = pkg.fmri.strip_pub_pfx(publisher) - url_prefix = img.get_url_by_publisher(publisher) - ssl_tuple = img.get_ssl_credentials(publisher) - uuid = img.get_uuid(publisher) - - # Tell the server why this resource is being requested. - headers = { - "X-IPkg-Intent": __get_intent_str(img, fmri) - } - - return versioned_urlopen(url_prefix, "manifest", [0], - fmri.get_url_path(), ssl_creds=ssl_tuple, imgtype=img.type, - method=method, headers=headers, uuid=uuid)[0] - -def get_manifest(img, fmri): - """Retrieve the manifest for the given fmri. Return it as a buffer to - the caller. - """ - - publisher = fmri.tuple()[0] - publisher = pkg.fmri.strip_pub_pfx(publisher) - url_prefix = img.get_url_by_publisher(publisher) - - try: - m = __get_manifest(img, fmri, "GET") - except urllib2.HTTPError, e: - if e.code in retryable_http_errors: - raise TransferTimedOutException(url_prefix, "%d - %s" % - (e.code, e.msg)) - - raise ManifestRetrievalError("Could not retrieve manifest" - " '%s' from '%s'\nHTTPError code: %d - %s" % - (fmri.get_url_path(), url_prefix, e.code, e.msg)) - except urllib2.URLError, e: - if isinstance(e.args[0], socket.timeout): - raise TransferTimedOutException(url_prefix, e.reason) - elif isinstance(e.args[0], socket.error): - sockerr = e.args[0] - if isinstance(sockerr.args, tuple) and \ - sockerr.args[0] in retryable_socket_errors: - raise TransferIOException(url_prefix, - "Retryable socket error: %s" % e.reason) - - raise ManifestRetrievalError("Could not retrieve manifest" - " '%s' from '%s'\nURLError, reason: %s" % - (fmri.get_url_path(), url_prefix, e.reason)) - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(url_prefix, - "Incomplete Read from remote host") - except httplib.BadStatusLine: - raise TransferContentException(url_prefix, - "Unable to read status of HTTP response") - except KeyboardInterrupt: - raise - except Exception, e: - raise ManifestRetrievalError("Could not retrieve manifest" - " '%s' from '%s'\nException: str:%s repr:%r" % - (fmri.get_url_path(), url_prefix, e, e)) - - cl_size = int(m.info().getheader("Content-Length", "-1")) - - try: - mfst = m.read() - mfst_len = len(mfst) - except socket.timeout, e: - raise TransferTimedOutException(url_prefix) - except socket.error, e: - if isinstance(e.args, tuple) \ - and e.args[0] in retryable_socket_errors: - raise TransferIOException(url_prefix, - "Retryable socket error: %s" % e) - - raise ManifestRetrievalError("Could not retrieve" - " manifest from '%s'\nsocket error, reason: %s" % - (url_prefix, e)) - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(url_prefix, - "Incomplete Read from remote host") - except EnvironmentError, e: - raise ManifestRetrievalError("Could not retrieve manifest" - " '%s' from '%s'\nException: str:%s repr:%r" % - (fmri.get_url_path(), url_prefix, e, e)) - - if cl_size > -1 and mfst_len != cl_size: - raise TruncatedTransferException(m.geturl(), mfst_len, cl_size) - - return mfst - -def touch_manifest(img, fmri): - """Perform a HEAD operation on the manifest for the given fmri. - """ - - publisher = fmri.get_publisher_str() - publisher = pkg.fmri.strip_pub_pfx(publisher) - - try: - __get_manifest(img, fmri, "HEAD") - except KeyboardInterrupt: - raise - except: - # All other errors are ignored as this is a non-critical - # operation that returns no information. - pass - -def get_versions(img, pub): - """Get version information from a remote host. - - Img is the image object that the retrieve is using. - pub is the publisher that will be queried for version information.""" - - prefix = pub["prefix"] - ssl_tuple = img.get_ssl_credentials(pubent=pub) - - try: - s, v = versioned_urlopen(pub["origin"], - "versions", [0], ssl_creds=ssl_tuple, - imgtype=img.type, uuid=img.get_uuid(prefix)) - except urllib2.HTTPError, e: - if e.code in retryable_http_errors: - raise TransferTimedOutException(prefix, "%d - %s" % - (e.code, e.msg)) - - raise VersionRetrievalError("Could not retrieve versions from" - " '%s'\nHTTPError code: %d - %s" % (prefix, e.code, e.msg)) - except urllib2.URLError, e: - if isinstance(e.args[0], socket.timeout): - raise TransferTimedOutException(prefix, e.reason) - elif isinstance(e.args[0], socket.error): - sockerr = e.args[0] - if isinstance(sockerr.args, tuple) and \ - sockerr.args[0] in retryable_socket_errors: - raise TransferIOException(prefix, - "Retryable socket error: %s" % e.reason) - - raise VersionRetrievalError("Could not retrieve versions from" - " '%s'\nURLError, reason: %s" % (prefix, e.reason)) - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(prefix, - "Incomplete Read from remote host") - except httplib.BadStatusLine: - raise TransferContentException(prefix, - "Unable to read status of HTTP response") - except KeyboardInterrupt: - raise - except Exception, e: - raise VersionRetrievalError("Could not retrieve versions " - "from '%s'\nException: str:%s repr:%r" % (prefix, - e, e), e, prefix) - - try: - verlines = s.readlines() - except (ValueError, httplib.IncompleteRead): - raise TransferContentException(prefix, - "Incomplete Read from remote host") - except socket.timeout, e: - raise TransferTimedOutException(prefix) - except socket.error, e: - if isinstance(e.args, tuple) and \ - e.args[0] in retryable_socket_errors: - raise TransferIOException(prefix, - "Retryable socket error: %s" % e) - - raise VersionRetrievalError("Could not retrieve versions" - " from '%s'\nsocket error, reason: %s" % (prefix, e)) - except EnvironmentError, e: - raise VersionRetrievalError("Could not retrieve versions " - "from '%s'\nException: str:%s repr:%r" % (prefix, - e, e), e, prefix) - - # Convert the version lines to a method:version dictionary - try: - return dict( - s.split(None, 1) - for s in (l.strip() for l in verlines) - ) - except ValueError: - raise InvalidDepotResponseException(pub["origin"], - "Unable to parse server response") --- /dev/null Wed Jul 1 14:56:37 2009 +++ new/src/modules/client/transport/__init__.py Wed Jul 1 14:56:36 2009 @@ -0,0 +1,28 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +__all__ = [ "transport" ] --- /dev/null Wed Jul 1 14:56:37 2009 +++ new/src/modules/client/transport/engine.py Wed Jul 1 14:56:37 2009 @@ -0,0 +1,692 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +import errno +import httplib +import os +import pycurl +import urlparse + +# Need to ignore SIGPIPE if using pycurl in NOSIGNAL mode. +try: + import signal + signal.signal(signal.SIGPIPE, signal.SIG_IGN) +except ImportError: + pass + +import pkg.client.api_errors as api_errors +import pkg.client.transport.exception as tx +import pkg.client.transport.fileobj as fileobj + +from collections import deque +from pkg.client import global_settings + +class TransportEngine(object): + """This is an abstract class. It shouldn't implement any + of the methods that it contains. Leave that to transport-specific + implementations.""" + + +class CurlTransportEngine(TransportEngine): + """Concrete class of TransportEngine for libcurl transport.""" + + def __init__(self, transport, max_conn=10): + + # Backpointer to transport object + self.__xport = transport + # Curl handles + self.__mhandle = pycurl.CurlMulti() + self.__chandles = [] + self.__active_handles = 0 + self.__max_handles = max_conn + # Request queue + self.__req_q = deque() + # List of failures + self.__failures = [] + # Set default file buffer size at 128k, callers override + # this setting after looking at VFS block size. + self.__file_bufsz = 131072 + # Header bits and pieces + self.__user_agent = None + self.__common_header = {} + + # Set options on multi-handle + self.__mhandle.setopt(pycurl.M_PIPELINING, 1) + + # initialize easy handles + for i in range(self.__max_handles): + eh = pycurl.Curl() + eh.url = None + eh.repourl = None + eh.fobj = None + eh.filepath = None + eh.success = False + eh.fileprog = None + self.__chandles.append(eh) + + # copy handles into handle freelist + self.__freehandles = self.__chandles[:] + + def __call_perform(self): + """An internal method that invokes the multi-handle's + perform method.""" + + while 1: + ret, active_handles = self.__mhandle.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + self.__active_handles = active_handles + return ret + + def add_url(self, url, filepath=None, writefunc=None, header=None, + progtrack=None, sslcert=None, sslkey=None, repourl=None): + """Add a URL to the transport engine. Caller must supply + either a filepath where the file should be downloaded, + or a callback to a function that will peform the write. + It may also optionally supply header information + in a dictionary. If the caller has a ProgressTracker, + supply the object in the progtrack argument.""" + + t = TransportRequest(url, filepath=filepath, + writefunc=writefunc, header=header, progtrack=progtrack, + sslcert=sslcert, sslkey=sslkey, repourl=repourl) + + self.__req_q.appendleft(t) + + def __cleanup_requests(self): + """Cleanup handles that have finished their request. + Return the handles to the freelist. Generate any + relevant error information.""" + + count, good, bad = self.__mhandle.info_read() + failures = self.__failures + done_handles = [] + ex_to_raise = None + + for h, en, em in bad: + + # Get statistics for each handle. + repostats = self.__xport.stats[h.repourl] + repostats.record_tx() + bytes = h.getinfo(pycurl.SIZE_DOWNLOAD) + seconds = h.getinfo(pycurl.TOTAL_TIME) + repostats.record_progress(bytes, seconds) + + httpcode = h.getinfo(pycurl.RESPONSE_CODE) + url = h.url + urlstem = h.repourl + proto = urlparse.urlsplit(url)[0] + + # All of these are errors + repostats.record_error() + + # If we were cancelled, raise an API error. + # Otherwise fall through to transport's exception + # generation. + if en == pycurl.E_ABORTED_BY_CALLBACK: + ex_to_raise = api_errors.CanceledException + elif en == pycurl.E_HTTP_RETURNED_ERROR: + ex = tx.TransportProtoError(proto, httpcode, + url, repourl=urlstem) + else: + ex = tx.TransportFrameworkError(en, url, em, + repourl=urlstem) + + if ex.retryable: + failures.append(ex) + elif not ex_to_raise: + ex_to_raise = ex + + done_handles.append(h) + if h.fileprog: + h.fileprog.abort() + + for h in good: + # Get statistics for each handle. + repostats = self.__xport.stats[h.repourl] + repostats.record_tx() + bytes = h.getinfo(pycurl.SIZE_DOWNLOAD) + seconds = h.getinfo(pycurl.TOTAL_TIME) + repostats.record_progress(bytes, seconds) + + httpcode = h.getinfo(pycurl.RESPONSE_CODE) + url = h.url + urlstem = h.repourl + proto = urlparse.urlsplit(url)[0] + + if httpcode == httplib.OK: + h.success = True + else: + ex = tx.TransportProtoError(proto, + httpcode, url, repourl=urlstem) + + # If code >= 400, record this as an error. + # Handlers above the engine get to decide + # for 200/300 codes that aren't OK + if httpcode >= 400: + repostats.record_error() + # If code == 0, libcurl failed to read + # any HTTP status. Response is almost + # certainly corrupted. + elif httpcode == 0: + reason = "Invalid HTTP status code " \ + "from server" + ex = tx.TransportProtoError(proto, + url=url, reason=reason, + repourl=urlstem) + ex.retryable = True + + # Stash retryable failures, arrange + # to raise first fatal error after + # cleanup. + if ex.retryable: + failures.append(ex) + elif not ex_to_raise: + ex_to_raise = ex + + if h.fileprog: + h.fileprog.abort() + + done_handles.append(h) + + # Call to remove_handle must be separate from info_read() + for h in done_handles: + self.__mhandle.remove_handle(h) + self.__teardown_handle(h) + self.__freehandles.append(h) + + self.__failures = failures + + if ex_to_raise: + raise ex_to_raise + + def check_status(self, urllist=None): + """Return information about retryable failures that occured + during the request. + + This is a list of transport exceptions. Caller + may raise these, or process them for failure information. + + Urllist is an optional argument to return only failures + for a specific URLs. Not all callers of check status + want to claim the error state of all pending transactions. + + Transient errors are part of standard control flow. + The caller will look at these and decide whether + to throw them or not. Permanent failures are raised + by the transport engine as soon as they occur.""" + + # if list not specified, return all failures + if not urllist: + rf = self.__failures + self.__failures = [] + + return rf + + # otherwise, look for failures that match just the URLs + # in urllist. + rf = [] + + for tf in self.__failures: + if hasattr(tf, "url") and tf.url in urllist: + rf.append(tf) + + # remove failues in separate pass, or else for loop gets + # confused. + for f in rf: + self.__failures.remove(f) + + return rf + + def get_url(self, url, header=None, sslcert=None, sslkey=None, + repourl=None, compressible=False): + """Invoke the engine to retrieve a single URL. Callers + wishing to obtain multiple URLs at once should use + addUrl() and run(). + + getUrl will return a read-only file object that allows access + to the URL's data.""" + + fobj = fileobj.StreamingFileObj(url, self) + + t = TransportRequest(url, writefunc=fobj.get_write_func(), + hdrfunc=fobj.get_header_func(), header=header, + sslcert=sslcert, sslkey=sslkey, repourl=repourl, + compressible=compressible) + + self.__req_q.appendleft(t) + + return fobj + + def get_url_header(self, url, header=None, sslcert=None, sslkey=None, + repourl=None): + """Invoke the engine to retrieve a single URL's headers. + + getUrlHeader will return a read-only file object that + contains no data.""" + + fobj = fileobj.StreamingFileObj(url, self) + + t = TransportRequest(url, writefunc=fobj.get_write_func(), + hdrfunc=fobj.get_header_func(), header=header, + httpmethod="HEAD", sslcert=sslcert, sslkey=sslkey, + repourl=repourl) + + self.__req_q.appendleft(t) + + return fobj + + @property + def pending(self): + """Returns true if the engine still has outstanding + work to perform, false otherwise.""" + + return len(self.__req_q) > 0 or self.__active_handles > 0 + + def run(self): + """Run the transport engine. This polls the underlying + framework to complete any asynchronous I/O. Synchronous + operations should have completed when startRequest + was invoked.""" + + if not self.pending: + return + + while 1: + + if self.__active_handles > 0: + rc = self.__mhandle.select(1.0) + if rc == -1: + # Select timed out, try again. + continue + + while self.__req_q and self.__freehandles: + t = self.__req_q.pop() + eh = self.__freehandles.pop(-1) + self.__setup_handle(eh, t) + self.__mhandle.add_handle(eh) + + self.__call_perform() + + self.__cleanup_requests() + + break + + def remove_request(self, url): + """In order to remove a request, it may be necessary + to walk all of the items in the request queue, all of the + currently active handles, and the list of any transient + failures. This is expensive, so only remove a request + if absolutely necessary.""" + + for h in self.__chandles: + if h.url == url and h not in self.__freehandles: + self.__mhandle.remove_handle(h) + self.__teardown_handle(h) + return + + for i, t in enumerate(self.__req_q): + if t.url == url: + del self.__req_q[i] + return + + for ex in self.__failures: + if ex.url == url: + self.__failures.remove(ex) + return + + def reset(self): + """Reset the state of the transport engine. Do this + before performing another type of request.""" + + for c in self.__chandles: + if c not in self.__freehandles: + self.__mhandle.remove_handle(c) + self.__teardown_handle(c) + + self.__active_handles = 0 + self.__freehandles = self.__chandles[:] + self.__req_q = deque() + + def send_data(self, url, data, header=None, sslcert=None, sslkey=None, + repourl=None): + """Invoke the engine to retrieve a single URL. + This routine sends the data in data, and returns the + server's response. + + Callers wishing to obtain multiple URLs at once should use + addUrl() and run(). + + sendData will return a read-only file object that allows access + to the server's response..""" + + fobj = fileobj.StreamingFileObj(url, self) + + t = TransportRequest(url, writefunc=fobj.get_write_func(), + hdrfunc=fobj.get_header_func(), header=header, data=data, + httpmethod="POST", sslcert=sslcert, sslkey=sslkey, + repourl=repourl) + + self.__req_q.appendleft(t) + + return fobj + + def set_file_bufsz(self, size): + """If the downloaded files are being written out by + the file() mechanism, and not written using a callback, + the I/O is buffered. Set the buffer size using + this function. If it's not set, a default of 131072 (128k) + is used.""" + + if size <= 0: + self.__file_bufsz = 8192 + return + + self.__file_bufsz = size + + def set_header(self, hdrdict=None): + """Supply a dictionary of name/value pairs in hdrdict. + These will be included on all requests issued by the transport + engine. To append a specific header to a certain request, + supply a dictionary to the header argument of addUrl.""" + + if not hdrdict: + self.__common_header = {} + return + + self.__common_header = hdrdict + + def set_user_agent(self, ua_str): + """Supply a string str and the transport engine will + use this string as its User-Agent header. This is + a header that will be common to all transport requests.""" + + self.__user_agent = ua_str + + def __setup_handle(self, hdl, treq): + """Setup the curl easy handle, hdl, with the parameters + specified in the TransportRequest treq. If global + parameters are set, apply these to the handle as well.""" + + # Set nosignal, so timeouts don't crash client + hdl.setopt(pycurl.NOSIGNAL, 1) + + # Use globally set timeout + hdl.setopt(pycurl.TIMEOUT, global_settings.PKG_CLIENT_TIMEOUT) + + # Follow redirects + hdl.setopt(pycurl.FOLLOWLOCATION, True) + + # Set user agent, if client has defined it + if self.__user_agent: + hdl.setopt(pycurl.USERAGENT, self.__user_agent) + + # Take header dictionaries and convert them into lists + # of header strings. + if len(self.__common_header) > 0 or \ + (treq.header and len(treq.header) > 0): + + headerlist = [] + + # Headers common to all requests + for k, v in self.__common_header.iteritems(): + headerstr = "%s: %s" % (k, v) + headerlist.append(headerstr) + + # Headers specific to this request + if treq.header: + for k, v in treq.header.iteritems(): + headerstr = "%s: %s" % (k, v) + headerlist.append(headerstr) + + hdl.setopt(pycurl.HTTPHEADER, headerlist) + + # Set request url. Also set attribute on handle. + hdl.setopt(pycurl.URL, treq.url) + hdl.url = treq.url + # The repourl is the url stem that identifies the + # repository. This is useful to have around for coalescing + # error output, and statistics reporting. + hdl.repourl = treq.repourl + if treq.filepath: + try: + hdl.fobj = open(treq.filepath, "wb+", + self.__file_bufsz) + except EnvironmentError, e: + if e.errno == errno.EACCES: + raise api_errors.PermissionsException( + e.filename) + # Raise OperationError if it's not EACCES + raise tx.TransportOperationError( + "Unable to open file: %s" % e) + + hdl.setopt(pycurl.WRITEDATA, hdl.fobj) + hdl.filepath = treq.filepath + elif treq.writefunc: + hdl.setopt(pycurl.WRITEFUNCTION, treq.writefunc) + hdl.setopt(pycurl.FAILONERROR, True) + hdl.filepath = None + hdl.fobj = None + else: + raise tx.TransportOperationError("Transport invocation" + " for URL %s did not specify filepath or write" + " function." % treq.url) + + if treq.progtrack: + hdl.setopt(pycurl.NOPROGRESS, 0) + hdl.fileprog = FileProgress(treq.progtrack) + hdl.setopt(pycurl.PROGRESSFUNCTION, + hdl.fileprog.progress_callback) + + if treq.compressible: + hdl.setopt(pycurl.ENCODING, "") + + if treq.hdrfunc: + hdl.setopt(pycurl.HEADERFUNCTION, treq.hdrfunc) + + if treq.httpmethod == "HEAD": + hdl.setopt(pycurl.NOBODY, True) + elif treq.httpmethod == "POST": + hdl.setopt(pycurl.POST, True) + hdl.setopt(pycurl.POSTFIELDS, treq.data) + else: + # Default to GET + hdl.setopt(pycurl.HTTPGET, True) + + # Set up SSL options + if treq.sslcert: + hdl.setopt(pycurl.SSLCERT, treq.sslcert) + if treq.sslkey: + hdl.setopt(pycurl.SSLKEY, treq.sslkey) + # Options that apply when SSL is enabled + if treq.sslcert or treq.sslkey: + # Verify that peer's CN matches CN on certificate + hdl.setopt(pycurl.SSL_VERIFYHOST, 2) + + cadir = self.__xport.get_ca_dir() + if cadir: + hdl.setopt(pycurl.SSL_VERIFYPEER, 1) + hdl.setopt(pycurl.CAPATH, cadir) + hdl.unsetopt(pycurl.CAINFO) + else: + hdl.setopt(pycurl.SSL_VERIFYPEER, 0) + + def __shutdown(self): + """Shutdown the transport engine, perform cleanup.""" + + self.reset() + + for c in self.__chandles: + c.close() + + self.__chandles = None + self.__freehandles = None + self.__mhandle.close() + self.__mhandle = None + + def __teardown_handle(self, hdl): + """Cleanup any state that we've associated with this handle. + After a handle has been torn down, it should still be valid + for use, but should have no previous state. To remove + handles from use completely, use __shutdown.""" + + hdl.reset() + if hdl.fobj: + hdl.fobj.close() + hdl.fobj = None + if not hdl.success: + try: + os.remove(hdl.filepath) + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise \ + tx.TransportOperationError( + "Unable to remove file: %s" + % e) + hdl.url = None + hdl.repourl = None + hdl.success = False + hdl.filepath = None + hdl.fileprog = None + + +class FileProgress(object): + """This class bridges the interfaces between a ProgressTracker + object and the progress callback that's provided by Pycurl. + Since progress callbacks are per curl handle, and handles aren't + guaranteed to succeed, this object watches a handle's progress + and updates the tracker accordingly. If the handle fails, + it will correctly remove the bytes from the file. The curl + callback reports bytes even when it doesn't make progress. + It's necessary to keep additonal state here, since the client's + ProgressTracker has global counts of the bytes. If we're + unable to keep a per-file count, the numbers will get + lost quickly.""" + + def __init__(self, progtrack): + self.progtrack = progtrack + self.dltotal = 0 + self.dlcurrent = 0 + self.completed = False + + def abort(self): + """Download failed. Remove the amount of bytes downloaded + by this file from the ProgressTracker.""" + + self.progtrack.download_add_progress(0, -self.dlcurrent) + if self.completed: + self.progtrack.download_add_progress(-1, 0) + + self.dltotal = 0 + self.dlcurrent = 0 + + def progress_callback(self, dltot, dlcur, ultot, ulcur): + """Called by pycurl/libcurl framework to update + progress tracking.""" + + if hasattr(self.progtrack, "check_cancelation") and \ + self.progtrack.check_cancelation(): + return -1 + + if self.dltotal != dltot: + self.dltotal = dltot + + new_progress = dlcur - self.dlcurrent + if new_progress > 0: + self.dlcurrent += new_progress + self.progtrack.download_add_progress(0, new_progress) + + if not self.completed and self.dlcurrent == self.dltotal: + self.completed = True + self.progtrack.download_add_progress(1, 0) + + return 0 + + +class TransportRequest(object): + """A class that contains per-request information for the underlying + transport engines. This is used to set per-request options that + are used either by the framework, the transport, or both.""" + + def __init__(self, url, filepath=None, writefunc=None, + hdrfunc=None, header=None, data=None, httpmethod="GET", + progtrack=None, sslcert=None, sslkey=None, repourl=None, + compressible=False): + """Create a TransportRequest with the following parameters: + + url - The url that the transport engine should retrieve + + filepath - If defined, the transport engine will download the + file to this path. If not defined, the caller should + supply a write function. + + writefunc - A function, supplied instead of filepath, that + reads the bytes supplied by the transport engine and writes + them somewhere for processing. This is a callback. + + hdrfunc - A callback for examining the contents of header + data in a response to a transport request. + + header - A dictionary of key/value pairs to be included + in the request's header. + + compressible - A boolean value that indicates whether + the content that is requested is a candidate for transport + level compression. + + data - If the request is sending a data payload, include + the data in this argument. + + httpmethod - If the request is a HTTP/HTTPS request, + this can override the default HTTP method of GET. + + progtrack - If the transport wants the engine to update + the progress of the download, supply a ProgressTracker + object in this argument. + + repouri - This is the URL stem that identifies the repo. + It's a subset of url. It's also used by the stats system. + + sslcert - If the request is using SSL, HTTPS for example, + provide a path to the SSL certificate here. + + sslkey - If the request is using SSL, liks HTTPS for example, + provide a path to the SSL key here.""" + + self.url = url + self.filepath = filepath + self.writefunc = writefunc + self.hdrfunc = hdrfunc + self.header = header + self.data = data + self.httpmethod = httpmethod + self.progtrack = progtrack + self.repourl = repourl + self.sslcert = sslcert + self.sslkey = sslkey + self.compressible = compressible --- /dev/null Wed Jul 1 14:56:37 2009 +++ new/src/modules/client/transport/exception.py Wed Jul 1 14:56:37 2009 @@ -0,0 +1,307 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +import httplib +import pycurl + +import pkg.client.api_errors as api_errors + +retryable_http_errors = set((httplib.REQUEST_TIMEOUT, httplib.BAD_GATEWAY, + httplib.GATEWAY_TIMEOUT, httplib.NOT_FOUND)) + +# Different protocols may have different retryable errors. Map proto +# to set of retryable errors. + +retryable_proto_errors = { "http" : retryable_http_errors, + "https" : retryable_http_errors } + +retryable_pycurl_errors = set((pycurl.E_COULDNT_CONNECT, pycurl.E_PARTIAL_FILE, + pycurl.E_OPERATION_TIMEOUTED, pycurl.E_GOT_NOTHING, pycurl.E_SEND_ERROR, + pycurl.E_RECV_ERROR, pycurl.E_COULDNT_RESOLVE_HOST)) + +class TransportException(api_errors.TransportError): + """Base class for various exceptions thrown by code in transport + package.""" + def __init__(self): + self.count = 1 + self.rcount = 1 + self.retryable = False + self.verbose = False + + def simple_cmp(self, other): + """Subclasses that wish to provided a simplified output + interface must implement this routine and simple_str.""" + + return self.__cmp__(self, other) + + def simple_str(self): + """Subclasses that wish to provided a simplified output + interface must implement this routine and simple_cmp.""" + + return self.__str__() + + +class TransportOperationError(TransportException): + """Used when transport operations fail for miscellaneous reasons.""" + def __init__(self, data): + TransportException.__init__(self) + self.data = data + + def __str__(self): + return str(self.data) + + +class TransportFailures(TransportException): + """This exception encapsulates multiple transport exceptions.""" + + # + # This class is a subclass of TransportException so that calling + # code can reasonably 'except TransportException' and get either + # a single-valued or in this case a multi-valued instance. + # + def __init__(self): + TransportException.__init__(self) + self.exceptions = [] + self.reduced_ex = [] + + def append(self, exc): + found = False + + assert isinstance(exc, TransportException) + for x in self.exceptions: + if cmp(x, exc) == 0: + x.count += 1 + found = True + break + + if not found: + self.exceptions.append(exc) + + found = False + for x in self.reduced_ex: + if x.simple_cmp(exc) == 0: + x.rcount += 1 + found = True + break + + if not found: + self.reduced_ex.append(exc) + + + def __str__(self): + if self.verbose: + return self.detailed_str() + + return self.simple_str() + + def detailed_str(self): + if len(self.exceptions) == 0: + return "[no errors accumulated]" + + s = "" + for i, x in enumerate(self.exceptions): + if len(self.exceptions) > 1: + s += "%d: " % (i + 1) + s += str(x) + if x.count > 1: + s += " (happened %d times)" % x.count + s += "\n" + return s + + def simple_str(self): + if len(self.reduced_ex) == 0: + return "[no errors accumulated]" + + s = "" + for i, x in enumerate(self.reduced_ex): + if len(self.reduced_ex) > 1: + s += "%d: " % (i + 1) + s += x.simple_str() + if x.rcount > 1: + s += " (happened %d times)" % x.rcount + s += "\n" + return s + + def __len__(self): + return len(self.exceptions) + + +class TransportProtoError(TransportException): + """Raised when errors occur in the transport protocol.""" + + def __init__(self, proto, code=None, url=None, reason=None, + repourl=None): + TransportException.__init__(self) + self.proto = proto + self.code = code + self.url = url + self.urlstem = repourl + self.reason = reason + self.retryable = self.code in retryable_proto_errors[self.proto] + + def __str__(self): + s = "%s protocol error" % self.proto + if self.code: + s += ": code: %d" % self.code + if self.reason: + s += "\nreason: %s" % self.reason + if self.url: + s += "\nURL: '%s'." % self.url + return s + + def __cmp__(self, other): + if not isinstance(other, TransportProtoError): + return -1 + r = cmp(self.proto, other.proto) + if r != 0: + return r + r = cmp(self.code, other.code) + if r != 0: + return r + r = cmp(self.url, other.url) + if r != 0: + return r + return cmp(self.reason, other.reason) + + def simple_cmp(self, other): + if not isinstance(other, TransportProtoError): + return -1 + r = cmp(self.proto, other.proto) + if r != 0: + return r + r = cmp(self.code, other.code) + if r != 0: + return r + return cmp(self.urlstem, other.urlstem) + + def simple_str(self): + s = "%s protocol error" % self.proto + if self.code: + s += ": code: %d" % self.code + if self.urlstem: + s += "\nURL: '%s'." % self.urlstem + return s + + +class TransportFrameworkError(TransportException): + """Raised when errors occur in the transport framework.""" + + def __init__(self, code, url=None, reason=None, repourl=None): + TransportException.__init__(self) + self.code = code + self.url = url + self.urlstem = repourl + self.reason = reason + self.retryable = self.code in retryable_pycurl_errors + + def __str__(self): + s = "Framework error: code: %d" % self.code + if self.reason: + s += " reason: %s" % self.reason + if self.url: + s += "\nURL: '%s'." % self.url + return s + + def __cmp__(self, other): + if not isinstance(other, TransportFrameworkError): + return -1 + r = cmp(self.code, other.code) + if r != 0: + return r + r = cmp(self.url, other.url) + if r != 0: + return r + return cmp(self.reason, other.reason) + + def simple_cmp(self, other): + if not isinstance(other, TransportFrameworkError): + return -1 + r = cmp(self.code, other.code) + if r != 0: + return r + r = cmp(self.reason, other.reason) + if r != 0: + return r + return cmp(self.urlstem, other.urlstem) + + def simple_str(self): + s = "Framework error: code: %d" % self.code + if self.reason: + s += " reason: %s" % self.reason + if self.urlstem: + s += "\nURL: '%s'." % self.urlstem + return s + + +class TransferContentException(TransportException): + """Raised when there are problems downloading the requested content.""" + def __init__(self, url, reason=None): + TransportException.__init__(self) + self.url = url + self.reason = reason + self.retryable = True + + def __str__(self): + s = "Transfer from '%s' failed" % self.url + if self.reason: + s += ": %s" % self.reason + s += "." + return s + + def __cmp__(self, other): + if not isinstance(other, TransferContentException): + return -1 + r = cmp(self.url, other.url) + if r != 0: + return r + return cmp(self.reason, other.reason) + + +class InvalidContentException(TransportException): + """Raised when the content's hash/chash doesn't verify, or the + content is received in an unreadable format.""" + def __init__(self, path, reason, size=0): + TransportException.__init__(self) + self.path = path + self.reason = reason + self.size = size + self.retryable = True + + def __str__(self): + s = "Invalid content for action with path %s" % self.path + if self.reason: + s += ": %s." % self.reason + return s + + def __cmp__(self, other): + if not isinstance(other, InvalidContentException): + return -1 + r = cmp(self.path, other.path) + if r != 0: + return r + return cmp(self.reason, other.reason) + --- /dev/null Wed Jul 1 14:56:37 2009 +++ new/src/modules/client/transport/fileobj.py Wed Jul 1 14:56:37 2009 @@ -0,0 +1,293 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +class StreamingFileObj(object): + + def __init__(self, url, engine): + """Create a streaming file object that wraps around a + transport engine. This is only necessary if the underlying + transport doesn't have its own streaming interface and the + repo operation needs a streaming response.""" + + self.__buf = "" + self.__url = url + self.__engine = engine + self.__data_callback_invoked = False + self.__headers_arrived = False + self.__httpmsg = None + self.__headers = {} + self.__done = False + + def __del__(self): + self.close() + + # File object methods + + def close(self): + self.__buf = "" + if not self.__done: + self.__engine.remove_request(self.__url) + self.__done = True + self.__engine = None + self.__url = None + + def flush(self): + """flush the buffer. Since this supports read, but + not write, this is a noop.""" + return + + def read(self, size=-1): + """Read size bytes from the remote connection. + If size isn't specified, read all of the data from + the remote side.""" + + if size < 0: + while self.__fill_buffer(): + # just fill the buffer + pass + curdata = self.__buf + self.__buf = "" + return curdata + else: + curdata = self.__buf + datalen = len(curdata) + if datalen >= size: + self.__buf = curdata[size:] + return curdata[:size] + while self.__fill_buffer(): + datalen = len(self.__buf) + if datalen >= size: + break + + curdata = self.__buf + datalen = len(curdata) + if datalen >= size: + self.__buf = curdata[size:] + return curdata[:size] + + self.__buf = "" + return curdata + + def readline(self, size=-1): + """Read a line from the remote host. If size is + specified, read to newline or size, whichever is smaller.""" + + if size < 0: + curdata = self.__buf + newline = curdata.find("\n") + if newline >= 0: + newline += 1 + self.__buf = curdata[newline:] + return curdata[:newline] + while self.__fill_buffer(): + newline = self.__buf.find("\n") + if newline >= 0: + break + + curdata = self.__buf + newline = curdata.find("\n") + if newline >= 0: + newline += 1 + self.__buf = curdata[newline:] + return curdata[:newline] + self.__buf = "" + return curdata + else: + curdata = self.__buf + newline = curdata.find("\n", 0, size) + datalen = len(curdata) + if newline >= 0: + newline += 1 + self.__buf = curdata[newline:] + return curdata[:newline] + if datalen >= size: + self.__buf = curdata[size:] + return curdata[:size] + while self.__fill_buffer(): + newline = self.__buf.find("\n", 0, size) + datalen = len(self.__buf) + if newline >= 0: + break + if datalen >= size: + break + + curdata = self.__buf + newline = curdata.find("\n", 0, size) + datalen = len(curdata) + if newline >= 0: + newline += 1 + self.__buf = curdata[newline:] + return curdata[:newline] + if datalen >= size: + self.__buf = curdata[size:] + return curdata[:size] + self.__buf = "" + return curdata + + def readlines(self, sizehint=0): + """Read lines from the remote host, returning an + array of the lines that were read. sizehint specifies + an approximate size, in bytes, of the total amount of data, + as lines, that should be returned to the caller.""" + + read = 0 + lines = [] + while True: + l = self.readline() + if not l: + break + lines.append(l) + read += len(l) + if sizehint and read >= sizehint: + break + + return lines + + def write(self, data): + raise NotImplementedError + + def writelines(self, llist): + raise NotImplementedError + + def get_write_func(self): + return self.__write_callback + + def get_header_func(self): + return self.__header_callback + + # Header and message methods + + def get_http_message(self): + """Return the status message that may be included + with a numerical HTTP response code. Not all HTTP + implementations are guaranteed to return this value. + In some cases it may be None.""" + + return self.__httpmsg + + def getheader(self, hdr, default): + """Return the HTTP header named hdr. If the hdr + isn't present, return default value instead.""" + + if not self.__headers_arrived: + self.__fill_headers() + + return self.__headers.get(hdr, default) + + def _prime(self): + """Used by the underlying transport before handing this + object off to other layers. It ensures that the object's + creator can catch errors that occur at connection time. + All callers must still catch transport exceptions, however.""" + + self.__fill_buffer(1) + + # Iterator methods + + def __iter__(self): + return self + + def next(self): + line = self.readline() + if not line: + raise StopIteration + return line + + # Private methods + + def __fill_buffer(self, size=-1): + """Call engine.run() to fill the file object's buffer. + Read until we might block. If size is specified, stop + once we get at least size bytes, or might block, + whichever comes first.""" + + engine = self.__engine + + while 1: + if not engine.pending: + # nothing pending means no more transfer + self.__done = True + s = engine.check_status([self.__url]) + if len(s) > 0: + # Cleanup prior to raising exception + self.close() + raise s[0] + return False + + engine.run() + + if size > 0 and len(self.__buf) < size: + # loop if we need more data in the buffer + continue + else: + # break out of this loop + break + + return True + + def __fill_headers(self): + """Run the transport until headers arrive. When the data + callback gets invoked, all headers have arrived. The + alternate scenario is when no data arrives, but the server + isn't providing more input isi over the network. In that case, + the client either received just headers, or had the transfer + close unexpectedly.""" + + while not self.__data_callback_invoked: + if not self.__fill_buffer(): + # We hit this case if we get headers + # but no data. + break + + self.__headers_arrived = True + + def __write_callback(self, data): + """A callback given to transport engine that writes data + into a buffer in this object.""" + + if not self.__data_callback_invoked: + self.__data_callback_invoked = True + + self.__buf = self.__buf + data + + def __header_callback(self, data): + """A callback given to the transport engine. It reads header + information from the transport. This function saves + the message from the http response, as well as a dictionary + of headers that it can parse.""" + + if data.startswith("HTTP/"): + rtup = data.split(None, 2) + try: + self.__httpmsg = rtup[2] + except IndexError: + pass + + elif data.find(":") > -1: + k, v = data.split(":", 1) + if v: + self.__headers[k] = v.strip() --- /dev/null Wed Jul 1 14:56:38 2009 +++ new/src/modules/client/transport/repo.py Wed Jul 1 14:56:38 2009 @@ -0,0 +1,343 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +import os +import urlparse +import urllib + +import pkg.client.transport.exception as tx + +class TransportRepo(object): + """The TransportRepo class handles transport requests. + It represents a repo, and provides the same interfaces as + the operations that are performed against a repo. Subclasses + should implement protocol specific repo modifications.""" + + def do_search(self, data, header=None): + """Perform a search request.""" + + raise NotImplementedError + + def get_catalog(self, ts=None, header=None): + """Get the catalog from the repo. If ts is defined, + request only changes newer than timestamp ts.""" + + raise NotImplementedError + + def get_manifest(self, mfst, header=None): + """Get a manifest from repo. The name of the + manifest is given in mfst. If dest is set, download + the manifest to dest.""" + + raise NotImplementedError + + def get_files(self, filelist, dest, progtrack): + """Get multiple files from the repo at once. + The files are named by hash and supplied in filelist. + If dest is specified, download to the destination + directory that is given. Progtrack is a ProgressTracker""" + + raise NotImplementedError + + def get_url(self): + """Return's the Repo's URL.""" + + raise NotImplementedError + + def get_versions(self, header=None): + """Query the repo for versions information. + Returns a fileobject.""" + + raise NotImplementedError + + def touch_manifest(self, mfst, header=None): + """Send data about operation intent without actually + downloading a manifest.""" + + raise NotImplementedError + + +class HTTPRepo(TransportRepo): + + def __init__(self, repostats, repouri, engine): + """Create a http repo. Repostats is a RepoStats object. + Repouri is a RepositoryURI object. Engine is a transport + engine object. + + The convenience function new_repo() can be used to create + the correct repo.""" + self._url = repostats.url + self._repouri = repouri + self._engine = engine + + def _add_file_url(self, url, filepath=None, progtrack=None): + self._engine.add_url(url, filepath=filepath, + progtrack=progtrack, repourl=self._url) + + def _fetch_url(self, url, header=None, compress=False): + return self._engine.get_url(url, header, repourl=self._url, + compressible=compress) + + def _fetch_url_header(self, url, header=None): + return self._engine.get_url_header(url, header, + repourl=self._url) + + def _post_url(self, url, data, header=None): + return self._engine.send_data(url, data, header, + repourl=self._url) + + def do_search(self, data, header=None): + """Perform a remote search against origin repos.""" + + methodstr = "search/1/" + + if len(data) > 1: + requesturl = urlparse.urljoin(self._repouri.uri, + methodstr) + request_data = urllib.urlencode( + [(i, str(q)) + for i, q in enumerate(data)]) + + resp = self._post_url(requesturl, request_data, + header) + + else: + baseurl = urlparse.urljoin(self._repouri.uri, + methodstr) + requesturl = urlparse.urljoin(baseurl, urllib.quote( + str(data[0]), safe='')) + + resp = self._fetch_url(requesturl, header) + + return resp + + def get_catalog(self, ts=None, header=None): + """Get the catalog from the repo. If ts is defined, + request only changes newer than timestamp ts.""" + + methodstr = "catalog/0/" + + requesturl = urlparse.urljoin(self._repouri.uri, methodstr) + + if ts: + if not header: + header = {"If-Modified-Since": ts} + else: + header["If-Modified-Since"] = ts + + return self._fetch_url(requesturl, header, compress=True) + + def get_datastream(self, fhash, header=None): + """Get a datastream from a repo. The name of the + file is given in fhash.""" + + methodstr = "file/0/" + + baseurl = urlparse.urljoin(self._repouri.uri, methodstr) + requesturl = urlparse.urljoin(baseurl, fhash) + + return self._fetch_url(requesturl, header) + + def get_manifest(self, mfst, header=None): + """Get a manifest from repo. The name of the + manifest is given in mfst.""" + + methodstr = "manifest/0/" + + baseurl = urlparse.urljoin(self._repouri.uri, methodstr) + requesturl = urlparse.urljoin(baseurl, mfst) + + return self._fetch_url(requesturl, header, compress=True) + + def get_files(self, filelist, dest, progtrack): + """Get multiple files from the repo at once. + The files are named by hash and supplied in filelist. + If dest is specified, download to the destination + directory that is given. If progtrack is not None, + it contains a ProgressTracker object for the + downloads.""" + + methodstr = "file/0/" + urllist = [] + + # create URL for requests + baseurl = urlparse.urljoin(self._repouri.uri, methodstr) + + for f in filelist: + url = urlparse.urljoin(baseurl, f) + urllist.append(url) + fn = os.path.join(dest, f) + self._add_file_url(url, filepath=fn, + progtrack=progtrack) + + while self._engine.pending: + self._engine.run() + + errors = self._engine.check_status(urllist) + + # Transient errors are part of standard control flow. + # The repo's caller will look at these and decide whether + # to throw them or not. Permanant failures are raised + # by the transport engine as soon as they occur. + # + # This adds an attribute that describes the request to the + # exception, if we were able to figure it out. + + for e in errors: + # when check_status is supplied with a list, + # all exceptions returned will have a url. + # If we didn't do this, we'd need a getattr check. + eurl = e.url + utup = urlparse.urlsplit(eurl) + req = utup[2] + req = os.path.basename(req) + e.request = req + + return errors + + def get_url(self): + """Returns the repo's url.""" + + return self._url + + def get_versions(self, header=None): + """Query the repo for versions information. + Returns a fileobject.""" + + requesturl = urlparse.urljoin(self._repouri.uri, "versions/0/") + + resp = self._fetch_url(requesturl, header) + + return resp + + def touch_manifest(self, mfst, header=None): + """Invoke HTTP HEAD to send manifest intent data.""" + + methodstr = "manifest/0/" + + baseurl = urlparse.urljoin(self._repouri.uri, methodstr) + requesturl = urlparse.urljoin(baseurl, mfst) + + resp = self._fetch_url_header(requesturl, header) + + # response is empty, or should be. + resp.read() + + return True + +class HTTPSRepo(HTTPRepo): + + def __init__(self, repostats, repouri, engine): + """Create a http repo. Repostats is a RepoStats object. + Repouri is a RepositoryURI object. Engine is a transport + engine object. + + The convenience function new_repo() can be used to create + the correct repo.""" + + HTTPRepo.__init__(self, repostats, repouri, engine) + + # override the download functions to use ssl cert/key + def _add_file_url(self, url, filepath=None, progtrack=None): + self._engine.add_url(url, filepath=filepath, + progtrack=progtrack, sslcert=self._repouri.ssl_cert, + sslkey=self._repouri.ssl_key, repourl=self._url) + + def _fetch_url(self, url, header=None, compress=False): + return self._engine.get_url(url, header=header, + sslcert=self._repouri.ssl_cert, + sslkey=self._repouri.ssl_key, repourl=self._url, + compressible=compress) + + def _fetch_url_header(self, url, header=None): + return self._engine.get_url_header(url, header=header, + sslcert=self._repouri.ssl_cert, + sslkey=self._repouri.ssl_key, repourl=self._url) + + def _post_url(self, url, data, header=None): + return self._engine.send_data(url, data, header=header, + sslcert=self._repouri.ssl_cert, + sslkey=self._repouri.ssl_key, repourl=self._url) + +# cache transport repo objects, so one isn't created on every operation + +class RepoCache(object): + """An Object that caches repository objects. Used to make + sure that repos are re-used instead of re-created for each + operation.""" + + # Schemes supported by the cache. + supported_schemes = { + "http": HTTPRepo, + "https": HTTPSRepo, + } + + def __init__(self, engine): + """Caller must include a TransportEngine.""" + + self.__engine = engine + self.__cache = {} + + def clear_cache(self): + """Flush the contents of the cache.""" + + self.__cache = {} + + def new_repo(self, repostats, repouri): + """Create a new repo server for the given repouri object.""" + + origin_url = repostats.url + urltuple = urlparse.urlparse(origin_url) + scheme = urltuple[0] + + if scheme not in RepoCache.supported_schemes: + raise tx.TransportOperationError("Scheme %s not" + " supported by transport." % scheme) + + if origin_url in self.__cache: + return self.__cache[origin_url] + + repo = RepoCache.supported_schemes[scheme](repostats, repouri, + self.__engine) + + self.__cache[origin_url] = repo + + return repo + + def remove_repo(self, repo=None, url=None): + """Remove a repo from the cache. Caller must supply + either a RepositoryURI object or a URL.""" + + if repo: + origin_url = repo.uri + elif url: + origin_url = url + else: + raise ValueError, "Must supply either a repo or a uri." + + if origin_url in self.__cache: + del self.__cache[origin_url] --- /dev/null Wed Jul 1 14:56:38 2009 +++ new/src/modules/client/transport/stats.py Wed Jul 1 14:56:38 2009 @@ -0,0 +1,194 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +import pkg.misc as misc + +class RepoChooser(object): + """An object that contains repo statistics. It applies algorithms + to choose an optimal set of repos for a given publisher, based + upon the observed repo statistics.""" + + def __init__(self): + self.__rsobj = {} + + def __getitem__(self, key): + return self.__rsobj[key] + + def __contains__(self, key): + return key in self.__rsobj + + def dump(self): + """Write the repo statistics to stdout.""" + + fmt = "%-30s %-8s %-8s %-12s %-4s %-4s" + print fmt % ("URL", "Good Tx", "Errors", "Speed", "Prio", + "Used") + + for ds in self.__rsobj.values(): + + speedstr = misc.bytes_to_str(ds.transfer_speed, + "%(num).0f %(unit)s/sec") + + print fmt % (ds.url, ds.success, ds.failures, + speedstr, ds.priority, ds.used) + + def get_repostats(self, repouri_list): + """Walk a list of repo uris and return a sorted list of + status objects. The better choices should be at the + beginning of the list.""" + + found_rs = [] + + # Walk the list of repouris that we were provided. + # If they're already in the dictionary, copy a reference + # into the found_rs list, otherwise create the object + # and then add it to our list of found objects. + for ruri in repouri_list: + url = ruri.uri.rstrip("/") + if url in self.__rsobj: + found_rs.append((self.__rsobj[url], ruri)) + else: + rs = RepoStats(ruri) + self.__rsobj[rs.url] = rs + found_rs.append((rs, ruri)) + + # XXX This is the existing sort algorithm for mirror + # selection. We should switch this to a positive definite + # quality function, where each RepoStats object is capable + # of generating its own quality number. + + found_rs.sort(key=lambda x: (x[0].failures, x[0].success)) + + # list of tuples, (repostatus, repouri) + return found_rs + + +class RepoStats(object): + """An object for keeping track of observed statistics for a particular + RepoURI. This includes things like observed performance, availability, + successful and unsuccessful transaction rates, etc.""" + + def __init__(self, repouri): + """Initialize a RepoStats object. Pass a RepositoryURI object + in repouri to configure an object for a particular + repository URI.""" + + self.__url = repouri.uri.rstrip("/") + self.__priority = repouri.priority + + self.__failed_tx = 0 + self.__total_tx = 0 + + self.__used = False + + self.__bytes_xfr = 0.0 + self.__seconds_xfr = 0.0 + + def record_error(self): + """Record that an operation to the RepositoryURI represented + by this RepoStats object failed with an error.""" + + if not self.__used: + self.__used = True + self.__failed_tx += 1 + + def record_progress(self, bytes, seconds): + """Record time and size of a network operation to a + particular RepositoryURI, represented by the RepoStats object. + Place the number of bytes transferred in the bytes argument. + The time, in seconds, should be supplied in the + seconds argument.""" + + if not self.__used: + self.__used = True + self.__bytes_xfr += bytes + self.__seconds_xfr += seconds + + def record_tx(self): + """Record that an operation to the URI represented + by this RepoStats object was initiated.""" + + if not self.__used: + self.__used = True + self.__total_tx += 1 + + @property + def bytes_xfr(self): + """Return the number of bytes transferred.""" + + return self.__bytes_xfr + + @property + def failures(self): + """Return the number of failures that the client has encountered + while trying to perform operations on this repository.""" + + return self.__failed_tx + + @property + def priority(self): + """Return the priority of the URI, if one is assigned.""" + + if self.__priority is None: + return 0 + + return self.__priority + + @property + def seconds_xfr(self): + """Return the total amount of time elapsed while performing + operations against this host.""" + + return self.__seconds_xfr + + @property + def success(self): + """Return the number of successful transaction that this client + has performed while communicating with this repository.""" + + return self.__total_tx - self.__failed_tx + + @property + def transfer_speed(self): + """Return the average transfer speed in bytes/sec for + operations against this uri.""" + + return float(self.__bytes_xfr / self.__seconds_xfr) + + @property + def url(self): + """Return the URL that identifies the repository that we're + keeping statistics about.""" + + return self.__url + + @property + def used(self): + """A boolean value that indicates whether the URI + has been used for network operations.""" + + return self.__used --- /dev/null Wed Jul 1 14:56:38 2009 +++ new/src/modules/client/transport/transport.py Wed Jul 1 14:56:38 2009 @@ -0,0 +1,801 @@ +#!/usr/bin/python +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +import sys +import os +import httplib +import statvfs +import errno +import zlib + +import pkg.fmri + +import pkg.client.api_errors as apx +import pkg.client.imageconfig as imageconfig +import pkg.client.publisher as publisher +import pkg.client.transport.engine as engine +import pkg.client.transport.exception as tx +import pkg.client.transport.repo as trepo +import pkg.client.transport.stats as tstats +import pkg.manifest as manifest +import pkg.misc as misc +import pkg.portable as portable +import pkg.updatelog as updatelog + +from pkg.actions import MalformedActionError +from pkg.client import global_settings + +class Transport(object): + """The generic transport wrapper object. Its public methods should + be used by all client code that wishes to perform file/network + packaging operations.""" + + def __init__(self, img): + """Initialize the Transport object. If an Image object + is provided in img, use that to determine some of the + destination locations for transport operations.""" + + self.__img = img + self.__engine = None + self.__cadir = None + self.__portal_test_executed = False + self.__repo_cache = None + self.stats = tstats.RepoChooser() + + def __setup(self): + self.__engine = engine.CurlTransportEngine(self) + + # Configure engine's user agent based upon img configuration + ua = misc.user_agent_str(self.__img, + global_settings.client_name) + self.__engine.set_user_agent(ua) + + self.__repo_cache = trepo.RepoCache(self.__engine) + + if not self.__portal_test_executed: + self.captive_portal_test() + + def reset(self): + """Resets the transport. This needs to be done + if an install plan has been canceled and needs to + be restarted. This clears the state of the + transport and its associated components.""" + + self.__engine.reset() + self.__repo_cache.clear_cache() + + def do_search(self, pub, data): + """Perform a search request. Returns a file-like object + that contains the search results. Callers need to catch + transport exceptions that this object may generate.""" + + failures = tx.TransportFailures() + fobj = None + retry_count = global_settings.PKG_TIMEOUT_MAX + header = None + + if isinstance(pub, publisher.Publisher): + header = self.__build_header(uuid=self.__get_uuid(pub)) + + for d in self.__gen_origins(pub, retry_count): + + try: + fobj = d.do_search(data, header) + fobj._prime() + return fobj + + except tx.TransportProtoError, e: + if e.code == httplib.NOT_FOUND: + raise apx.UnsupportedSearchError(e.url, + "Search/1") + elif e.code == httplib.NO_CONTENT: + raise apx.NegativeSearchResult(e.url) + elif e.code == httplib.BAD_REQUEST: + raise apx.MalformedSearchRequest(e.url) + elif e.retryable: + failures.append(e) + else: + raise + + except tx.TransportException, e: + if e.retryable: + failures.append(e) + fobj = None + else: + raise + + raise failures + + def get_ca_dir(self): + """Return the path to the directory that contains CA + certificates.""" + if self.__cadir is None: + cadir = os.path.join(os.path.sep, "usr", "share", + "pkg", "cacert") + if os.path.exists(cadir): + self.__cadir = cadir + return cadir + else: + self.__cadir = "" + + if self.__cadir == "": + return None + + return self.__cadir + + def get_catalog(self, pub, ts=None): + """Get the catalog for the specified publisher. If + ts is defined, request only changes newer than timestamp + ts.""" + + failures = tx.TransportFailures() + retry_count = global_settings.PKG_TIMEOUT_MAX + header = self.__build_header(uuid=self.__get_uuid(pub)) + croot = pub.meta_root + + for d in self.__gen_origins(pub, retry_count): + + repostats = self.stats[d.get_url()] + + # If a transport exception occurs, + # save it if it's retryable, otherwise + # raise the error to a higher-level handler. + try: + + resp = d.get_catalog(ts, header) + + updatelog.recv(resp, croot, ts, pub) + + return + + except tx.TransportProtoError, e: + if e.code == httplib.NOT_MODIFIED: + return + elif e.retryable: + failures.append(e) + else: + raise + except tx.TransportException, e: + if e.retryable: + failures.append(e) + else: + raise + except pkg.fmri.IllegalFmri, e: + repostats.record_error() + raise tx.TransportOperationError( + "Could not retrieve catalog from '%s'\n" + " Unable to parse FMRI. Details follow:\n%s" + % (pub.prefix, e)) + except EnvironmentError, e: + repostats.record_error() + raise tx.TransportOperationError( + "Could not retrieve catalog from '%s'\n" + " Exception: str:%s repr:%r" % (pub.prefix, + e, e)) + + raise failures + + def get_datastream(self, fmri, fhash): + """Given a fmri, and fhash, return a data stream for the remote + object. + + Since the caller handles the streaming object, instead + of having the transport manage it, the caller must catch + TransportError and perform any retry logic that is desired. + """ + + retry_count = global_settings.PKG_TIMEOUT_MAX + failures = tx.TransportFailures() + pub_prefix = fmri.get_publisher() + pub = self.__img.get_publisher(pub_prefix) + header = self.__build_header(uuid=self.__get_uuid(pub)) + + for d in self.__gen_repos(pub, retry_count): + + try: + resp = d.get_datastream(fhash, header) + return resp + + except tx.TransportException, e: + if e.retryable: + failures.append(e) + else: + raise + raise failures + + def touch_manifest(self, fmri, intent=None): + """Touch a manifest. This operation does not + return the manifest's content. The FMRI is given + as fmri. An optional intent string may be supplied + as intent.""" + + failures = tx.TransportFailures() + pub_prefix = fmri.get_publisher() + pub = self.__img.get_publisher(pub_prefix) + mfst = fmri.get_url_path() + retry_count = global_settings.PKG_TIMEOUT_MAX + header = self.__build_header(intent=intent, + uuid=self.__get_uuid(pub)) + + for d in self.__gen_origins(pub, retry_count): + + # If a transport exception occurs, + # save it if it's retryable, otherwise + # raise the error to a higher-level handler. + try: + d.touch_manifest(mfst, header) + return + + except tx.TransportException, e: + if e.retryable: + failures.append(e) + else: + raise + + raise failures + + def get_manifest(self, fmri, excludes=misc.EmptyI, intent=None): + """Given a fmri, and optional excludes, return a manifest + object.""" + + retry_count = global_settings.PKG_TIMEOUT_MAX + failures = tx.TransportFailures() + pub_prefix = fmri.get_publisher() + pub = self.__img.get_publisher(pub_prefix) + mfst = fmri.get_url_path() + mcontent = None + header = self.__build_header(intent=intent, + uuid=self.__get_uuid(pub)) + + for d in self.__gen_origins(pub, retry_count): + + repostats = self.stats[d.get_url()] + + try: + resp = d.get_manifest(mfst, header) + mcontent = resp.read() + m = manifest.CachedManifest(fmri, + self.__img.pkgdir, + self.__img.cfg_cache.preferred_publisher, + excludes, mcontent) + + return m + + except tx.TransportException, e: + if e.retryable: + failures.append(e) + mcontent = None + else: + raise + + except MalformedActionError, e: + repostats.record_error() + te = tx.TransferContentException( + d.get_url(), reason=str(e)) + failures.append(te) + + raise failures + + @staticmethod + def __build_header(intent=None, uuid=None): + """Return a dictionary that contains various + header fields, depending upon what arguments + were passed to the function. Supply intent header in intent + argument, uuid information in uuid argument.""" + + header = {} + + if intent: + header["X-IPkg-Intent"] = intent + + if uuid: + header["X-IPkg-UUID"] = uuid + + if len(header) == 0: + return None + + return header + + def __get_uuid(self, pub): + if not self.__img.cfg_cache.get_policy(imageconfig.SEND_UUID): + return None + + try: + return pub.client_uuid + except KeyError: + return None + + @staticmethod + def _makedirs(newdir): + """A helper function for _get_files that makes directories, + if needed.""" + + if not os.path.exists(newdir): + try: + os.makedirs(newdir) + except EnvironmentError, e: + if e.errno == errno.EACCES or \ + e.errno == errno.EROFS: + raise apx.PermissionsException( + e.filename) + else: + raise tx.TransportOperationError( + "Unable to make directory: %s" % e) + + def _get_files(self, mfile): + """Perform an operation that gets multiple files at once. + A mfile object contains information about the multiple-file + request that will be performed.""" + + retry_count = global_settings.PKG_TIMEOUT_MAX + failures = [] + filelist = mfile.keys() + pub = mfile.get_publisher() + progtrack = mfile.get_progtrack() + + # download_dir is temporary download path. Completed_dir + # is the cache where valid content lives. + completed_dir = self.__img.cached_download_dir() + download_dir = self.__img.incoming_download_dir() + + # Check if the download_dir exists. If it doesn't create + # the directories. + self._makedirs(download_dir) + + # Call statvfs to find the blocksize of download_dir's + # filesystem. + try: + destvfs = os.statvfs(download_dir) + except EnvironmentError, e: + if e.errno == errno.EACCES: + raise apx.PermissionsException(e.filename) + else: + raise tx.TransportOperationError( + "Unable to stat VFS: %s" % e) + + # set the file buffer size to the blocksize of our filesystem + self.__engine.set_file_bufsz(destvfs[statvfs.F_BSIZE]) + + for d in self.__gen_repos(pub, retry_count): + + failedreqs = [] + repostats = self.stats[d.get_url()] + + # This returns a list of transient errors + # that occurred during the transport operation. + # An exception handler here isn't necessary + # unless we want to supress a permanant failure. + errlist = d.get_files(filelist, download_dir, progtrack) + + for e in errlist: + req = getattr(e, "request", None) + if req: + failedreqs.append(req) + failures.append(e) + else: + raise e + + if len(failedreqs) > 0: + success = filter(lambda x: x not in failedreqs, + filelist) + filelist = failedreqs + else: + success = filelist + filelist = None + + for s in success: + + dl_path = os.path.join(download_dir, s) + + try: + self._verify_content(mfile[s][0], + dl_path) + except tx.InvalidContentException, e: + mfile.subtract_progress(e.size) + e.request = s + repostats.record_error() + failedreqs.append(s) + failures.append(e) + if not filelist: + filelist = failedreqs + continue + + final_path = os.path.normpath( + os.path.join(completed_dir, + misc.hash_file_name(s))) + finaldir = os.path.dirname(final_path) + + self._makedirs(finaldir) + portable.rename(dl_path, final_path) + + mfile.make_openers(s, final_path) + + # Return if everything was successful + if not filelist and len(errlist) == 0: + return + + if len(failedreqs) > 0 and len(failures) > 0: + failures = filter(lambda x: x.request in failedreqs, + failures) + tfailurex = tx.TransportFailures() + for f in failures: + tfailurex.append(f) + raise tfailurex + + def get_versions(self, pub): + """Query the publisher's origin servers for versions + information. Return a dictionary of "name":"versions" """ + + retry_count = global_settings.PKG_TIMEOUT_MAX + failures = tx.TransportFailures() + verlines = None + header = self.__build_header(uuid=self.__get_uuid(pub)) + + for d in self.__gen_origins(pub, retry_count): + + # If a transport exception occurs, + # save it if it's retryable, otherwise + # raise the error to a higher-level handler. + try: + resp = d.get_versions(header) + verlines = resp.readlines() + + return dict( + s.split(None, 1) + for s in (l.strip() for l in verlines) + ) + + except tx.TransportException, e: + if e.retryable: + failures.append(e) + verlines = None + else: + raise + except ValueError: + raise apx.InvalidDepotResponseException( + pub["origin"], + "Unable to parse server response") + + raise failures + + def __gen_origins(self, pub, count): + """The pub argument may either be a Publisher or a + RepositoryURI object.""" + + if not self.__engine: + self.__setup() + + if isinstance(pub, publisher.Publisher): + origins = pub.selected_repository.origins + else: + # If search was invoked with -s option, we'll have + # a RepoURI instead of a publisher. Convert + # this to a repo uri + origins = [pub] + + for i in xrange(count): + rslist = self.stats.get_repostats(origins) + for rs, ruri in rslist: + yield self.__repo_cache.new_repo(rs, ruri) + + def __gen_repos(self, pub, count): + + if not self.__engine: + self.__setup() + + for i in xrange(count): + repo = pub.selected_repository + rslist = self.stats.get_repostats(repo.mirrors) + for rs, ruri in rslist: + yield self.__repo_cache.new_repo(rs, ruri) + rslist = self.stats.get_repostats(repo.origins) + for rs, ruri in rslist: + yield self.__repo_cache.new_repo(rs, ruri) + + def valid_publisher_test(self, pub): + """Test that the publisher supplied in pub actually + points to a valid packaging server.""" + + try: + vd = self.get_versions(pub) + except tx.TransportException, e: + # Failure when contacting server. Report + # this as an error. + raise apx.InvalidDepotResponseException(pub["origin"], + "Transport errors encountered when trying to " + "contact depot server.\nReported the following " + "errors:\n%s" % e) + + if not self._valid_versions_test(vd): + raise apx.InvalidDepotResponseException(pub["origin"], + "Invalid or unparseable version information.") + + return True + + def captive_portal_test(self): + """A captive portal forces a HTTP client on a network + to see a special web page, usually for pubentication + purposes. (http://en.wikipedia.org/wiki/Captive_portal).""" + + vd = None + + self.__portal_test_executed = True + + for pub in self.__img.gen_publishers(): + try: + vd = self.get_versions(pub) + except tx.TransportException: + # Encountered a transport error while + # trying to contact this publisher. + # Pick another publisher instead. + continue + + if self._valid_versions_test(vd): + return + else: + continue + + if not vd: + # We got all the way through the list of publishers but + # encountered transport errors in every case. This is + # likely a network configuration problem. Report our + # inability to contact a server. + raise apx.InvalidDepotResponseException(None, + "Unable to contact any configured publishers. " + "This is likely a network configuration problem.") + + @staticmethod + def _valid_versions_test(versdict): + """Check that the versions information contained in + versdict contains valid version specifications. + + In order to test for this condition, pick a publisher + from the list of active publishers. Check to see if + we can connect to it. If so, test to see if it supports + the versions/0 operation. If versions/0 is not found, + we get an unparseable response, or the response does + not contain pkg-server, or versions 0 then we're not + talking to a depot. Return an error in these cases.""" + + if "pkg-server" in versdict: + # success! + return True + elif "versions" in versdict: + try: + versids = [ + int(v) + for v in versdict["versions"].split() + ] + except ValueError: + # Unable to determine version number. Fail. + return False + + if 0 not in versids: + # Paranoia. Version 0 should be in the + # output for versions/0. If we're here, + # something has gone very wrong. EPIC FAIL! + return False + + # Found versions/0, success! + return True + + # Some other error encountered. Fail. + return False + + def multi_file(self, fmri, progtrack, ccancel): + """Creates a MultiFile object for this transport. + The caller may add actions to the multifile object + and wait for the download to complete.""" + + if not fmri: + return None + + publisher = self.__img.get_publisher(fmri.get_publisher()) + mfile = MultiFile(publisher, self, progtrack, ccancel) + + return mfile + + def _action_cached(self, action): + """If a file with the name action.hash is cached, + and if it has the same content hash as action.chash, + then return the path to the file. If the file can't + be found, return None.""" + + hashval = action.hash + + cache_path = os.path.normpath(os.path.join( + self.__img.cached_download_dir(), + misc.hash_file_name(hashval))) + + try: + if os.path.exists(cache_path): + self._verify_content(action, cache_path) + return cache_path + except tx.InvalidContentException: + # If the content in the cache doesn't match the hash of + # the action, verify will have already purged the item + # from the cache. + pass + + return None + + @staticmethod + def _verify_content(action, filepath): + """If action contains an attribute that has the compressed + hash, read the file specified in filepath and verify + that the hash values match. If the values do not match, + remove the file and raise an InvalidContentException.""" + + chash = action.attrs.get("chash", None) + path = action.attrs.get("path", None) + if not chash: + # Compressed hash doesn't exist. Decompress and + # generate hash of uncompressed content. + ifile = open(filepath, "rb") + ofile = open(os.devnull, "wb") + + try: + fhash = misc.gunzip_from_stream(ifile, ofile) + except zlib.error, e: + os.remove(filepath) + raise tx.InvalidContentException(path, + "zlib.error:%s" % + (" ".join([str(a) for a in e.args]))) + + ifile.close() + ofile.close() + + if action.hash != fhash: + os.remove(filepath) + raise tx.InvalidContentException(action.path, + "hash failure: expected: %s" + "computed: %s" % (action.hash, fhash)) + return + + newhash = misc.get_data_digest(filepath)[0] + if chash != newhash: + s = os.stat(filepath) + os.remove(filepath) + raise tx.InvalidContentException(path, + "chash failure: expected: %s computed: %s" % \ + (chash, newhash), size=s.st_size) + + +class MultiFile(object): + """A transport object for performing multi-file requests + using pkg actions. This takes care of matching the publisher + with the actions, and performs the download and content + verification necessary to assure correct content installation.""" + + def __init__(self, pub, xport, progtrack, ccancel): + """Supply the destination publisher in the pub argument. + The transport object should be passed in xport.""" + + self._publisher = pub + self._transport = xport + self._progtrack = progtrack + # Add the check_cancelation to the progress tracker + self._progtrack.check_cancelation = ccancel + self._fhash = { } + + def __getitem__(self, key): + return self._fhash[key] + + def __contains__(self, key): + return key in self._fhash + + def add_action(self, action): + """The multiple file retrieval operation is asynchronous. + Add files to retrieve with this function. Supply the + publisher in pub and the list of files in filelist. + Wait for the operation by calling waitFiles.""" + + cachedpath = self._transport._action_cached(action) + if cachedpath: + action.data = self._make_opener(cachedpath) + filesz = misc.get_pkg_otw_size(action) + self._progtrack.download_add_progress(1, filesz) + return + + hashval = action.hash + + # Each fhash key accesses a list of one or more actions. If we + # already have a key in the dictionary, get the list and append + # the action to it. Otherwise, create a new list with the first + # action. + if hashval in self._fhash: + self._fhash[hashval].append(action) + else: + self._fhash[hashval] = [ action ] + + def del_hash(self, hashval): + """Remove the hashval from the dictionary, if it exists.""" + + if hashval in self._fhash: + del self._fhash[hashval] + + def get_publisher(self): + """Return the publisher object that will be used + for this MultiFile request.""" + + return self._publisher + + def get_progtrack(self): + """Return the progress tracker object for this MFile, + if it has one.""" + + return self._progtrack + + def keys(self): + """Return a list of the keys in the fhash.""" + + return self._fhash.keys() + + @staticmethod + def _make_opener(filepath): + def opener(): + f = open(filepath, "rb") + return f + return opener + + def make_openers(self, hashval, path): + """Find each action associated with the hash value hashval. + Create an opener that points to the file at path for the + action's data method.""" + + nactions = 0 + pkgsz = 0 + + for action in self._fhash[hashval]: + action.data = self._make_opener(path) + nactions += 1 + if pkgsz == 0: + pkgsz = misc.get_pkg_otw_size(action) + + # mark the actions with openers as done + # The progress tracker already added bytes for the first + # instance of this file, add the bytes for any other + # instances. + bytes = pkgsz * (nactions - 1) + self._progtrack.download_add_progress((nactions - 1), bytes) + + def subtract_progress(self, size): + """Subtract the progress accumulated by the download of + file with hash of hashval. make_openers accounts for + hashes with multiple actions. If this has been invoked, + it has happened before make_openers, so it's only necessary + to adjust the progress for a single file.""" + + self._progtrack.download_add_progress(-1, -size) + + def wait_files(self): + """Wait for outstanding file retrieval operations to + complete.""" + + if len(self._fhash) > 0: + self._transport._get_files(self) + --- old/src/modules/misc.py Wed Jul 1 14:56:39 2009 +++ new/src/modules/misc.py Wed Jul 1 14:56:38 2009 @@ -115,6 +115,17 @@ (VERSION, portable.util.get_canonical_os_name(), platform.machine(), portable.util.get_os_release(), platform.version()) +def user_agent_str(img, client_name): + + if not img or img.type is None: + imgtype = IMG_NONE + else: + imgtype = img.type + + useragent = _client_version % (img_type_names[imgtype], client_name) + + return useragent + def versioned_urlopen(base_uri, operation, versions = None, tail = None, data = None, headers = None, ssl_creds = None, imgtype = IMG_NONE, method = "GET", uuid = None): @@ -608,181 +619,6 @@ def __init__(self, args=None): self.args = args -class TransportException(Exception): - """ Abstract base class for various transport exceptions """ - def __init__(self): - self.count = 1 - -class TransportFailures(TransportException): - """ This exception encapsulates multiple transport exceptions """ - - # - # This class is a subclass of TransportException so that calling - # code can reasonably 'except TransportException' and get either - # a single-valued or in this case a multi-valued instance. - # - def __init__(self): - TransportException.__init__(self) - self.exceptions = [] - - def append(self, exc): - assert isinstance(exc, TransportException) - for x in self.exceptions: - if cmp(x, exc) == 0: - x.count += 1 - return - - self.exceptions.append(exc) - - def __str__(self): - if len(self.exceptions) == 0: - return "[no errors accumulated]" - - s = "" - for i, x in enumerate(self.exceptions): - if len(self.exceptions) > 1: - s += "%d: " % (i + 1) - s += str(x) - if x.count > 1: - s += " (happened %d times)" % x.count - s += "\n" - return s - - def __len__(self): - return len(self.exceptions) - -class TransferIOException(TransportException): - """Raised for retryable IO errors on underlying transport. - Protocol errors are TransferContentExceptions, timeouts - are TransferTimedOutExceptions.""" - def __init__(self, url, reason=None): - TransportException.__init__(self) - self.url = url - self.reason = reason - - def __str__(self): - s = "IO Error while communicating with '%s'" % self.url - if self.reason: - s += ": %s" % self.reason - s += "." - return s - - def __cmp__(self, other): - if not isinstance(other, TransferIOException): - return -1 - r = cmp(self.url, other.url) - if r != 0: - return r - return cmp(self.reason, other.reason) - -class TransferTimedOutException(TransportException): - """Raised when the transfer times out, or is terminated with a - retryable error.""" - def __init__(self, url, reason=None): - TransportException.__init__(self) - self.url = url - self.reason = reason - - def __str__(self): - s = "Transfer from '%s' timed out" % self.url - if self.reason: - s += ": %s" % self.reason - s += "." - return s - - def __cmp__(self, other): - if not isinstance(other, TransferTimedOutException): - return -1 - r = cmp(self.url, other.url) - if r != 0: - return r - return cmp(self.reason, other.reason) - - -# Retryable http errors. These are the HTTP errors that we'll catch. When we -# catch them, we throw a TransferTimedOutException instead re-raising the -# HTTPError and letting some other handler catch it. - -# XXX consider moving to pkg.client module -retryable_http_errors = set((httplib.REQUEST_TIMEOUT, httplib.BAD_GATEWAY, - httplib.GATEWAY_TIMEOUT)) -retryable_socket_errors = set((errno.ECONNABORTED, errno.ECONNRESET, - errno.ECONNREFUSED)) - - -class TransferContentException(TransportException): - """Raised when there are problems downloading the requested content.""" - def __init__(self, url, reason=None): - TransportException.__init__(self) - self.url = url - self.reason = reason - - def __str__(self): - s = "Transfer from '%s' failed" % self.url - if self.reason: - s += ": %s" % self.reason - s += "." - return s - - def __cmp__(self, other): - if not isinstance(other, TransferContentException): - return -1 - r = cmp(self.url, other.url) - if r != 0: - return r - return cmp(self.reason, other.reason) - -class TruncatedTransferException(TransportException): - """Raised when the transfer that was received doesn't match the - expected length.""" - def __init__(self, url, recd=-1, expected=-1): - TransportException.__init__(self) - self.url = url - self.recd = recd - self.expected = expected - - def __str__(self): - s = "Transfer from '%s' unexpectedly terminated" % self.url - if self.recd > -1 and self.expected > -1: - s += ": received %d of %d bytes" % (self.recd, - self.expected) - s += "." - return s - - def __cmp__(self, other): - if not isinstance(other, TruncatedTransferException): - return -1 - r = cmp(self.url, other.url) - if r != 0: - return r - r = cmp(self.expected, other.expected) - if r != 0: - return r - return cmp(self.recd, other.recd) - - -class InvalidContentException(TransportException): - """Raised when the content's hash/chash doesn't verify, or the - content is received in an unreadable format.""" - def __init__(self, path, data): - TransportException.__init__(self) - self.path = path - self.data = data - - def __str__(self): - s = "Invalid content for action with path %s" % self.path - if self.data: - s += " %s." % self.data - return s - - def __cmp__(self, other): - if not isinstance(other, InvalidContentException): - return -1 - r = cmp(self.path, other.path) - if r != 0: - return r - return cmp(self.data, other.data) - # ImmutableDict and EmptyI for argument defaults EmptyI = tuple() --- old/src/modules/server/catalog.py Wed Jul 1 14:56:39 2009 +++ new/src/modules/server/catalog.py Wed Jul 1 14:56:39 2009 @@ -30,6 +30,7 @@ import errno import os +import random import shutil import signal import sys @@ -369,3 +370,45 @@ ServerCatalog.cache_fmri(cat, f, pub) catf.close() + +class NastyServerCatalog(ServerCatalog): + """The catalog for the nasty server.""" + + def as_lines(self, scfg=None): + """Returns a generator function that produces the contents of + the catalog as a list of strings.""" + + be_nasty = False + + # NASTY + # First roll the dice to decide whether we should be nasty. + # Later roll again to decide when to be nasty. + if scfg and scfg.need_nasty_occasionally(): + be_nasty = True + + try: + cfile = file(self.catalog_file, "r") + except EnvironmentError, e: + # Missing catalog is fine; other errors need to + # be reported. + if e.errno == errno.ENOENT: + return + raise + + for e in cfile: + # NASTY + # There's only one opportunity to truncate + # the request, but if we don't truncate the + # request we can try to truncate a line too. + if be_nasty and scfg.need_nasty_occasionally(): + return + elif be_nasty and \ + scfg.need_nasty_infrequently(): + linelen = random.randint(1, len(e)) + badline = e[0:linelen] + yield badline + else: + yield e + + cfile.close() + --- old/src/modules/server/config.py Wed Jul 1 14:56:40 2009 +++ new/src/modules/server/config.py Wed Jul 1 14:56:40 2009 @@ -28,6 +28,7 @@ import errno import os import os.path +import random import shutil import pkg.server.catalog as catalog @@ -236,3 +237,85 @@ def search_available(self): return self.catalog.search_available() + +class NastySvrConfig(SvrConfig): + """A subclass of SvrConfig that helps implement options + for the Nasty server, which misbehaves in order to test + the client's failure resistance.""" + + def __init__(self, repo_root, content_root, publisher, + auto_create=False, fork_allowed=False, writable_root=None): + + # Call parent's constructor + SvrConfig.__init__(self, repo_root, content_root, publisher, + auto_create, fork_allowed, writable_root) + + self.nasty = 0 + + def acquire_catalog(self, rebuild=False, verbose=False): + """Tell the catalog to set itself up. Associate an + instance of the catalog with this depot.""" + + if self.is_mirror(): + return + + if rebuild: + self.destroy_catalog() + + self.catalog = catalog.NastyServerCatalog(self.cat_root, + pkg_root=self.pkg_root, read_only=self.read_only, + index_root=self.index_root, repo_root=self.repo_root, + rebuild=rebuild, verbose=verbose, + fork_allowed=self.fork_allowed, + has_writable_root=self.has_writable_root) + + # UpdateLog allows server to issue incremental catalog updates + self.updatelog = updatelog.NastyUpdateLog(self.update_root, + self.catalog) + + def set_nasty(self, level): + """Set the nasty level using an integer.""" + + self.nasty = level + + def is_nasty(self): + """Returns true if nasty has been enabled.""" + + if self.nasty > 0: + return True + return False + + def need_nasty(self): + """Randomly returns true when the server should misbehave.""" + + if random.randint(1, 100) <= self.nasty: + return True + return False + + def need_nasty_bonus(self, bonus=0): + """Used to temporarily apply extra nastiness to an operation.""" + + if self.nasty + bonus > 95: + nasty = 95 + else: + nasty = self.nasty + bonus + + if random.randint(1, 100) <= nasty: + return True + return False + + def need_nasty_occasionally(self): + if random.randint(1, 500) <= self.nasty: + return True + return False + + def need_nasty_infrequently(self): + if random.randint(1, 2000) <= self.nasty: + return True + return False + + def need_nasty_rarely(self): + if random.randint(1, 20000) <= self.nasty: + return True + return False + --- old/src/modules/server/depot.py Wed Jul 1 14:56:41 2009 +++ new/src/modules/server/depot.py Wed Jul 1 14:56:40 2009 @@ -34,9 +34,11 @@ import inspect import itertools import os +import random import re import socket import tarfile +import time # Without the below statements, tarfile will trigger calls to getpwuid and # getgrgid for every file downloaded. This in turn leads to nscd usage which # limits the throughput of the depot process. Setting these attributes to @@ -939,3 +941,371 @@ raise cherrypy.HTTPError(httplib.NOT_FOUND, str(e)) buf.seek(0) return buf.getvalue() + + +class NastyDepotHTTP(DepotHTTP): + """A class that creates a depot that misbehaves. Naughty + depots are useful for testing.""" + + + def __init__(self, scfg, cfgpathname=None): + """Include config in scfg, and cfgpathname, if needed.""" + + DepotHTTP.__init__(self, scfg, cfgpathname) + + self.__repo = repo.NastyRepository(scfg, cfgpathname) + self.rcfg = self.__repo.rcfg + self.scfg = self.__repo.scfg + + # Handles the BUI (Browser User Interface). + face.init(scfg, self.rcfg) + + # Store any possible configuration changes. + self.__repo.write_config() + + self.requested_files = [] + + cherrypy.tools.nasty_httperror = cherrypy.Tool('before_handler', + NastyDepotHTTP.nasty_retryable_error) + + # Method for CherryPy tool for Nasty Depot + @staticmethod + def nasty_retryable_error(bonus=0): + """A static method that's used by the cherrpy tools, + and in depot code, to generate a retryable HTTP error.""" + + retryable_errors = [httplib.REQUEST_TIMEOUT, + httplib.BAD_GATEWAY, httplib.GATEWAY_TIMEOUT] + + # NASTY + # emit error code that client should know how to retry + if cherrypy.request.app.root.scfg.need_nasty_bonus(bonus): + code = retryable_errors[random.randint(0, + len(retryable_errors) - 1)] + raise cherrypy.HTTPError(code) + + # Override _cp_config for catalog_0 operation + def catalog_0(self, *tokens): + """Provide an incremental update or full version of the + catalog, as appropriate, to the requesting client.""" + + request = cherrypy.request + + response = cherrypy.response + response.headers["Content-type"] = "text/plain" + response.headers["Last-Modified"] = \ + self.scfg.catalog.last_modified() + + lm = request.headers.get("If-Modified-Since", None) + if lm is not None: + try: + lm = catalog.ts_to_datetime(lm) + except ValueError: + lm = None + else: + if not self.scfg.updatelog.enough_history(lm): + # Ignore incremental requests if there + # isn't enough history to provide one. + lm = None + elif self.scfg.updatelog.up_to_date(lm): + response.status = httplib.NOT_MODIFIED + return + + if lm: + # If a last modified date and time was provided, then an + # incremental update is being requested. + response.headers["X-Catalog-Type"] = "incremental" + else: + response.headers["X-Catalog-Type"] = "full" + response.headers["Content-Length"] = str( + self.scfg.catalog.size()) + + def output(): + try: + for l in self.__repo.catalog(lm): + yield l + except repo.RepositoryError, e: + # Can't do anything in a streaming generator + # except log the error and return. + cherrypy.log("Request failed: %s" % str(e)) + return + + return output() + + catalog_0._cp_config = { "response.stream": True, + "tools.nasty_httperror.on": True, + "tools.nasty_httperror.bonus": 1 } + + def manifest_0(self, *tokens): + """The request is an encoded pkg FMRI. If the version is + specified incompletely, we return an error, as the client is + expected to form correct requests based on its interpretation + of the catalog and its image policies.""" + + # Parse request into FMRI component and decode. + try: + # If more than one token (request path component) was + # specified, assume that the extra components are part + # of the fmri and have been split out because of bad + # proxy behaviour. + pfmri = "/".join(tokens) + fpath = self.__repo.manifest(pfmri) + except (IndexError, repo.RepositoryInvalidFMRIError), e: + raise cherrypy.HTTPError(httplib.BAD_REQUEST, str(e)) + except repo.RepositoryError, e: + # Treat any remaining repository error as a 404, but + # log the error and include the real failure + # information. + cherrypy.log("Request failed: %s" % str(e)) + raise cherrypy.HTTPError(httplib.NOT_FOUND, str(e)) + + # NASTY + # Send an error before serving the file, perhaps + if self.scfg.need_nasty(): + self.nasty_retryable_error() + elif self.scfg.need_nasty_infrequently(): + # Fall asleep before finishing the request + time.sleep(35) + elif self.scfg.need_nasty_rarely(): + # Forget that the manifest is here + raise cherrypy.HTTPError(httplib.NOT_FOUND) + + # NASTY + # Call a misbehaving serve_file + return self.nasty_serve_file(fpath, "text/plain") + + manifest_0._cp_config = { "response.stream": True } + + def filelist_0(self, *tokens, **params): + """Request data contains application/x-www-form-urlencoded + entries with the requested filenames. The resulting tar stream + is output directly to the client. """ + + try: + self.scfg.inc_flist() + + # NASTY + if self.scfg.need_nasty_occasionally(): + return + + # Create a dummy file object that hooks to the write() + # callable which is all tarfile needs to output the + # stream. This will write the bytes to the client + # through our parent server process. + f = Dummy() + f.write = cherrypy.response.write + + tar_stream = tarfile.open(mode = "w|", + fileobj = f) + + # We can use the request object for storage of data + # specific to this request. In this case, it allows us + # to provide our on_end_request function with access to + # the stream we are processing. + cherrypy.request.tar_stream = tar_stream + + # This is a special hook just for this request so that + # if an exception is encountered, the stream will be + # closed properly regardless of which thread is + # executing. + cherrypy.request.hooks.attach("on_end_request", + self._tar_stream_close, failsafe = True) + + # NASTY + if self.scfg.need_nasty_infrequently(): + time.sleep(35) + + for v in params.values(): + + # NASTY + # Stash filename for later use. + # Toss out the list if it's larger than 1024 + # items. + if len(self.requested_files) > 1024: + self.requested_files = [v] + else: + self.requested_files.append(v) + + # NASTY + if self.scfg.need_nasty_infrequently(): + # Give up early + break + elif self.scfg.need_nasty_infrequently(): + # Skip this file + continue + elif self.scfg.need_nasty_rarely(): + # Take a nap + time.sleep(35) + + filepath = os.path.normpath(os.path.join( + self.scfg.file_root, + misc.hash_file_name(v))) + + # If file isn't here, skip it + if not os.path.exists(filepath): + continue + + # NASTY + # Send a file with the wrong content + if self.scfg.need_nasty_rarely(): + pick = random.randint(0, + len(self.requested_files) - 1) + badfn = self.requested_files[pick] + badpath = os.path.normpath( + os.path.join(self.scfg.file_root, + misc.hash_file_name(badfn))) + + tar_stream.add(badpath, v, False) + else: + tar_stream.add(filepath, v, False) + + self.scfg.inc_flist_files() + + # NASTY + # Write garbage into the stream + if self.scfg.need_nasty_infrequently(): + f.write("NASTY!") + + # NASTY + # Send an extraneous file + if self.scfg.need_nasty_infrequently(): + pick = random.randint(0, + len(self.requested_files) - 1) + extrafn = self.requested_files[pick] + extrapath = os.path.normpath( + os.path.join(self.scfg.file_root, + misc.hash_file_name(extrafn))) + + tar_stream.add(extrapath, extrafn, False) + + + # Flush the remaining bytes to the client. + tar_stream.close() + cherrypy.request.tar_stream = None + + except Exception, e: + # If we find an exception of this type, the + # client has most likely been interrupted. + if isinstance(e, socket.error) \ + and e.args[0] == errno.EPIPE: + return + raise + + yield "" + + # We have to configure the headers either through the _cp_config + # namespace, or inside the function itself whenever we are using + # a streaming generator. This is because headers have to be setup + # before the response even begins and the point at which @tools + # hooks in is too late. + filelist_0._cp_config = { + "response.stream": True, + "tools.nasty_httperror.on": True, + "tools.response_headers.on": True, + "tools.response_headers.headers": [("Content-Type", + "application/data")] + } + + def file_0(self, *tokens): + """Outputs the contents of the file, named by the SHA-1 hash + name in the request path, directly to the client.""" + + try: + fhash = tokens[0] + except IndexError: + fhash = None + + try: + fpath = self.__repo.file(fhash) + except repo.RepositoryFileNotFoundError, e: + raise cherrypy.HTTPError(httplib.NOT_FOUND, str(e)) + except repo.RepositoryError, e: + # Treat any remaining repository error as a 404, but + # log the error and include the real failure + # information. + cherrypy.log("Request failed: %s" % str(e)) + raise cherrypy.HTTPError(httplib.NOT_FOUND, str(e)) + + # NASTY + # Stash filename for later use. + # Toss out the list if it's larger than 1024 + # items. + if len(self.requested_files) > 1024: + self.requested_files = [fhash] + else: + self.requested_files.append(fhash) + + # NASTY + # Send an error before serving the file, perhaps + if self.scfg.need_nasty(): + self.nasty_retryable_error() + elif self.scfg.need_nasty_rarely(): + # Fall asleep before finishing the request + time.sleep(35) + elif self.scfg.need_nasty_rarely(): + # Forget that the manifest is here + raise cherrypy.HTTPError(httplib.NOT_FOUND) + + # NASTY + # Send the wrong file + if self.scfg.need_nasty_rarely(): + pick = random.randint(0, len(self.requested_files) - 1) + badfn = self.requested_files[pick] + badpath = os.path.normpath(os.path.join( + self.scfg.file_root, misc.hash_file_name(badfn))) + + return serve_file(badpath, "application/data") + + # NASTY + # Call a misbehaving serve_file + return self.nasty_serve_file(fpath, "application/data") + + file_0._cp_config = { "response.stream": True } + + def nasty_serve_file(self, filepath, content_type): + """A method that imitates the functionality of serve_file(), + but behaves in a nasty manner.""" + + already_nasty = False + + response = cherrypy.response + response.headers["Content-Type"] = content_type + try: + fst = os.stat(filepath) + filesz = fst.st_size + file = open(filepath, "rb") + except EnvironmentError: + raise cherrypy.HTTPError(httplib.NOT_FOUND) + + # NASTY + # Send incorrect content length + if self.scfg.need_nasty_rarely(): + response.headers["Content-Length"] = str(filesz + + random.randint(1, 1024)) + already_nasty = True + else: + response.headers["Content-Length"] = str(filesz) + + # NASTY + # Send truncated file + if self.scfg.need_nasty_rarely() and not already_nasty: + response.body = file.read(filesz - random.randint(1, + filesz - 1)) + # If we're sending data, lie about the length and + # make the client catch us. + if content_type == "application/data": + response.headers["Content-Length"] = str( + len(response.body)) + elif self.scfg.need_nasty_rarely() and not already_nasty: + # Write garbage into the response + response.body = file.read(filesz) + response.body += "NASTY!" + # If we're sending data, lie about the length and + # make the client catch us. + if content_type == "application/data": + response.headers["Content-Length"] = str( + len(response.body)) + else: + response.body = file.read(filesz) + + return response.body --- old/src/modules/server/repository.py Wed Jul 1 14:56:41 2009 +++ new/src/modules/server/repository.py Wed Jul 1 14:56:41 2009 @@ -335,3 +335,50 @@ for q in query_lst ] return res_list + +class NastyRepository(Repository): + """A repository object that helps the Nasty server misbehave. + At the present time, this only overrides the catalog method, + so that the catalog may pass a scfg object to the Catalog and + UpdateLog.""" + + def __init__(self, scfg, cfgpathname=None): + """Prepare the repository for use.""" + + Repository.__init__(self, scfg, cfgpathname) + + def catalog(self, last_modified=None): + """Returns a generator object containing an incremental update + if 'last_modified' is provided. If 'last_modified' is not + provided, a generator object for the full version of the catalog + will be returned instead. 'last_modified' should be a datetime + object or an ISO8601 formatted string.""" + + self.scfg.inc_catalog() + + if isinstance(last_modified, basestring): + last_modified = catalog.ts_to_datetime(last_modified) + + # Incremental catalog updates + c = self.scfg.catalog + ul = self.scfg.updatelog + if last_modified: + if not ul.up_to_date(last_modified) and \ + ul.enough_history(last_modified): + for line in ul._gen_updates(last_modified, + self.scfg): + yield line + else: + raise RepositoryCatalogNoUpdatesError( + "incremental", c.last_modified()) + return + + # Full catalog request. + # Return attributes first. + for line in c.attrs_as_lines(): + yield line + + # Return the contents last. + for line in c.as_lines(self.scfg): + yield line + --- old/src/modules/updatelog.py Wed Jul 1 14:56:42 2009 +++ new/src/modules/updatelog.py Wed Jul 1 14:56:42 2009 @@ -41,7 +41,6 @@ import pkg.client.api_errors as api_errors import pkg.fmri as fmri import pkg.portable as portable -from pkg.misc import TruncatedTransferException class UpdateLogException(Exception): def __init__(self, args=None): @@ -284,15 +283,13 @@ Ts is the timestamp when the local copy of the catalog was last modified.""" - update_type = c.info().getheader("X-Catalog-Type", "full") + update_type = c.getheader("X-Catalog-Type", "full") - cl_size = int(c.info().getheader("Content-Length", "-1")) - try: if update_type == "incremental": - UpdateLog._recv_updates(c, path, ts, cl_size) + UpdateLog._recv_updates(c, path, ts) else: - catalog.recv(c, path, pub, cl_size) + catalog.recv(c, path, pub) except EnvironmentError, e: if isinstance(e, EnvironmentError): if e.errno == errno.EACCES: @@ -301,7 +298,7 @@ raise @staticmethod - def _recv_updates(filep, path, cts, content_size=-1): + def _recv_updates(filep, path, cts): """A static method that takes a file-like object, a path, and a timestamp. This is the other half of send_updates(). It reads a stream as an incoming updatelog and @@ -322,12 +319,9 @@ unknown_lines = [] bad_fmri = None attrs = {} - size = 0 for s in filep: - size += len(s) - l = s.split(None, 3) if len(l) < 4: continue @@ -382,18 +376,9 @@ (l[2], sf, sv, rf, rv) add_lines.append(line) - # Check that content was properly received before - # modifying any files. - if content_size > -1 and size != content_size: - url = None - if hasattr(filep, "geturl") and callable(filep.geturl): - url = filep.geturl() - raise TruncatedTransferException(url, size, - content_size) - # If we got a parse error on FMRIs and transfer # wasn't truncated, raise a retryable transport - elif bad_fmri: + if bad_fmri: raise bad_fmri # Verify that they aren't already in the catalog @@ -644,4 +629,66 @@ # Allow these methods to be invoked without explictly naming the UpdateLog # class. recv = UpdateLog.recv + +class NastyUpdateLog(UpdateLog): + + def __init__(self, update_root, cat, maxfiles = 336): + + UpdateLog.__init__(self, update_root, cat, maxfiles) + + def _gen_updates(self, ts, scfg=None): + """Look through the logs for updates that have occurred after + the timestamp. Yield each of those updates in line-update + format.""" + + # The files that need to be examined depend upon the timestamp + # supplied by the client, and the log files actually present. + # + # The following cases exist: + # + # 1. No updates have occurred since timestamp. Send nothing. + # + # 2. Timestamp is older than oldest log record. Client needs + # to download full catalog. + # + # 3. Timestamp falls within a range for which update records + # exist. If the timestamp is in the middle of a log-file, send + # the entire file -- the client will pick what entries to add. + # Then send all newer files. + + be_nasty = False + + if scfg and scfg.need_nasty_occasionally(): + be_nasty = True + + rts = datetime.datetime(ts.year, ts.month, ts.day, ts.hour) + assert rts <= ts + + # send data from logfiles newer or equal to rts + for lf in self.logfiles: + + lf_time = datetime.datetime( + *time.strptime(lf, "%Y%m%d%H")[0:6]) + + if lf_time >= rts: + fn = "%s" % lf + logf = file(os.path.join(self.rootdir, fn), + "r") + for line in logf: + # NASTY + # There's only one opportunity to + # truncate the request, but if we don't + # truncate the request we can try to + # truncate a line too. + if be_nasty and \ + scfg.need_nasty_occasionally(): + return + elif be_nasty and \ + scfg.need_nasty_infrequently(): + linelen = random.randint(1, + len(line)) + yield line[0:linelen] + else: + yield line + logf.close() --- old/src/packagemanager.py Wed Jul 1 14:56:43 2009 +++ new/src/packagemanager.py Wed Jul 1 14:56:43 2009 @@ -122,7 +122,6 @@ import pkg.client.api_errors as api_errors import pkg.client.api as api import pkg.client.publisher as publisher -import pkg.client.retrieve as retrieve import pkg.portable as portable import pkg.fmri as fmri import pkg.gui.repository as repository @@ -194,14 +193,20 @@ self.current_search_option = 0 self.in_search_mode = False - socket.setdefaulttimeout( - int(os.environ.get("PKG_CLIENT_TIMEOUT", "30"))) # in seconds + # Override default PKG_TIMEOUT_MAX and PKG_CLIENT_TIMEOUT + # if a value has been specified in the environment. + global_settings.PKG_TIMEOUT_MAX = int(os.environ.get( + "PKG_TIMEOUT_MAX", global_settings.PKG_TIMEOUT_MAX)) - # Override default PKG_TIMEOUT_MAX if a value has been specified - # in the environment. - global_settings.PKG_TIMEOUT_MAX = int(os.environ.get("PKG_TIMEOUT_MAX", - global_settings.PKG_TIMEOUT_MAX)) + global_settings.PKG_CLIENT_TIMEOUT = int(os.environ.get( + "PKG_CLIENT_TIMEOUT", global_settings.PKG_CLIENT_TIMEOUT)) + # This call only affects sockets created by Python. The + # transport framework must set the timeout value internally + # + # value is in seconds + socket.setdefaulttimeout(global_settings.PKG_TIMEOUT_MAX) + global_settings.client_name = PKG_CLIENT_NAME try: @@ -1397,8 +1402,7 @@ info = self.api_o.info(pkg_stems_and_itr_to_fetch.keys(), False, frozenset([api.PackageInfo.IDENTITY, api.PackageInfo.SUMMARY])) - except (misc.TransportFailures, retrieve.ManifestRetrievalError, - retrieve.DatastreamRetrievalError): + except api_errors.TransportError: self.update_statusbar() return if info and len(info.get(0)) == 0: @@ -2987,8 +2991,7 @@ try: info = self.api_o.info([selected_pkgstem], True, frozenset([api.PackageInfo.LICENSES])) - except (misc.TransportFailures, retrieve.ManifestRetrievalError, - retrieve.DatastreamRetrievalError): + except (api_errors.TransportError): pass if self.showing_empty_details or (license_id != self.last_show_licenses_id): @@ -2998,8 +3001,7 @@ # Get license from remote info = self.api_o.info([selected_pkgstem], False, frozenset([api.PackageInfo.LICENSES])) - except (misc.TransportFailures, retrieve.ManifestRetrievalError, - retrieve.DatastreamRetrievalError): + except (api_errors.TransportError): pass if self.showing_empty_details or (license_id != self.last_show_licenses_id): @@ -3027,8 +3029,7 @@ info = self.api_o.info([pkg_stem], local, api.PackageInfo.ALL_OPTIONS - frozenset([api.PackageInfo.LICENSES])) - except (misc.TransportFailures, retrieve.ManifestRetrievalError, - retrieve.DatastreamRetrievalError): + except (api_errors.TransportError): return info pkgs_info = None package_info = None --- /dev/null Wed Jul 1 14:56:44 2009 +++ new/src/patch/pycurl/pyc-719-setup.patch Wed Jul 1 14:56:44 2009 @@ -0,0 +1,16 @@ +--- setup.py.old Mon Nov 24 18:03:47 2008 ++++ setup.py Mon Nov 24 18:04:06 2008 +@@ -95,12 +95,11 @@ + if not re.search(r"^\/+usr\/+include\/*$", e[2:]): + include_dirs.append(e[2:]) + else: + extra_compile_args.append(e) + libs = split_quoted( +- os.popen("'%s' --libs" % CURL_CONFIG).read()+\ +- os.popen("'%s' --static-libs" % CURL_CONFIG).read()) ++ os.popen("'%s' --libs" % CURL_CONFIG).read()) + for e in libs: + if e[:2] == "-l": + libraries.append(e[2:]) + if e[2:] == 'ssl': + define_macros.append(('HAVE_CURL_OPENSSL', 1)) --- /dev/null Wed Jul 1 14:56:44 2009 +++ new/src/patch/pycurl/pycurl-reset.patch Wed Jul 1 14:56:44 2009 @@ -0,0 +1,153 @@ +--- src/pycurl.c.old Tue Sep 9 10:40:34 2008 ++++ src/pycurl.c Tue Apr 28 20:22:53 2009 +@@ -745,69 +745,86 @@ + memset(self->error, 0, sizeof(self->error)); + + return self; + } + +- +-/* constructor - this is a module-level function returning a new instance */ +-static CurlObject * +-do_curl_new(PyObject *dummy) ++/* initializer - used to intialize curl easy handles for use with pycurl */ ++static int ++util_curl_init(CurlObject *self) + { +- CurlObject *self = NULL; + int res; + char *s = NULL; + +- UNUSED(dummy); +- +- /* Allocate python curl object */ +- self = util_curl_new(); +- if (self == NULL) +- return NULL; +- +- /* Initialize curl handle */ +- self->handle = curl_easy_init(); +- if (self->handle == NULL) +- goto error; +- + /* Set curl error buffer and zero it */ + res = curl_easy_setopt(self->handle, CURLOPT_ERRORBUFFER, self->error); +- if (res != CURLE_OK) +- goto error; ++ if (res != CURLE_OK) { ++ return (-1); ++ } + memset(self->error, 0, sizeof(self->error)); + + /* Set backreference */ + res = curl_easy_setopt(self->handle, CURLOPT_PRIVATE, (char *) self); +- if (res != CURLE_OK) +- goto error; ++ if (res != CURLE_OK) { ++ return (-1); ++ } + + /* Enable NOPROGRESS by default, i.e. no progress output */ + res = curl_easy_setopt(self->handle, CURLOPT_NOPROGRESS, (long)1); +- if (res != CURLE_OK) +- goto error; ++ if (res != CURLE_OK) { ++ return (-1); ++ } + + /* Disable VERBOSE by default, i.e. no verbose output */ + res = curl_easy_setopt(self->handle, CURLOPT_VERBOSE, (long)0); +- if (res != CURLE_OK) +- goto error; ++ if (res != CURLE_OK) { ++ return (-1); ++ } + + /* Set FTP_ACCOUNT to NULL by default */ + res = curl_easy_setopt(self->handle, CURLOPT_FTP_ACCOUNT, NULL); +- if (res != CURLE_OK) +- goto error; ++ if (res != CURLE_OK) { ++ return (-1); ++ } + + /* Set default USERAGENT */ + s = (char *) malloc(7 + strlen(LIBCURL_VERSION) + 1); +- if (s == NULL) +- goto error; ++ if (s == NULL) { ++ return (-1); ++ } + strcpy(s, "PycURL/"); strcpy(s+7, LIBCURL_VERSION); + res = curl_easy_setopt(self->handle, CURLOPT_USERAGENT, (char *) s); + if (res != CURLE_OK) { + free(s); +- goto error; ++ return (-1); + } + self->options[ OPT_INDEX(CURLOPT_USERAGENT) ] = s; s = NULL; + ++ return (0); ++} ++ ++/* constructor - this is a module-level function returning a new instance */ ++static CurlObject * ++do_curl_new(PyObject *dummy) ++{ ++ CurlObject *self = NULL; ++ int res; ++ ++ UNUSED(dummy); ++ ++ /* Allocate python curl object */ ++ self = util_curl_new(); ++ if (self == NULL) ++ return NULL; ++ ++ /* Initialize curl handle */ ++ self->handle = curl_easy_init(); ++ if (self->handle == NULL) ++ goto error; ++ ++ res = util_curl_init(self); ++ if (res < 0) ++ goto error; + /* Success - return new object */ + return self; + + error: + Py_DECREF(self); /* this also closes self->handle */ +@@ -1423,10 +1440,11 @@ + + static PyObject* + do_curl_reset(CurlObject *self) + { + unsigned int i; ++ int res; + + curl_easy_reset(self->handle); + + /* Decref callbacks and file handles */ + util_curl_xdecref(self, 4 | 8, self->handle); +@@ -1450,11 +1468,21 @@ + free(self->options[i]); + self->options[i] = NULL; + } + } + ++ res = util_curl_init(self); ++ if (res < 0) ++ goto error; ++ ++ Py_INCREF(Py_None); + return Py_None; ++ ++error: ++ Py_DECREF(self); /* this also closes self->handle */ ++ PyErr_SetString(ErrorObject, "resetting curl failed"); ++ return NULL; + } + + /* --------------- unsetopt/setopt/getinfo --------------- */ + + static PyObject * --- old/src/pkgdefs/Makefile Wed Jul 1 14:56:45 2009 +++ new/src/pkgdefs/Makefile Wed Jul 1 14:56:45 2009 @@ -18,7 +18,7 @@ # # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # @@ -34,7 +34,7 @@ SUBDIRS= SUNWipkg SUNWipkg-brand SUNWipkg-gui SUNWipkg-gui-data \ SUNWipkg-gui-l10n SUNWipkg-um SUNWpython-cherrypy SUNWpython-mako \ - SUNWpython-ply $(BUILDPYOPENSSL) + SUNWpython-ply SUNWpython-pycurl $(BUILDPYOPENSSL) all: --- old/src/pkgdefs/SUNWipkg/prototype Wed Jul 1 14:56:45 2009 +++ new/src/pkgdefs/SUNWipkg/prototype Wed Jul 1 14:56:45 2009 @@ -1,5 +1,5 @@ -i pkginfo i copyright +i pkginfo d none lib 755 root bin d none lib/svc 755 root bin d none lib/svc/method 755 root bin @@ -81,8 +81,6 @@ f none usr/lib/python2.4/vendor-packages/pkg/client/constraint.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/debugvalues.py 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/debugvalues.pyc 444 root bin -f none usr/lib/python2.4/vendor-packages/pkg/client/filelist.py 444 root bin -f none usr/lib/python2.4/vendor-packages/pkg/client/filelist.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/filter.py 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/filter.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/history.py 444 root bin @@ -107,8 +105,21 @@ f none usr/lib/python2.4/vendor-packages/pkg/client/publisher.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/query_parser.py 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/query_parser.pyc 444 root bin -f none usr/lib/python2.4/vendor-packages/pkg/client/retrieve.py 444 root bin -f none usr/lib/python2.4/vendor-packages/pkg/client/retrieve.pyc 444 root bin +d none usr/lib/python2.4/vendor-packages/pkg/client/transport 755 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/__init__.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/__init__.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/engine.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/engine.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/exception.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/exception.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/fileobj.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/fileobj.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/repo.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/repo.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/stats.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/stats.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/transport.py 444 root bin +f none usr/lib/python2.4/vendor-packages/pkg/client/transport/transport.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/variant.py 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/client/variant.pyc 444 root bin f none usr/lib/python2.4/vendor-packages/pkg/cpiofile.py 444 root bin @@ -260,6 +271,10 @@ f none usr/share/man/cat1m/pkg.depotd.1m 444 root bin d none usr/share/man/cat5 755 root bin f none usr/share/man/cat5/pkg.5 444 root bin +d none usr/share/pkg 755 root bin +d none usr/share/pkg/cacert 755 root bin +s none usr/share/pkg/cacert/72fa7371.0=Verisign_Class_3_Public_Primary_Certification_Authority-G2.pem +f none usr/share/pkg/cacert/Verisign_Class_3_Public_Primary_Certification_Authority-G2.pem 444 root bin d none var 755 root sys d none var/svc 755 root sys d none var/svc/manifest 755 root sys --- /dev/null Wed Jul 1 14:56:46 2009 +++ new/src/pkgdefs/SUNWpython-pycurl/Makefile Wed Jul 1 14:56:46 2009 @@ -0,0 +1,44 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +MACH:sh = uname -p + +ROOT = ../../../proto/root_$(MACH) +PKG = SUNWpython-pycurl +PKGARCHIVE = ../../../packages/$(MACH) + +install: sysv-pkg + +sysv-pkg: $(PKGARCHIVE) + [ -f $(PKGARCHIVE)/$(PKG)/pkgmap ] || pkgmk -a $(MACH) -o -r $(ROOT) -d $(PKGARCHIVE) + +clean: FRC + +clobber: clean + rm -rf $(PKGARCHIVE)/$(PKG) + +$(PKGARCHIVE): + [ -d $(PKGARCHIVE) ] || mkdir -p $(PKGARCHIVE) + +FRC: --- /dev/null Wed Jul 1 14:56:46 2009 +++ new/src/pkgdefs/SUNWpython-pycurl/copyright Wed Jul 1 14:56:46 2009 @@ -0,0 +1,33 @@ +Sun elects to have this file available under and governed by the Curl MIT +derivative license (see below for full license text). However the following +notice accompanied the original version of the file: + +License +------- + +Copyright (C) 2001-2008 by Kjetil Jacobsen +Copyright (C) 2001-2008 by Markus F.X.J. Oberhumer + +All rights reserved. + +PycURL is dual licensed under the LGPL and an MIT/X derivative license +based on the cURL license. A full copy of the LGPL license is included +in the file COPYING. A full copy of the MIT/X derivative license is +included in the file COPYING2. You can redistribute and/or modify PycURL +according to the terms of either license. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. --- /dev/null Wed Jul 1 14:56:47 2009 +++ new/src/pkgdefs/SUNWpython-pycurl/pkginfo Wed Jul 1 14:56:47 2009 @@ -0,0 +1,34 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +PKG="SUNWpython-pycurl" +NAME="PycURL" +DESC="Python bindings for libcURL" +ARCH="ISA" +VERSION="7.19.0" +SUNW_PKGVERS="1.0" +BASEDIR=/ +CATEGORY="system" +CLASSES="none" +VENDOR="Sun Microsystems, Inc." --- /dev/null Wed Jul 1 14:56:47 2009 +++ new/src/pkgdefs/SUNWpython-pycurl/prototype Wed Jul 1 14:56:47 2009 @@ -0,0 +1,60 @@ +i copyright +i pkginfo +d none usr 755 root sys +d none usr/lib 755 root bin +d none usr/lib/python2.4 755 root bin +d none usr/lib/python2.4/vendor-packages 755 root bin +d none usr/lib/python2.4/vendor-packages/curl 755 root bin +f none usr/lib/python2.4/vendor-packages/curl/__init__.py 444 root bin +f none usr/lib/python2.4/vendor-packages/curl/__init__.pyc 444 root bin +f none usr/lib/python2.4/vendor-packages/pycurl.so 444 root bin +d none usr/lib/python2.4/vendor-packages/share 755 root bin +d none usr/lib/python2.4/vendor-packages/share/doc 755 root bin +d none usr/lib/python2.4/vendor-packages/share/doc/pycurl 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/COPYING 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/COPYING2 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/ChangeLog 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/INSTALL 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/README 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/TODO 444 root bin +d none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/basicfirst.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/file_upload.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/linksys.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/retriever-multi.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/retriever.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/sfquery.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/examples/xmlrpc_curl.py 755 root bin +d none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html/callbacks.html 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html/curlmultiobject.html 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html/curlobject.html 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html/curlshareobject.html 444 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/html/pycurl.html 444 root bin +d none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_cb.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_debug.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_ftp.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_getinfo.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_gtk.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_internals.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_memleak.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi2.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi3.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi4.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi5.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi6.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi_socket.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi_socket_select.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi_timer.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_multi_vs_thread.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_post.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_post2.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_post3.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_share.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_socketopen.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_stringio.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/test_xmlrpc.py 755 root bin +f none usr/lib/python2.4/vendor-packages/share/doc/pycurl/tests/util.py 755 root bin --- old/src/setup.py Wed Jul 1 14:56:47 2009 +++ new/src/setup.py Wed Jul 1 14:56:47 2009 @@ -94,6 +94,14 @@ PLYURL = 'http://www.dabeaz.com/ply/%s' % (PLYARC) PLYHASH = '38efe9e03bc39d40ee73fa566eb9c1975f1a8003' +PC = 'pycurl' +PCIDIR = 'pycurl' +PCVER = '7.19.0' +PCARC = '%s-%s.tar.gz' % (PC, PCVER) +PCDIR = '%s-%s' % (PC, PCVER) +PCURL = 'http://pycurl.sourceforge.net/download/%s' % PCARC +PCHASH = '3fb59eca1461331bb9e9e8d6fe3b23eda961a416' + osname = platform.uname()[0].lower() ostype = arch = 'unknown' if osname == 'sunos': @@ -127,6 +135,9 @@ pkgs_dir = os.path.normpath(os.path.join(pwd, os.pardir, "packages", arch)) extern_dir = os.path.normpath(os.path.join(pwd, "extern")) +cacert_dir = os.path.normpath(os.path.join(pwd, "cacert")) +cacert_install_dir = 'usr/share/pkg/cacert' + py_install_dir = 'usr/lib/python2.4/vendor-packages' scripts_dir = 'usr/bin' @@ -216,6 +227,7 @@ 'pkg.actions', 'pkg.bundle', 'pkg.client', + 'pkg.client.transport', 'pkg.portable', 'pkg.publish', 'pkg.server' @@ -409,6 +421,10 @@ os.stat(dst_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + # Take cacerts in cacert_dir and install them in + # proto-area-relative cacert_install_dir + install_cacerts() + install_sw(CP, CPVER, CPARC, CPDIR, CPURL, CPIDIR, CPHASH) if "BUILD_PYOPENSSL" in os.environ and \ os.environ["BUILD_PYOPENSSL"] != "": @@ -431,6 +447,7 @@ MAKOHASH) install_sw(PLY, PLYVER, PLYARC, PLYDIR, PLYURL, PLYIDIR, PLYHASH) + install_sw(PC, PCVER, PCARC, PCDIR, PCURL, PCIDIR, PCHASH) # Remove some bits that we're not going to package, but be sure # not to complain if we try to remove them twice. @@ -468,7 +485,35 @@ print "bad checksum! %s != %s" % (swhash, hash.hexdigest()) return False +def install_cacerts(): + findir = os.path.join(root_dir, cacert_install_dir) + dir_util.mkpath(findir, verbose = True) + for f in os.listdir(cacert_dir): + + # Copy certificate + srcname = os.path.normpath(os.path.join(cacert_dir, f)) + dn, copied = file_util.copy_file(srcname, findir, update = True) + + if not copied: + continue + + # Call openssl to create hash symlink + cmd = ["/usr/bin/openssl", "x509", "-noout", "-hash", "-in", + srcname] + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + hashval = p.stdout.read() + p.wait() + + hashval = hashval.strip() + hashval += ".0" + + hashpath = os.path.join(findir, hashval) + if os.path.exists(hashpath): + os.unlink(hashpath) + os.symlink(f, hashpath) + def install_sw(swname, swver, swarc, swdir, swurl, swidir, swhash): swarc = os.path.join(extern_dir, swarc) swdir = os.path.join(extern_dir, swdir) @@ -499,6 +544,20 @@ for m in tar.getmembers(): tar.extract(m, extern_dir) tar.close() + + # If there are patches, apply them now. + patchdir = os.path.join("patch", swname) + already_patched = os.path.join(swdir, ".patched") + if os.path.exists(patchdir) and not os.path.exists(already_patched): + patches = os.listdir(patchdir) + for p in patches: + patchpath = os.path.join(os.path.pardir, + os.path.pardir, patchdir, p) + print "Applying %s to %s" % (p, swname) + subprocess.Popen(['patch', '-d', swdir, '-i', + patchpath]).wait() + file(already_patched, "w").close() + swinst_dir = os.path.join(root_dir, py_install_dir, swidir) if not os.path.exists(swinst_dir): print "installing %s" % swname --- old/src/tests/api/t_manifest.py Wed Jul 1 14:56:48 2009 +++ new/src/tests/api/t_manifest.py Wed Jul 1 14:56:48 2009 @@ -20,7 +20,7 @@ # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. import unittest @@ -34,7 +34,6 @@ import pkg.manifest as manifest import pkg.actions as actions import pkg.fmri as fmri -import pkg.client.retrieve as retrieve import pkg.client.filter as filter # Set the path so that modules above can be found --- old/src/tests/cli/t_api_search.py Wed Jul 1 14:56:49 2009 +++ new/src/tests/cli/t_api_search.py Wed Jul 1 14:56:49 2009 @@ -1344,6 +1344,7 @@ self._search_op(api_obj, remote, '*space', self.res_space_space_star) self._search_op(api_obj, remote, 'space', set()) + time.sleep(1) self.pkgsend_bulk(durl, self.space_pkg10, optional=False) # Need to add install of subsequent package and # local side search as well as remote @@ -1811,7 +1812,7 @@ self.res_remote_path, servers=[{"origin": durl}]) lfh = file(self.dc.get_logpath(), "rb") found = 0 - num_expected = 4 + num_expected = 6 for line in lfh: if "X-IPKG-UUID:" in line: tmp = line.split() @@ -1902,7 +1903,7 @@ set(), servers=[{"origin": self.durl1}]) self._search_op(api_obj, True, "example_path", set(), servers=[{"origin": self.durl3}]) - num_expected = { 1: 4, 2: 3, 3: 0 } + num_expected = { 1: 7, 2: 3, 3: 0 } for d in range(1,(len(self.dcs) + 1)): try: pub = api_obj.img.get_publisher( --- /dev/null Wed Jul 1 14:56:50 2009 +++ new/src/util/distro-import/118/common/SUNWipkg Wed Jul 1 14:56:50 2009 @@ -0,0 +1,10 @@ +package SUNWipkg +classification "System/Packaging" +import SUNWipkg +depend SUNWpython-cherrypy@3.1.1 +depend SUNWpython-mako@0.2.2 +depend SUNWpython-ply@3.1 +depend SUNWpython-pycurl@7.19.0 +depend SUNWpython-pyopenssl@0.8 +depend SUNWpython24-simplejson +end package --- /dev/null Wed Jul 1 14:56:51 2009 +++ new/src/util/distro-import/118/common/SUNWpython-pycurl Wed Jul 1 14:56:50 2009 @@ -0,0 +1,5 @@ +package SUNWpython-pycurl +classification "Development/Python" +import SUNWpython-pycurl +version 7.19.0 +end package