1 #!/usr/bin/python2.4
2 #
3 # CDDL HEADER START
4 #
5 # The contents of this file are subject to the terms of the
6 # Common Development and Distribution License (the "License").
7 # You may not use this file except in compliance with the License.
8 #
9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 # or http://www.opensolaris.org/os/licensing.
11 # See the License for the specific language governing permissions
12 # and limitations under the License.
13 #
14 # When distributing Covered Code, include this CDDL HEADER in each
15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 # If applicable, add the following below this CDDL HEADER, with the
17 # fields enclosed by brackets "[]" replaced with your own identifying
18 # information: Portions Copyright [yyyy] [name of copyright owner]
19 #
20 # CDDL HEADER END
21 #
22 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 # Use is subject to license terms.
24 #
25
26 # pkg.depotd - package repository daemon
27
28 # XXX The prototype pkg.depotd combines both the version management server that
29 # answers to pkgsend(1) sessions and the HTTP file server that answers to the
30 # various GET operations that a pkg(1) client makes. This split is expected to
31 # be made more explicit, by constraining the pkg(1) operations such that they
32 # can be served as a typical HTTP/HTTPS session. Thus, pkg.depotd will reduce
33 # to a special purpose HTTP/HTTPS server explicitly for the version management
34 # operations, and must manipulate the various state files--catalogs, in
35 # particular--such that the pkg(1) pull client can operately accurately with
36 # only a basic HTTP/HTTPS server in place.
37
38 # XXX We should support simple "last-modified" operations via HEAD queries.
39
40 # XXX Although we pushed the evaluation of next-version, etc. to the pull
41 # client, we should probably provide a query API to do same on the server, for
42 # dumb clients (like a notification service).
43
44 # The default authority for the depot.
45 AUTH_DEFAULT = "opensolaris.org"
46 # The default repository path.
47 REPO_PATH_DEFAULT = "/var/pkg/repo"
48 # The default path for static and other web content.
49 CONTENT_PATH_DEFAULT = "/usr/share/lib/pkg"
50 # cherrypy has a max_request_body_size parameter that determines whether the
51 # server should abort requests with REQUEST_ENTITY_TOO_LARGE when the request
52 # body is larger than the specified size (in bytes). The maximum size supported
53 # by cherrypy is 2048 * 1024 * 1024 - 1 (just short of 2048MB), but the default
54 # here is purposefully conservative.
55 MAX_REQUEST_BODY_SIZE = 128 * 1024 * 1024
56 # The default port(s) to serve data from.
57 PORT_DEFAULT = 80
58 SSL_PORT_DEFAULT = 443
59 # The minimum number of threads allowed.
60 THREADS_MIN = 1
61 # The default number of threads to start.
62 THREADS_DEFAULT = 10
63 # The maximum number of threads that can be started.
64 THREADS_MAX = 100
65 # The default server socket timeout in seconds. We want this to be longer than
66 # the normal default of 10 seconds to accommodate clients with poor quality
67 # connections.
68 SOCKET_TIMEOUT_DEFAULT = 60
69 # Whether modify operations should be allowed.
70 READONLY_DEFAULT = False
71 # Whether the repository catalog should be rebuilt on startup.
72 REBUILD_DEFAULT = False
73 # Whether the indexes should be rebuilt
74 REINDEX_DEFAULT = False
75 # Not in mirror mode by default
76 MIRROR_DEFAULT = False
77
78 import getopt
79 import gettext
80 import locale
81 import logging
82 import os
83 import os.path
84 import OpenSSL.crypto as crypto
85 import subprocess
86 import sys
87 import tempfile
88 import urlparse
89
90 try:
91 import cherrypy
92 version = cherrypy.__version__.split('.')
93 if map(int, version) < [3, 1, 0]:
94 raise ImportError
95 elif map(int, version) >= [3, 2, 0]:
96 raise ImportError
97 except ImportError:
98 print >> sys.stderr, """cherrypy 3.1.0 or greater (but less than """ \
99 """3.2.0) is required to use this program."""
100 sys.exit(2)
101
102 import pkg.catalog as catalog
103 from pkg.misc import port_available, msg, emsg, setlocale
104 import pkg.portable.util as os_util
105 import pkg.search_errors as search_errors
106 import pkg.server.config as config
107 import pkg.server.depot as depot
108 import pkg.server.depotresponse as dr
109 import pkg.server.errors as errors
110 import pkg.server.repositoryconfig as rc
111
112 class LogSink(object):
113 """This is a dummy object that we can use to discard log entries
114 without relying on non-portable interfaces such as /dev/null."""
115
116 def write(self, *args, **kwargs):
117 """Discard the bits."""
118 pass
119
120 def flush(self, *args, **kwargs):
121 """Discard the bits."""
122 pass
123
124 def usage(text):
125 if text:
126 emsg(text)
127
128 print """\
129 Usage: /usr/lib/pkg.depotd [-d repo_dir] [-p port] [-s threads]
130 [-t socket_timeout] [--cfg-file] [--content-root] [--debug]
131 [--log-access dest] [--log-errors dest] [--mirror] [--nasty]
132 [--proxy-base url] [--readonly] [--rebuild] [--ssl-cert-file]
133 [--ssl-dialog] [--ssl-key-file] [--writable-root dir]
134
135 --cfg-file The pathname of the file from which to read and to
136 write configuration information.
137 --content-root The file system path to the directory containing the
138 the static and other web content used by the depot's
139 browser user interface. The default value is
140 '/usr/share/lib/pkg'.
141 --debug The name of a debug feature to enable; or a whitespace
142 or comma separated list of features to enable. Possible
143 values are: headers.
144 --log-access The destination for any access related information
145 logged by the depot process. Possible values are:
146 stderr, stdout, none, or an absolute pathname. The
147 default value is stdout if stdout is a tty; otherwise
148 the default value is none.
149 --log-errors The destination for any errors or other information
150 logged by the depot process. Possible values are:
151 stderr, stdout, none, or an absolute pathname. The
152 default value is stderr.
153 --mirror Package mirror mode; publishing and metadata operations
154 disallowed. Cannot be used with --readonly or
155 --rebuild.
156 --nasty Instruct the server to misbehave. At random intervals
157 it will time-out, send bad responses, hang up on
158 clients, and generally be hostile. The option
159 takes a value (1 to 100) for how nasty the server
160 should be.
161 --proxy-base The url to use as the base for generating internal
162 redirects and content.
163 --readonly Read-only operation; modifying operations disallowed.
164 Cannot be used with --mirror or --rebuild.
165 --rebuild Re-build the catalog from pkgs in depot. Cannot be
166 used with --mirror or --readonly.
167 --ssl-cert-file The absolute pathname to a PEM-encoded Certificate file.
168 This option must be used with --ssl-key-file. Usage of
169 this option will cause the depot to only respond to SSL
170 requests on the provided port.
171 --ssl-dialog Specifies what method should be used to obtain the
172 passphrase needed to decrypt the file specified by
173 --ssl-key-file. Supported values are: builtin,
174 exec:/path/to/program, or smf:fmri. The default value
175 is builtin.
176 --ssl-key-file The absolute pathname to a PEM-encoded Private Key file.
177 This option must be used with --ssl-cert-file. Usage of
178 this option will cause the depot to only respond to SSL
179 requests on the provided port.
180 --writable-root The path to a directory to which the program has write
181 access. Used with --readonly to allow server to
182 create needed files, such as search indices, without
183 needing write access to the package information.
184 """
185 sys.exit(2)
186
187 class OptionError(Exception):
188 """Option exception. """
189
190 def __init__(self, *args):
191 Exception.__init__(self, *args)
192
193 if __name__ == "__main__":
194
195 setlocale(locale.LC_ALL, "")
196 gettext.install("pkg", "/usr/share/locale")
197
198 debug_features = {
199 "headers": False,
200 }
201 port = PORT_DEFAULT
202 port_provided = False
203 threads = THREADS_DEFAULT
204 socket_timeout = SOCKET_TIMEOUT_DEFAULT
205 readonly = READONLY_DEFAULT
206 rebuild = REBUILD_DEFAULT
207 reindex = REINDEX_DEFAULT
208 proxy_base = None
209 mirror = MIRROR_DEFAULT
210 nasty = False
211 nasty_value = 0
212 repo_config_file = None
213 ssl_cert_file = None
214 ssl_key_file = None
215 ssl_dialog = "builtin"
216 writable_root = None
217
218 if "PKG_REPO" in os.environ:
219 repo_path = os.environ["PKG_REPO"]
220 else:
221 repo_path = REPO_PATH_DEFAULT
222
223 try:
224 content_root = os.environ["PKG_DEPOT_CONTENT"]
225 except KeyError:
226 try:
227 content_root = os.path.join(os.environ['PKG_HOME'],
228 'share/lib/pkg')
229 except KeyError:
230 content_root = CONTENT_PATH_DEFAULT
231
232 # By default, if the destination for a particular log type is not
233 # specified, this is where we will send the output.
234 log_routes = {
235 "access": "none",
236 "errors": "stderr"
237 }
238 log_opts = ["--log-%s" % log_type for log_type in log_routes]
239
240 # If stdout is a tty, then send access output there by default instead
241 # of discarding it.
242 if os.isatty(sys.stdout.fileno()):
243 log_routes["access"] = "stdout"
244
245 opt = None
246 try:
247 long_opts = ["cfg-file=", "content-root=", "debug=", "mirror",
248 "nasty=", "proxy-base=", "readonly", "rebuild",
249 "refresh-index", "ssl-cert-file=", "ssl-dialog=",
250 "ssl-key-file=", "writable-root="]
251 for opt in log_opts:
252 long_opts.append("%s=" % opt.lstrip('--'))
253 opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
254 long_opts)
255 for opt, arg in opts:
256 if opt == "-n":
257 sys.exit(0)
258 elif opt == "-d":
259 repo_path = arg
260 elif opt == "-p":
261 port = int(arg)
262 port_provided = True
263 elif opt == "-s":
264 threads = int(arg)
265 if threads < THREADS_MIN:
266 raise OptionError, \
267 "minimum value is %d" % THREADS_MIN
268 if threads > THREADS_MAX:
269 raise OptionError, \
270 "maximum value is %d" % THREADS_MAX
271 elif opt == "-t":
272 socket_timeout = int(arg)
273 elif opt == "--cfg-file":
274 repo_config_file = os.path.abspath(arg)
275 elif opt == "--content-root":
276 if arg == "":
277 raise OptionError, "You must specify " \
278 "a directory path."
279 content_root = arg
280 elif opt == "--debug":
281 if arg is None or arg == "":
282 raise OptionError, \
283 "A debug feature must be specified."
284
285 # A list of features can be specified using a
286 # "," or any whitespace character as separators.
287 if "," in arg:
288 features = arg.split(",")
289 else:
290 features = arg.split()
291
292 for f in features:
293 if f not in debug_features:
294 raise OptionError, \
295 "Invalid debug feature: " \
296 "%s." % f
297 debug_features[f] = True
298 elif opt in log_opts:
299 if arg is None or arg == "":
300 raise OptionError, \
301 "You must specify a log " \
302 "destination."
303 log_routes[opt.lstrip("--log-")] = arg
304 elif opt == "--mirror":
305 mirror = True
306 elif opt == "--nasty":
307 value_err = None
308 try:
309 nasty_value = int(arg)
310 except ValueError, e:
311 value_err = e
312
313 if value_err or (nasty_value > 100 or
314 nasty_value < 1):
315 raise OptionError, "Invalid value " \
316 "for nasty option.\n Please " \
317 "choose a value between 1 and 100."
318 nasty = True
319 elif opt == "--proxy-base":
320 # Attempt to decompose the url provided into
321 # its base parts. This is done so we can
322 # remove any scheme information since we
323 # don't need it.
324 scheme, netloc, path, params, query, \
325 fragment = urlparse.urlparse(arg,
326 "http", allow_fragments=0)
327
328 if not netloc:
329 raise OptionError, "Unable to " \
330 "determine the hostname from " \
331 "the provided URL; please use a " \
332 "fully qualified URL."
333
334 scheme = scheme.lower()
335 if scheme not in ("http", "https"):
336 raise OptionError, "Invalid URL; http " \
337 "and https are the only supported " \
338 "schemes."
339
340 # Rebuild the url with the sanitized components.
341 proxy_base = urlparse.urlunparse((scheme,
342 netloc, path, params, query, fragment))
343 elif opt == "--readonly":
344 readonly = True
345 elif opt == "--rebuild":
346 rebuild = True
347 elif opt == "--refresh-index":
348 # Note: This argument is for internal use
349 # only. It's used when pkg.depotd is reexecing
350 # itself and needs to know that's the case.
351 # This flag is purposefully omitted in usage.
352 # The supported way to forcefully reindex is to
353 # kill any pkg.depot using that directory,
354 # remove the index directory, and restart the
355 # pkg.depot process. The index will be rebuilt
356 # automatically on startup.
357 reindex = True
358 elif opt == "--ssl-cert-file":
359 if arg == "none":
360 continue
361
362 ssl_cert_file = arg
363 if not os.path.isabs(ssl_cert_file):
364 raise OptionError, "The path to " \
365 "the Certificate file must be " \
366 "absolute."
367 elif not os.path.exists(ssl_cert_file):
368 raise OptionError, "The specified " \
369 "file does not exist."
370 elif not os.path.isfile(ssl_cert_file):
371 raise OptionError, "The specified " \
372 "pathname is not a file."
373 elif opt == "--ssl-key-file":
374 if arg == "none":
375 continue
376
377 ssl_key_file = arg
378 if not os.path.isabs(ssl_key_file):
379 raise OptionError, "The path to " \
380 "the Private Key file must be " \
381 "absolute."
382 elif not os.path.exists(ssl_key_file):
383 raise OptionError, "The specified " \
384 "file does not exist."
385 elif not os.path.isfile(ssl_key_file):
386 raise OptionError, "The specified " \
387 "pathname is not a file."
388 elif opt == "--ssl-dialog":
389 if arg != "builtin" and not \
390 arg.startswith("exec:/") and not \
391 arg.startswith("smf:"):
392 raise OptionError, "Invalid value " \
393 "specified. Expected: builtin, " \
394 "exec:/path/to/program, or " \
395 "smf:fmri."
396
397 f = arg
398 if f.startswith("exec:"):
399 if os_util.get_canonical_os_type() != \
400 "unix":
401 # Don't allow a somewhat
402 # insecure authentication method
403 # on some platforms.
404 raise OptionError, "exec is " \
405 "not a supported dialog " \
406 "type for this operating " \
407 "system."
408
409 f = os.path.abspath(f.split(
410 "exec:")[1])
411
412 if not os.path.isfile(f):
413 raise OptionError, "Invalid " \
414 "file path specified for " \
415 "exec."
416
417 f = "exec:%s" % f
418
419 ssl_dialog = f
420 elif opt == "--writable-root":
421 if arg == "":
422 raise OptionError, "You must specify " \
423 "a directory path."
424 writable_root = arg
425 except getopt.GetoptError, _e:
426 usage("pkg.depotd: %s" % _e.msg)
427 except OptionError, _e:
428 usage("pkg.depotd: option: %s -- %s" % (opt, _e))
429 except (ArithmeticError, ValueError):
430 usage("pkg.depotd: illegal option value: %s specified " \
431 "for option: %s" % (arg, opt))
432
433 if rebuild and reindex:
434 usage("--refresh-index cannot be used with --rebuild")
435 if rebuild and (readonly or mirror):
436 usage("--readonly and --mirror cannot be used with --rebuild")
437 if reindex and mirror:
438 usage("--mirror cannot be used with --refresh-index")
439 if reindex and readonly and not writable_root:
440 usage("--readonly can only be used with --refresh-index if "
441 "--writable-root is used")
442
443 if (ssl_cert_file and not ssl_key_file) or (ssl_key_file and not
444 ssl_cert_file):
445 usage("The --ssl-cert-file and --ssl-key-file options must "
446 "must both be provided when using either option.")
447 elif ssl_cert_file and ssl_key_file and not port_provided:
448 # If they didn't already specify a particular port, use the
449 # default SSL port instead.
450 port = SSL_PORT_DEFAULT
451
452 # If the program is going to reindex, the port is irrelevant since
453 # the program will not bind to a port.
454 if not reindex:
455 available, msg = port_available(None, port)
456 if not available:
457 print "pkg.depotd: unable to bind to the specified " \
458 "port: %d. Reason: %s" % (port, msg)
459 sys.exit(1)
460 else:
461 # Not applicable for reindexing operations.
462 content_root = None
463
464 fork_allowed = not reindex
465
466 if nasty:
467 scfg = config.NastySvrConfig(repo_path, content_root,
468 AUTH_DEFAULT, auto_create=not readonly,
469 fork_allowed=fork_allowed, writable_root=writable_root)
470 scfg.set_nasty(nasty_value)
471 else:
472 scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT,
473 auto_create=not readonly, fork_allowed=fork_allowed,
474 writable_root=writable_root)
475
476 if readonly:
477 scfg.set_read_only()
478
479 if mirror:
480 scfg.set_mirror()
481
482
483 try:
484 scfg.init_dirs()
485 except (errors.SvrConfigError, EnvironmentError), _e:
486 print "pkg.depotd: an error occurred while trying to " \
487 "initialize the depot repository directory " \
488 "structures:\n%s" % _e
489 sys.exit(1)
490
491 key_data = None
492 if not reindex and ssl_cert_file and ssl_key_file and \
493 ssl_dialog != "builtin":
494 cmdline = None
495 def get_ssl_passphrase(*ignored):
496 p = None
497 try:
498 p = subprocess.Popen(cmdline, shell=True,
499 stdout=subprocess.PIPE,
500 stderr=None)
501 p.wait()
502 except Exception, __e:
503 print "pkg.depotd: an error occurred while " \
504 "executing [%s]; unable to obtain the " \
505 "passphrase needed to decrypt the SSL" \
506 "private key file: %s" % (cmdline, __e)
507 sys.exit(1)
508 return p.stdout.read().strip("\n")
509
510 if ssl_dialog.startswith("exec:"):
511 cmdline = "%s %s %d" % (ssl_dialog.split("exec:")[1],
512 "''", port)
513 elif ssl_dialog.startswith("smf:"):
514 cmdline = "/usr/bin/svcprop -p " \
515 "pkg_secure/ssl_key_passphrase %s" % (
516 ssl_dialog.split("smf:")[1])
517
518 # The key file requires decryption, but the user has requested
519 # exec-based authentication, so it will have to be decoded first
520 # to an un-named temporary file.
521 try:
522 key_file = file(ssl_key_file, "rb")
523 pkey = crypto.load_privatekey(crypto.FILETYPE_PEM,
524 key_file.read(), get_ssl_passphrase)
525
526 key_data = tempfile.TemporaryFile()
527 key_data.write(crypto.dump_privatekey(
528 crypto.FILETYPE_PEM, pkey))
529 key_data.seek(0)
530 except EnvironmentError, _e:
531 print "pkg.depotd: unable to read the SSL private " \
532 "key file: %s" % _e
533 sys.exit(1)
534 except crypto.Error, _e:
535 print "pkg.depotd: authentication or cryptography " \
536 "failure while attempting to decode\nthe SSL " \
537 "private key file: %s" % _e
538 sys.exit(1)
539 else:
540 # Redirect the server to the decrypted key file.
541 ssl_key_file = "/dev/fd/%d" % key_data.fileno()
542
543 # Setup our global configuration.
544 gconf = {
545 "checker.on": True,
546 "environment": "production",
547 "log.screen": False,
548 "server.max_request_body_size": MAX_REQUEST_BODY_SIZE,
549 "server.shutdown_timeout": 0,
550 "server.socket_host": "0.0.0.0",
551 "server.socket_port": port,
552 "server.socket_timeout": socket_timeout,
553 "server.ssl_certificate": ssl_cert_file,
554 "server.ssl_private_key": ssl_key_file,
555 "server.thread_pool": threads,
556 "tools.log_headers.on": True,
557 "tools.encode.on": True
558 }
559
560 if debug_features["headers"]:
561 # Despite its name, this only logs headers when there is an
562 # error; it's redundant with the debug feature enabled.
563 gconf["tools.log_headers.on"] = False
564
565 # Causes the headers of every request to be logged to the error
566 # log; even if an exception occurs.
567 gconf["tools.log_headers_always.on"] = True
568 cherrypy.tools.log_headers_always = cherrypy.Tool(
569 "on_start_resource",
570 cherrypy.lib.cptools.log_request_headers)
571
572 log_type_map = {
573 "errors": {
574 "param": "log.error_file",
575 "attr": "error_log"
576 },
577 "access": {
578 "param": "log.access_file",
579 "attr": "access_log"
580 }
581 }
582
583 for log_type in log_type_map:
584 dest = log_routes[log_type]
585 if dest in ("stdout", "stderr", "none"):
586 if dest == "none":
587 h = logging.StreamHandler(LogSink())
588 else:
589 h = logging.StreamHandler(eval("sys.%s" % \
590 dest))
591
592 h.setLevel(logging.DEBUG)
593 h.setFormatter(cherrypy._cplogging.logfmt)
594 log_obj = eval("cherrypy.log.%s" % \
595 log_type_map[log_type]["attr"])
596 log_obj.addHandler(h)
597 # Since we've replaced cherrypy's log handler with our
598 # own, we don't want the output directed to a file.
599 dest = ""
600
601 gconf[log_type_map[log_type]["param"]] = dest
602
603 cherrypy.config.update(gconf)
604
605 # Now that our logging, etc. has been setup, it's safe to perform any
606 # remaining preparation.
607 if reindex:
608 try:
609 scfg.acquire_catalog(rebuild=False, verbose=True)
610 except (search_errors.IndexingException,
611 catalog.CatalogPermissionsException,
612 errors.SvrConfigError), e:
613 cherrypy.log(str(e), "INDEX")
614 sys.exit(1)
615 sys.exit(0)
616
617 # Now build our site configuration.
618 conf = {
619 "/": {
620 # We have to override cherrypy's default response_class so that
621 # we have access to the write() callable to stream data
622 # directly to the client.
623 "wsgi.response_class": dr.DepotResponse,
624 },
625 "/robots.txt": {
626 "tools.staticfile.on": True,
627 "tools.staticfile.filename": os.path.join(scfg.web_root,
628 "robots.txt")
629 },
630 }
631
632 if proxy_base:
633 # This changes the base URL for our server, and is primarily
634 # intended to allow our depot process to operate behind Apache
635 # or some other webserver process.
636 #
637 # Visit the following URL for more information:
638 # http://cherrypy.org/wiki/BuiltinTools#tools.proxy
639 proxy_conf = {
640 "tools.proxy.on": True,
641 "tools.proxy.local": "",
642 "tools.proxy.base": proxy_base
643 }
644
645 # Now merge or add our proxy configuration information into the
646 # existing configuration.
647 for entry in proxy_conf:
648 conf["/"][entry] = proxy_conf[entry]
649
650 scfg.acquire_in_flight()
651 try:
652 scfg.acquire_catalog(rebuild=rebuild, verbose=True)
653 except (catalog.CatalogPermissionsException, errors.SvrConfigError), _e:
654 emsg("pkg.depotd: %s" % _e)
655 sys.exit(1)
656
657 try:
658 if nasty:
659 root = cherrypy.Application(depot.NastyDepotHTTP(scfg,
660 repo_config_file))
661 else:
662 root = cherrypy.Application(depot.DepotHTTP(scfg,
663 repo_config_file))
664 except rc.InvalidAttributeValueError, _e:
665 emsg("pkg.depotd: repository.conf error: %s" % _e)
666 sys.exit(1)
667
668 try:
669 cherrypy.quickstart(root, config=conf)
670 except Exception, _e:
671 emsg("pkg.depotd: unknown error starting depot server, " \
672 "illegal option value specified?")
673 emsg(_e)
674 sys.exit(1)