1 #!/usr/bin/python2.4
2 #
3 # CDDL HEADER START
4 #
5 # The contents of this file are subject to the terms of the
6 # Common Development and Distribution License (the "License").
7 # You may not use this file except in compliance with the License.
8 #
9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 # or http://www.opensolaris.org/os/licensing.
11 # See the License for the specific language governing permissions
12 # and limitations under the License.
13 #
14 # When distributing Covered Code, include this CDDL HEADER in each
15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 # If applicable, add the following below this CDDL HEADER, with the
17 # fields enclosed by brackets "[]" replaced with your own identifying
18 # information: Portions Copyright [yyyy] [name of copyright owner]
19 #
20 # CDDL HEADER END
21 #
22 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 # Use is subject to license terms.
24 #
25
26 # pkg.depotd - package repository daemon
27
28 # XXX The prototype pkg.depotd combines both the version management server that
29 # answers to pkgsend(1) sessions and the HTTP file server that answers to the
30 # various GET operations that a pkg(1) client makes. This split is expected to
31 # be made more explicit, by constraining the pkg(1) operations such that they
32 # can be served as a typical HTTP/HTTPS session. Thus, pkg.depotd will reduce
33 # to a special purpose HTTP/HTTPS server explicitly for the version management
34 # operations, and must manipulate the various state files--catalogs, in
35 # particular--such that the pkg(1) pull client can operately accurately with
36 # only a basic HTTP/HTTPS server in place.
37
38 # XXX We should support simple "last-modified" operations via HEAD queries.
39
40 # XXX Although we pushed the evaluation of next-version, etc. to the pull
41 # client, we should probably provide a query API to do same on the server, for
42 # dumb clients (like a notification service).
43
44 # The default authority for the depot.
45 AUTH_DEFAULT = "opensolaris.org"
46 # The default repository path.
47 REPO_PATH_DEFAULT = "/var/pkg/repo"
48 # The default path for static and other web content.
49 CONTENT_PATH_DEFAULT = "/usr/share/lib/pkg"
50 # cherrypy has a max_request_body_size parameter that determines whether the
51 # server should abort requests with REQUEST_ENTITY_TOO_LARGE when the request
52 # body is larger than the specified size (in bytes). The maximum size supported
53 # by cherrypy is 2048 * 1024 * 1024 - 1 (just short of 2048MB), but the default
54 # here is purposefully conservative.
55 MAX_REQUEST_BODY_SIZE = 128 * 1024 * 1024
56 # The default port(s) to serve data from.
57 PORT_DEFAULT = 80
58 SSL_PORT_DEFAULT = 443
59 # The minimum number of threads allowed.
60 THREADS_MIN = 1
61 # The default number of threads to start.
62 THREADS_DEFAULT = 10
63 # The maximum number of threads that can be started.
64 THREADS_MAX = 100
65 # The default server socket timeout in seconds. We want this to be longer than
66 # the normal default of 10 seconds to accommodate clients with poor quality
67 # connections.
68 SOCKET_TIMEOUT_DEFAULT = 60
69 # Whether modify operations should be allowed.
70 READONLY_DEFAULT = False
71 # Whether the repository catalog should be rebuilt on startup.
72 REBUILD_DEFAULT = False
73 # Whether the indexes should be rebuilt
74 REINDEX_DEFAULT = False
75 # Not in mirror mode by default
76 MIRROR_DEFAULT = False
77
78 import getopt
79 import gettext
80 import locale
81 import logging
82 import os
83 import os.path
84 import OpenSSL.crypto as crypto
85 import subprocess
86 import sys
87 import tempfile
88 import urlparse
89
90 try:
91 import cherrypy
92 version = cherrypy.__version__.split('.')
93 if map(int, version) < [3, 1, 0]:
94 raise ImportError
95 elif map(int, version) >= [3, 2, 0]:
96 raise ImportError
97 except ImportError:
98 print >> sys.stderr, """cherrypy 3.1.0 or greater (but less than """ \
99 """3.2.0) is required to use this program."""
100 sys.exit(2)
101
102 import pkg.catalog as catalog
103 from pkg.misc import port_available, msg, emsg, setlocale
104 import pkg.portable.util as os_util
105 import pkg.search_errors as search_errors
106 import pkg.server.config as config
107 import pkg.server.depot as depot
108 import pkg.server.depotresponse as dr
109 import pkg.server.errors as errors
110 import pkg.server.repositoryconfig as rc
111
112 class LogSink(object):
113 """This is a dummy object that we can use to discard log entries
114 without relying on non-portable interfaces such as /dev/null."""
115
116 def write(self, *args, **kwargs):
117 """Discard the bits."""
118 pass
119
120 def flush(self, *args, **kwargs):
121 """Discard the bits."""
122 pass
123
124 def usage(text):
125 if text:
126 emsg(text)
127
128 print """\
129 Usage: /usr/lib/pkg.depotd [-d repo_dir] [-p port] [-s threads]
130 [-t socket_timeout] [--cfg-file] [--content-root] [--debug]
131 [--log-access dest] [--log-errors dest] [--mirror] [--proxy-base url]
132 [--readonly] [--rebuild] [--ssl-cert-file] [--ssl-dialog]
133 [--ssl-key-file] [--writable-root dir]
134
135 --cfg-file The pathname of the file from which to read and to
136 write configuration information.
137 --content-root The file system path to the directory containing the
138 the static and other web content used by the depot's
139 browser user interface. The default value is
140 '/usr/share/lib/pkg'.
141 --debug The name of a debug feature to enable; or a whitespace
142 or comma separated list of features to enable. Possible
143 values are: headers.
144 --log-access The destination for any access related information
145 logged by the depot process. Possible values are:
146 stderr, stdout, none, or an absolute pathname. The
147 default value is stdout if stdout is a tty; otherwise
148 the default value is none.
149 --log-errors The destination for any errors or other information
150 logged by the depot process. Possible values are:
151 stderr, stdout, none, or an absolute pathname. The
152 default value is stderr.
153 --mirror Package mirror mode; publishing and metadata operations
154 disallowed. Cannot be used with --readonly or
155 --rebuild.
156 --proxy-base The url to use as the base for generating internal
157 redirects and content.
158 --readonly Read-only operation; modifying operations disallowed.
159 Cannot be used with --mirror or --rebuild.
160 --rebuild Re-build the catalog from pkgs in depot. Cannot be
161 used with --mirror or --readonly.
162 --ssl-cert-file The absolute pathname to a PEM-encoded Certificate file.
163 This option must be used with --ssl-key-file. Usage of
164 this option will cause the depot to only respond to SSL
165 requests on the provided port.
166 --ssl-dialog Specifies what method should be used to obtain the
167 passphrase needed to decrypt the file specified by
168 --ssl-key-file. Supported values are: builtin,
169 exec:/path/to/program, or smf:fmri. The default value
170 is builtin.
171 --ssl-key-file The absolute pathname to a PEM-encoded Private Key file.
172 This option must be used with --ssl-cert-file. Usage of
173 this option will cause the depot to only respond to SSL
174 requests on the provided port.
175 --writable-root The path to a directory to which the program has write
176 access. Used with --readonly to allow server to
177 create needed files, such as search indices, without
178 needing write access to the package information.
179 """
180 sys.exit(2)
181
182 class OptionError(Exception):
183 """Option exception. """
184
185 def __init__(self, *args):
186 Exception.__init__(self, *args)
187
188 if __name__ == "__main__":
189
190 setlocale(locale.LC_ALL, "")
191 gettext.install("pkg", "/usr/share/locale")
192
193 debug_features = {
194 "headers": False,
195 }
196 port = PORT_DEFAULT
197 port_provided = False
198 threads = THREADS_DEFAULT
199 socket_timeout = SOCKET_TIMEOUT_DEFAULT
200 readonly = READONLY_DEFAULT
201 rebuild = REBUILD_DEFAULT
202 reindex = REINDEX_DEFAULT
203 proxy_base = None
204 mirror = MIRROR_DEFAULT
205 repo_config_file = None
206 ssl_cert_file = None
207 ssl_key_file = None
208 ssl_dialog = "builtin"
209 writable_root = None
210
211 if "PKG_REPO" in os.environ:
212 repo_path = os.environ["PKG_REPO"]
213 else:
214 repo_path = REPO_PATH_DEFAULT
215
216 try:
217 content_root = os.environ["PKG_DEPOT_CONTENT"]
218 except KeyError:
219 try:
220 content_root = os.path.join(os.environ['PKG_HOME'],
221 'share/lib/pkg')
222 except KeyError:
223 content_root = CONTENT_PATH_DEFAULT
224
225 # By default, if the destination for a particular log type is not
226 # specified, this is where we will send the output.
227 log_routes = {
228 "access": "none",
229 "errors": "stderr"
230 }
231 log_opts = ["--log-%s" % log_type for log_type in log_routes]
232
233 # If stdout is a tty, then send access output there by default instead
234 # of discarding it.
235 if os.isatty(sys.stdout.fileno()):
236 log_routes["access"] = "stdout"
237
238 opt = None
239 try:
240 long_opts = ["cfg-file=", "content-root=", "debug=", "mirror",
241 "proxy-base=", "readonly", "rebuild", "refresh-index",
242 "ssl-cert-file=", "ssl-dialog=", "ssl-key-file=",
243 "writable-root="]
244 for opt in log_opts:
245 long_opts.append("%s=" % opt.lstrip('--'))
246 opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
247 long_opts)
248 for opt, arg in opts:
249 if opt == "-n":
250 sys.exit(0)
251 elif opt == "-d":
252 repo_path = arg
253 elif opt == "-p":
254 port = int(arg)
255 port_provided = True
256 elif opt == "-s":
257 threads = int(arg)
258 if threads < THREADS_MIN:
259 raise OptionError, \
260 "minimum value is %d" % THREADS_MIN
261 if threads > THREADS_MAX:
262 raise OptionError, \
263 "maximum value is %d" % THREADS_MAX
264 elif opt == "-t":
265 socket_timeout = int(arg)
266 elif opt == "--cfg-file":
267 repo_config_file = os.path.abspath(arg)
268 elif opt == "--content-root":
269 if arg == "":
270 raise OptionError, "You must specify " \
271 "a directory path."
272 content_root = arg
273 elif opt == "--debug":
274 if arg is None or arg == "":
275 raise OptionError, \
276 "A debug feature must be specified."
277
278 # A list of features can be specified using a
279 # "," or any whitespace character as separators.
280 if "," in arg:
281 features = arg.split(",")
282 else:
283 features = arg.split()
284
285 for f in features:
286 if f not in debug_features:
287 raise OptionError, \
288 "Invalid debug feature: " \
289 "%s." % f
290 debug_features[f] = True
291 elif opt in log_opts:
292 if arg is None or arg == "":
293 raise OptionError, \
294 "You must specify a log " \
295 "destination."
296 log_routes[opt.lstrip("--log-")] = arg
297 elif opt == "--mirror":
298 mirror = True
299 elif opt == "--proxy-base":
300 # Attempt to decompose the url provided into
301 # its base parts. This is done so we can
302 # remove any scheme information since we
303 # don't need it.
304 scheme, netloc, path, params, query, \
305 fragment = urlparse.urlparse(arg,
306 "http", allow_fragments=0)
307
308 if not netloc:
309 raise OptionError, "Unable to " \
310 "determine the hostname from " \
311 "the provided URL; please use a " \
312 "fully qualified URL."
313
314 scheme = scheme.lower()
315 if scheme not in ("http", "https"):
316 raise OptionError, "Invalid URL; http " \
317 "and https are the only supported " \
318 "schemes."
319
320 # Rebuild the url with the sanitized components.
321 proxy_base = urlparse.urlunparse((scheme,
322 netloc, path, params, query, fragment))
323 elif opt == "--readonly":
324 readonly = True
325 elif opt == "--rebuild":
326 rebuild = True
327 elif opt == "--refresh-index":
328 # Note: This argument is for internal use
329 # only. It's used when pkg.depotd is reexecing
330 # itself and needs to know that's the case.
331 # This flag is purposefully omitted in usage.
332 # The supported way to forcefully reindex is to
333 # kill any pkg.depot using that directory,
334 # remove the index directory, and restart the
335 # pkg.depot process. The index will be rebuilt
336 # automatically on startup.
337 reindex = True
338 elif opt == "--ssl-cert-file":
339 if arg == "none":
340 continue
341
342 ssl_cert_file = arg
343 if not os.path.isabs(ssl_cert_file):
344 raise OptionError, "The path to " \
345 "the Certificate file must be " \
346 "absolute."
347 elif not os.path.exists(ssl_cert_file):
348 raise OptionError, "The specified " \
349 "file does not exist."
350 elif not os.path.isfile(ssl_cert_file):
351 raise OptionError, "The specified " \
352 "pathname is not a file."
353 elif opt == "--ssl-key-file":
354 if arg == "none":
355 continue
356
357 ssl_key_file = arg
358 if not os.path.isabs(ssl_key_file):
359 raise OptionError, "The path to " \
360 "the Private Key file must be " \
361 "absolute."
362 elif not os.path.exists(ssl_key_file):
363 raise OptionError, "The specified " \
364 "file does not exist."
365 elif not os.path.isfile(ssl_key_file):
366 raise OptionError, "The specified " \
367 "pathname is not a file."
368 elif opt == "--ssl-dialog":
369 if arg != "builtin" and not \
370 arg.startswith("exec:/") and not \
371 arg.startswith("smf:"):
372 raise OptionError, "Invalid value " \
373 "specified. Expected: builtin, " \
374 "exec:/path/to/program, or " \
375 "smf:fmri."
376
377 f = arg
378 if f.startswith("exec:"):
379 if os_util.get_canonical_os_type() != \
380 "unix":
381 # Don't allow a somewhat
382 # insecure authentication method
383 # on some platforms.
384 raise OptionError, "exec is " \
385 "not a supported dialog " \
386 "type for this operating " \
387 "system."
388
389 f = os.path.abspath(f.split(
390 "exec:")[1])
391
392 if not os.path.isfile(f):
393 raise OptionError, "Invalid " \
394 "file path specified for " \
395 "exec."
396
397 f = "exec:%s" % f
398
399 ssl_dialog = f
400 elif opt == "--writable-root":
401 if arg == "":
402 raise OptionError, "You must specify " \
403 "a directory path."
404 writable_root = arg
405 except getopt.GetoptError, _e:
406 usage("pkg.depotd: %s" % _e.msg)
407 except OptionError, _e:
408 usage("pkg.depotd: option: %s -- %s" % (opt, _e))
409 except (ArithmeticError, ValueError):
410 usage("pkg.depotd: illegal option value: %s specified " \
411 "for option: %s" % (arg, opt))
412
413 if rebuild and reindex:
414 usage("--refresh-index cannot be used with --rebuild")
415 if rebuild and (readonly or mirror):
416 usage("--readonly and --mirror cannot be used with --rebuild")
417 if reindex and mirror:
418 usage("--mirror cannot be used with --refresh-index")
419 if reindex and readonly and not writable_root:
420 usage("--readonly can only be used with --refresh-index if "
421 "--writable-root is used")
422
423 if (ssl_cert_file and not ssl_key_file) or (ssl_key_file and not
424 ssl_cert_file):
425 usage("The --ssl-cert-file and --ssl-key-file options must "
426 "must both be provided when using either option.")
427 elif ssl_cert_file and ssl_key_file and not port_provided:
428 # If they didn't already specify a particular port, use the
429 # default SSL port instead.
430 port = SSL_PORT_DEFAULT
431
432 # If the program is going to reindex, the port is irrelevant since
433 # the program will not bind to a port.
434 if not reindex:
435 available, msg = port_available(None, port)
436 if not available:
437 print "pkg.depotd: unable to bind to the specified " \
438 "port: %d. Reason: %s" % (port, msg)
439 sys.exit(1)
440 else:
441 # Not applicable for reindexing operations.
442 content_root = None
443
444 fork_allowed = not reindex
445
446 scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT,
447 auto_create=not readonly, fork_allowed=fork_allowed,
448 writable_root=writable_root)
449
450 if readonly:
451 scfg.set_read_only()
452
453 if mirror:
454 scfg.set_mirror()
455
456 try:
457 scfg.init_dirs()
458 except (errors.SvrConfigError, EnvironmentError), _e:
459 print "pkg.depotd: an error occurred while trying to " \
460 "initialize the depot repository directory " \
461 "structures:\n%s" % _e
462 sys.exit(1)
463
464 key_data = None
465 if not reindex and ssl_cert_file and ssl_key_file and \
466 ssl_dialog != "builtin":
467 cmdline = None
468 def get_ssl_passphrase(*ignored):
469 p = None
470 try:
471 p = subprocess.Popen(cmdline, shell=True,
472 stdout=subprocess.PIPE,
473 stderr=None)
474 p.wait()
475 except Exception, __e:
476 print "pkg.depotd: an error occurred while " \
477 "executing [%s]; unable to obtain the " \
478 "passphrase needed to decrypt the SSL" \
479 "private key file: %s" % (cmdline, __e)
480 sys.exit(1)
481 return p.stdout.read().strip("\n")
482
483 if ssl_dialog.startswith("exec:"):
484 cmdline = "%s %s %d" % (ssl_dialog.split("exec:")[1],
485 "''", port)
486 elif ssl_dialog.startswith("smf:"):
487 cmdline = "/usr/bin/svcprop -p " \
488 "pkg_secure/ssl_key_passphrase %s" % (
489 ssl_dialog.split("smf:")[1])
490
491 # The key file requires decryption, but the user has requested
492 # exec-based authentication, so it will have to be decoded first
493 # to an un-named temporary file.
494 try:
495 key_file = file(ssl_key_file, "rb")
496 pkey = crypto.load_privatekey(crypto.FILETYPE_PEM,
497 key_file.read(), get_ssl_passphrase)
498
499 key_data = tempfile.TemporaryFile()
500 key_data.write(crypto.dump_privatekey(
501 crypto.FILETYPE_PEM, pkey))
502 key_data.seek(0)
503 except EnvironmentError, _e:
504 print "pkg.depotd: unable to read the SSL private " \
505 "key file: %s" % _e
506 sys.exit(1)
507 except crypto.Error, _e:
508 print "pkg.depotd: authentication or cryptography " \
509 "failure while attempting to decode\nthe SSL " \
510 "private key file: %s" % _e
511 sys.exit(1)
512 else:
513 # Redirect the server to the decrypted key file.
514 ssl_key_file = "/dev/fd/%d" % key_data.fileno()
515
516 # Setup our global configuration.
517 gconf = {
518 "checker.on": True,
519 "environment": "production",
520 "log.screen": False,
521 "server.max_request_body_size": MAX_REQUEST_BODY_SIZE,
522 "server.shutdown_timeout": 0,
523 "server.socket_host": "0.0.0.0",
524 "server.socket_port": port,
525 "server.socket_timeout": socket_timeout,
526 "server.ssl_certificate": ssl_cert_file,
527 "server.ssl_private_key": ssl_key_file,
528 "server.thread_pool": threads,
529 "tools.log_headers.on": True,
530 "tools.encode.on": True
531 }
532
533 if debug_features["headers"]:
534 # Despite its name, this only logs headers when there is an
535 # error; it's redundant with the debug feature enabled.
536 gconf["tools.log_headers.on"] = False
537
538 # Causes the headers of every request to be logged to the error
539 # log; even if an exception occurs.
540 gconf["tools.log_headers_always.on"] = True
541 cherrypy.tools.log_headers_always = cherrypy.Tool(
542 "on_start_resource",
543 cherrypy.lib.cptools.log_request_headers)
544
545 log_type_map = {
546 "errors": {
547 "param": "log.error_file",
548 "attr": "error_log"
549 },
550 "access": {
551 "param": "log.access_file",
552 "attr": "access_log"
553 }
554 }
555
556 for log_type in log_type_map:
557 dest = log_routes[log_type]
558 if dest in ("stdout", "stderr", "none"):
559 if dest == "none":
560 h = logging.StreamHandler(LogSink())
561 else:
562 h = logging.StreamHandler(eval("sys.%s" % \
563 dest))
564
565 h.setLevel(logging.DEBUG)
566 h.setFormatter(cherrypy._cplogging.logfmt)
567 log_obj = eval("cherrypy.log.%s" % \
568 log_type_map[log_type]["attr"])
569 log_obj.addHandler(h)
570 # Since we've replaced cherrypy's log handler with our
571 # own, we don't want the output directed to a file.
572 dest = ""
573
574 gconf[log_type_map[log_type]["param"]] = dest
575
576 cherrypy.config.update(gconf)
577
578 # Now that our logging, etc. has been setup, it's safe to perform any
579 # remaining preparation.
580 if reindex:
581 try:
582 scfg.acquire_catalog(rebuild=False, verbose=True)
583 except (search_errors.IndexingException,
584 catalog.CatalogPermissionsException,
585 errors.SvrConfigError), e:
586 cherrypy.log(str(e), "INDEX")
587 sys.exit(1)
588 sys.exit(0)
589
590 # Now build our site configuration.
591 conf = {
592 "/": {
593 # We have to override cherrypy's default response_class so that
594 # we have access to the write() callable to stream data
595 # directly to the client.
596 "wsgi.response_class": dr.DepotResponse,
597 },
598 "/robots.txt": {
599 "tools.staticfile.on": True,
600 "tools.staticfile.filename": os.path.join(scfg.web_root,
601 "robots.txt")
602 },
603 }
604
605 if proxy_base:
606 # This changes the base URL for our server, and is primarily
607 # intended to allow our depot process to operate behind Apache
608 # or some other webserver process.
609 #
610 # Visit the following URL for more information:
611 # http://cherrypy.org/wiki/BuiltinTools#tools.proxy
612 proxy_conf = {
613 "tools.proxy.on": True,
614 "tools.proxy.local": "",
615 "tools.proxy.base": proxy_base
616 }
617
618 # Now merge or add our proxy configuration information into the
619 # existing configuration.
620 for entry in proxy_conf:
621 conf["/"][entry] = proxy_conf[entry]
622
623 scfg.acquire_in_flight()
624 try:
625 scfg.acquire_catalog(rebuild=rebuild, verbose=True)
626 except (catalog.CatalogPermissionsException, errors.SvrConfigError), _e:
627 emsg("pkg.depotd: %s" % _e)
628 sys.exit(1)
629
630 try:
631 root = cherrypy.Application(depot.DepotHTTP(scfg,
632 repo_config_file))
633 except rc.InvalidAttributeValueError, _e:
634 emsg("pkg.depotd: repository.conf error: %s" % _e)
635 sys.exit(1)
636
637 try:
638 cherrypy.quickstart(root, config=conf)
639 except Exception, _e:
640 emsg("pkg.depotd: unknown error starting depot server, " \
641 "illegal option value specified?")
642 emsg(_e)
643 sys.exit(1)