1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /*
  27  * graph.c - master restarter graph engine
  28  *
  29  *   The graph engine keeps a dependency graph of all service instances on the
  30  *   system, as recorded in the repository.  It decides when services should
  31  *   be brought up or down based on service states and dependencies and sends
  32  *   commands to restarters to effect any changes.  It also executes
  33  *   administrator commands sent by svcadm via the repository.
  34  *
  35  *   The graph is stored in uu_list_t *dgraph and its vertices are
  36  *   graph_vertex_t's, each of which has a name and an integer id unique to
  37  *   its name (see dict.c).  A vertex's type attribute designates the type
  38  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  39  *   service objects (since service instances may depend on another service,
  40  *   rather than service instance), GVT_FILE for files (which services may
  41  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  42  *   vertices are necessary because dependency lists may have particular
  43  *   grouping types (require any, require all, optional, or exclude) and
  44  *   event-propagation characteristics.
  45  *
  46  *   The initial graph is built by libscf_populate_graph() invoking
  47  *   dgraph_add_instance() for each instance in the repository.  The function
  48  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  49  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  50  *   The resulting web of vertices & edges associated with an instance's vertex
  51  *   includes
  52  *
  53  *     - an edge from the GVT_SVC vertex for the instance's service
  54  *
  55  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  56  *       restarter is not svc.startd
  57  *
  58  *     - edges from other GVT_INST vertices if the instance is a restarter
  59  *
  60  *     - for each dependency property group in the instance's "running"
  61  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  62  *       instance and the name of the property group
  63  *
  64  *     - for each value of the "entities" property in each dependency property
  65  *       group, an edge from the corresponding GVT_GROUP vertex to a
  66  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  67  *
  68  *     - edges from GVT_GROUP vertices for each dependent instance
  69  *
  70  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  71  *   there are problems, or if a service is mentioned in a dependency but does
  72  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  73  *
  74  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  75  *   See restarter.c for more information.
  76  *
  77  *   The properties of an instance fall into two classes: immediate and
  78  *   snapshotted.  Immediate properties should have an immediate effect when
  79  *   changed.  Snapshotted properties should be read from a snapshot, so they
  80  *   only change when the snapshot changes.  The immediate properties used by
  81  *   the graph engine are general/enabled, general/restarter, and the properties
  82  *   in the restarter_actions property group.  Since they are immediate, they
  83  *   are not read out of a snapshot.  The snapshotted properties used by the
  84  *   graph engine are those in the property groups with type "dependency" and
  85  *   are read out of the "running" snapshot.  The "running" snapshot is created
  86  *   by the the graph engine as soon as possible, and it is updated, along with
  87  *   in-core copies of the data (dependency information for the graph engine) on
  88  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  89  *   updates the "start" snapshot from the "running" snapshot whenever a service
  90  *   comes online.
  91  *
  92  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  93  *   the dependents first before shutting down the requested service.
  94  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  95  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  96  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  97  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
  98  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
  99  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 100  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 101  *   exposed leaves. We do the same until we reach the last leaf (the one with
 102  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 103  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 104  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 105  *   continue to propagate the offline event to the vertex's dependencies.
 106  */
 107 
 108 #include <sys/uadmin.h>
 109 #include <sys/wait.h>
 110 
 111 #include <assert.h>
 112 #include <errno.h>
 113 #include <fcntl.h>
 114 #include <libscf.h>
 115 #include <libscf_priv.h>
 116 #include <libuutil.h>
 117 #include <locale.h>
 118 #include <poll.h>
 119 #include <pthread.h>
 120 #include <signal.h>
 121 #include <stddef.h>
 122 #include <stdio.h>
 123 #include <stdlib.h>
 124 #include <string.h>
 125 #include <strings.h>
 126 #include <sys/statvfs.h>
 127 #include <sys/uadmin.h>
 128 #include <zone.h>
 129 
 130 #include "startd.h"
 131 #include "protocol.h"
 132 
 133 
 134 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 135 
 136 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 137 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 138 
 139 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 140 #define VERTEX_INUSE    1       /* vertex is still in use */
 141 
 142 /*
 143  * Services in these states are not considered 'down' by the
 144  * milestone/shutdown code.
 145  */
 146 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 147         (state) == RESTARTER_STATE_DEGRADED || \
 148         (state) == RESTARTER_STATE_OFFLINE)
 149 
 150 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 151 static uu_list_t *dgraph;
 152 static pthread_mutex_t dgraph_lock;
 153 
 154 /*
 155  * milestone indicates the current subgraph.  When NULL, it is the entire
 156  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 157  * services on which the target vertex depends.
 158  */
 159 static graph_vertex_t *milestone = NULL;
 160 static boolean_t initial_milestone_set = B_FALSE;
 161 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 162 
 163 /* protected by dgraph_lock */
 164 static boolean_t sulogin_thread_running = B_FALSE;
 165 static boolean_t sulogin_running = B_FALSE;
 166 static boolean_t console_login_ready = B_FALSE;
 167 
 168 /* Number of services to come down to complete milestone transition. */
 169 static uint_t non_subgraph_svcs;
 170 
 171 /*
 172  * These variables indicate what should be done when we reach the milestone
 173  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 174  * dgraph_set_instance_state().
 175  */
 176 static int halting = -1;
 177 static boolean_t go_single_user_mode = B_FALSE;
 178 static boolean_t go_to_level1 = B_FALSE;
 179 
 180 /*
 181  * This tracks the legacy runlevel to ensure we signal init and manage
 182  * utmpx entries correctly.
 183  */
 184 static char current_runlevel = '\0';
 185 
 186 /* Number of single user threads currently running */
 187 static pthread_mutex_t single_user_thread_lock;
 188 static int single_user_thread_count = 0;
 189 
 190 /* Statistics for dependency cycle-checking */
 191 static u_longlong_t dep_inserts = 0;
 192 static u_longlong_t dep_cycle_ns = 0;
 193 static u_longlong_t dep_insert_ns = 0;
 194 
 195 
 196 static const char * const emsg_invalid_restarter =
 197         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 198         "(see 'svcs -xv' for details).\n";
 199 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 200 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 201 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 202 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 203 
 204 
 205 /*
 206  * These services define the system being "up".  If none of them can come
 207  * online, then we will run sulogin on the console.  Note that the install ones
 208  * are for the miniroot and when installing CDs after the first.  can_come_up()
 209  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 210  * started by dgraph_set_instance_state() or single_user_thread().
 211  *
 212  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 213  * entry, which is only used when booting_to_single_user (boot -s) is set.
 214  * This is because when doing a "boot -s", sulogin is started from specials.c
 215  * after milestone/single-user comes online, for backwards compatibility.
 216  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 217  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 218  */
 219 static const char * const up_svcs[] = {
 220         SCF_MILESTONE_SINGLE_USER,
 221         CONSOLE_LOGIN_FMRI,
 222         "svc:/system/install-setup:default",
 223         "svc:/system/install:default",
 224         NULL
 225 };
 226 
 227 /* This array must have an element for each non-NULL element of up_svcs[]. */
 228 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 229 
 230 /* These are for seed repository magic.  See can_come_up(). */
 231 static const char * const manifest_import =
 232         "svc:/system/manifest-import:default";
 233 static graph_vertex_t *manifest_import_p = NULL;
 234 
 235 
 236 static char target_milestone_as_runlevel(void);
 237 static void graph_runlevel_changed(char rl, int online);
 238 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 239 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 240 static int mark_subtree(graph_edge_t *, void *);
 241 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 242 
 243 /*
 244  * graph_vertex_compare()
 245  *      This function can compare either int *id or * graph_vertex_t *gv
 246  *      values, as the vertex id is always the first element of a
 247  *      graph_vertex structure.
 248  */
 249 /* ARGSUSED */
 250 static int
 251 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 252 {
 253         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 254         int rc_id = *(int *)rc_arg;
 255 
 256         if (lc_id > rc_id)
 257                 return (1);
 258         if (lc_id < rc_id)
 259                 return (-1);
 260         return (0);
 261 }
 262 
 263 void
 264 graph_init()
 265 {
 266         graph_edge_pool = startd_list_pool_create("graph_edges",
 267             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 268             UU_LIST_POOL_DEBUG);
 269         assert(graph_edge_pool != NULL);
 270 
 271         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 272             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 273             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 274         assert(graph_vertex_pool != NULL);
 275 
 276         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 277         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 278         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 279         assert(dgraph != NULL);
 280 
 281         if (!st->st_initial)
 282                 current_runlevel = utmpx_get_runlevel();
 283 
 284         log_framework(LOG_DEBUG, "Initialized graph\n");
 285 }
 286 
 287 static graph_vertex_t *
 288 vertex_get_by_name(const char *name)
 289 {
 290         int id;
 291 
 292         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 293 
 294         id = dict_lookup_byname(name);
 295         if (id == -1)
 296                 return (NULL);
 297 
 298         return (uu_list_find(dgraph, &id, NULL, NULL));
 299 }
 300 
 301 static graph_vertex_t *
 302 vertex_get_by_id(int id)
 303 {
 304         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 305 
 306         if (id == -1)
 307                 return (NULL);
 308 
 309         return (uu_list_find(dgraph, &id, NULL, NULL));
 310 }
 311 
 312 /*
 313  * Creates a new vertex with the given name, adds it to the graph, and returns
 314  * a pointer to it.  The graph lock must be held by this thread on entry.
 315  */
 316 static graph_vertex_t *
 317 graph_add_vertex(const char *name)
 318 {
 319         int id;
 320         graph_vertex_t *v;
 321         void *p;
 322         uu_list_index_t idx;
 323 
 324         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 325 
 326         id = dict_insert(name);
 327 
 328         v = startd_zalloc(sizeof (*v));
 329 
 330         v->gv_id = id;
 331 
 332         v->gv_name = startd_alloc(strlen(name) + 1);
 333         (void) strcpy(v->gv_name, name);
 334 
 335         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 336         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 337 
 338         p = uu_list_find(dgraph, &id, NULL, &idx);
 339         assert(p == NULL);
 340 
 341         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 342         uu_list_insert(dgraph, v, idx);
 343 
 344         return (v);
 345 }
 346 
 347 /*
 348  * Removes v from the graph and frees it.  The graph should be locked by this
 349  * thread, and v should have no edges associated with it.
 350  */
 351 static void
 352 graph_remove_vertex(graph_vertex_t *v)
 353 {
 354         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 355 
 356         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 357         assert(uu_list_numnodes(v->gv_dependents) == 0);
 358         assert(v->gv_refs == 0);
 359 
 360         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 361         uu_list_destroy(v->gv_dependencies);
 362         uu_list_destroy(v->gv_dependents);
 363         uu_list_remove(dgraph, v);
 364 
 365         startd_free(v, sizeof (graph_vertex_t));
 366 }
 367 
 368 static void
 369 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 370 {
 371         graph_edge_t *e, *re;
 372         int r;
 373 
 374         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 375 
 376         e = startd_alloc(sizeof (graph_edge_t));
 377         re = startd_alloc(sizeof (graph_edge_t));
 378 
 379         e->ge_parent = fv;
 380         e->ge_vertex = tv;
 381 
 382         re->ge_parent = tv;
 383         re->ge_vertex = fv;
 384 
 385         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 386         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 387         assert(r == 0);
 388 
 389         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 390         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 391         assert(r == 0);
 392 }
 393 
 394 static void
 395 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 396 {
 397         graph_edge_t *e;
 398 
 399         for (e = uu_list_first(v->gv_dependencies);
 400             e != NULL;
 401             e = uu_list_next(v->gv_dependencies, e)) {
 402                 if (e->ge_vertex == dv) {
 403                         uu_list_remove(v->gv_dependencies, e);
 404                         startd_free(e, sizeof (graph_edge_t));
 405                         break;
 406                 }
 407         }
 408 
 409         for (e = uu_list_first(dv->gv_dependents);
 410             e != NULL;
 411             e = uu_list_next(dv->gv_dependents, e)) {
 412                 if (e->ge_vertex == v) {
 413                         uu_list_remove(dv->gv_dependents, e);
 414                         startd_free(e, sizeof (graph_edge_t));
 415                         break;
 416                 }
 417         }
 418 }
 419 
 420 static void
 421 remove_inst_vertex(graph_vertex_t *v)
 422 {
 423         graph_edge_t *e;
 424         graph_vertex_t *sv;
 425         int i;
 426 
 427         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 428         assert(uu_list_numnodes(v->gv_dependents) == 1);
 429         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 430         assert(v->gv_refs == 0);
 431         assert((v->gv_flags & GV_CONFIGURED) == 0);
 432 
 433         e = uu_list_first(v->gv_dependents);
 434         sv = e->ge_vertex;
 435         graph_remove_edge(sv, v);
 436 
 437         for (i = 0; up_svcs[i] != NULL; ++i) {
 438                 if (up_svcs_p[i] == v)
 439                         up_svcs_p[i] = NULL;
 440         }
 441 
 442         if (manifest_import_p == v)
 443                 manifest_import_p = NULL;
 444 
 445         graph_remove_vertex(v);
 446 
 447         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 448             uu_list_numnodes(sv->gv_dependents) == 0 &&
 449             sv->gv_refs == 0)
 450                 graph_remove_vertex(sv);
 451 }
 452 
 453 static void
 454 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 455     void *arg)
 456 {
 457         graph_edge_t *e;
 458 
 459         for (e = uu_list_first(v->gv_dependents);
 460             e != NULL;
 461             e = uu_list_next(v->gv_dependents, e))
 462                 func(e->ge_vertex, arg);
 463 }
 464 
 465 static void
 466 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 467         void *), void *arg)
 468 {
 469         graph_edge_t *e;
 470 
 471         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 472 
 473         for (e = uu_list_first(v->gv_dependencies);
 474             e != NULL;
 475             e = uu_list_next(v->gv_dependencies, e)) {
 476 
 477                 func(e->ge_vertex, arg);
 478         }
 479 }
 480 
 481 /*
 482  * Generic graph walking function.
 483  *
 484  * Given a vertex, this function will walk either dependencies
 485  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 486  * for the entire graph.  It will avoid cycles and never visit the same vertex
 487  * twice.
 488  *
 489  * We avoid traversing exclusion dependencies, because they are allowed to
 490  * create cycles in the graph.  When propagating satisfiability, there is no
 491  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 492  * test for satisfiability.
 493  *
 494  * The walker takes two callbacks.  The first is called before examining the
 495  * dependents of each vertex.  The second is called on each vertex after
 496  * examining its dependents.  This allows is_path_to() to construct a path only
 497  * after the target vertex has been found.
 498  */
 499 typedef enum {
 500         WALK_DEPENDENTS,
 501         WALK_DEPENDENCIES
 502 } graph_walk_dir_t;
 503 
 504 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 505 
 506 typedef struct graph_walk_info {
 507         graph_walk_dir_t        gi_dir;
 508         uchar_t                 *gi_visited;    /* vertex bitmap */
 509         int                     (*gi_pre)(graph_vertex_t *, void *);
 510         void                    (*gi_post)(graph_vertex_t *, void *);
 511         void                    *gi_arg;        /* callback arg */
 512         int                     gi_ret;         /* return value */
 513 } graph_walk_info_t;
 514 
 515 static int
 516 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 517 {
 518         uu_list_t *list;
 519         int r;
 520         graph_vertex_t *v = e->ge_vertex;
 521         int i;
 522         uint_t b;
 523 
 524         i = v->gv_id / 8;
 525         b = 1 << (v->gv_id % 8);
 526 
 527         /*
 528          * Check to see if we've visited this vertex already.
 529          */
 530         if (gip->gi_visited[i] & b)
 531                 return (UU_WALK_NEXT);
 532 
 533         gip->gi_visited[i] |= b;
 534 
 535         /*
 536          * Don't follow exclusions.
 537          */
 538         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 539                 return (UU_WALK_NEXT);
 540 
 541         /*
 542          * Call pre-visit callback.  If this doesn't terminate the walk,
 543          * continue search.
 544          */
 545         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 546                 /*
 547                  * Recurse using appropriate list.
 548                  */
 549                 if (gip->gi_dir == WALK_DEPENDENTS)
 550                         list = v->gv_dependents;
 551                 else
 552                         list = v->gv_dependencies;
 553 
 554                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 555                     gip, 0);
 556                 assert(r == 0);
 557         }
 558 
 559         /*
 560          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 561          */
 562         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 563 
 564         /*
 565          * If given a post-callback, call the function for every vertex.
 566          */
 567         if (gip->gi_post != NULL)
 568                 (void) gip->gi_post(v, gip->gi_arg);
 569 
 570         /*
 571          * Preserve the callback's return value.  If the callback returns
 572          * UU_WALK_DONE, then we propagate that to the caller in order to
 573          * terminate the walk.
 574          */
 575         return (gip->gi_ret);
 576 }
 577 
 578 static void
 579 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 580     int (*pre)(graph_vertex_t *, void *),
 581     void (*post)(graph_vertex_t *, void *), void *arg)
 582 {
 583         graph_walk_info_t gi;
 584         graph_edge_t fake;
 585         size_t sz = dictionary->dict_new_id / 8 + 1;
 586 
 587         gi.gi_visited = startd_zalloc(sz);
 588         gi.gi_pre = pre;
 589         gi.gi_post = post;
 590         gi.gi_arg = arg;
 591         gi.gi_dir = dir;
 592         gi.gi_ret = 0;
 593 
 594         /*
 595          * Fake up an edge for the first iteration
 596          */
 597         fake.ge_vertex = v;
 598         (void) graph_walk_recurse(&fake, &gi);
 599 
 600         startd_free(gi.gi_visited, sz);
 601 }
 602 
 603 typedef struct child_search {
 604         int     id;             /* id of vertex to look for */
 605         uint_t  depth;          /* recursion depth */
 606         /*
 607          * While the vertex is not found, path is NULL.  After the search, if
 608          * the vertex was found then path should point to a -1-terminated
 609          * array of vertex id's which constitute the path to the vertex.
 610          */
 611         int     *path;
 612 } child_search_t;
 613 
 614 static int
 615 child_pre(graph_vertex_t *v, void *arg)
 616 {
 617         child_search_t *cs = arg;
 618 
 619         cs->depth++;
 620 
 621         if (v->gv_id == cs->id) {
 622                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 623                 cs->path[cs->depth] = -1;
 624                 return (UU_WALK_DONE);
 625         }
 626 
 627         return (UU_WALK_NEXT);
 628 }
 629 
 630 static void
 631 child_post(graph_vertex_t *v, void *arg)
 632 {
 633         child_search_t *cs = arg;
 634 
 635         cs->depth--;
 636 
 637         if (cs->path != NULL)
 638                 cs->path[cs->depth] = v->gv_id;
 639 }
 640 
 641 /*
 642  * Look for a path from from to to.  If one exists, returns a pointer to
 643  * a NULL-terminated array of pointers to the vertices along the path.  If
 644  * there is no path, returns NULL.
 645  */
 646 static int *
 647 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 648 {
 649         child_search_t cs;
 650 
 651         cs.id = to->gv_id;
 652         cs.depth = 0;
 653         cs.path = NULL;
 654 
 655         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 656 
 657         return (cs.path);
 658 }
 659 
 660 /*
 661  * Given an array of int's as returned by is_path_to, allocates a string of
 662  * their names joined by newlines.  Returns the size of the allocated buffer
 663  * in *sz and frees path.
 664  */
 665 static void
 666 path_to_str(int *path, char **cpp, size_t *sz)
 667 {
 668         int i;
 669         graph_vertex_t *v;
 670         size_t allocd, new_allocd;
 671         char *new, *name;
 672 
 673         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 674         assert(path[0] != -1);
 675 
 676         allocd = 1;
 677         *cpp = startd_alloc(1);
 678         (*cpp)[0] = '\0';
 679 
 680         for (i = 0; path[i] != -1; ++i) {
 681                 name = NULL;
 682 
 683                 v = vertex_get_by_id(path[i]);
 684 
 685                 if (v == NULL)
 686                         name = "<deleted>";
 687                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 688                         name = v->gv_name;
 689 
 690                 if (name != NULL) {
 691                         new_allocd = allocd + strlen(name) + 1;
 692                         new = startd_alloc(new_allocd);
 693                         (void) strcpy(new, *cpp);
 694                         (void) strcat(new, name);
 695                         (void) strcat(new, "\n");
 696 
 697                         startd_free(*cpp, allocd);
 698 
 699                         *cpp = new;
 700                         allocd = new_allocd;
 701                 }
 702         }
 703 
 704         startd_free(path, sizeof (int) * (i + 1));
 705 
 706         *sz = allocd;
 707 }
 708 
 709 
 710 /*
 711  * This function along with run_sulogin() implements an exclusion relationship
 712  * between system/console-login and sulogin.  run_sulogin() will fail if
 713  * system/console-login is online, and the graph engine should call
 714  * graph_clogin_start() to bring system/console-login online, which defers the
 715  * start if sulogin is running.
 716  */
 717 static void
 718 graph_clogin_start(graph_vertex_t *v)
 719 {
 720         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 721 
 722         if (sulogin_running)
 723                 console_login_ready = B_TRUE;
 724         else
 725                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 726 }
 727 
 728 static void
 729 graph_su_start(graph_vertex_t *v)
 730 {
 731         /*
 732          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 733          * entry with a runlevel of 'S', before jumping to the final
 734          * target runlevel (as set in initdefault).  We mimic that legacy
 735          * behavior here.
 736          */
 737         utmpx_set_runlevel('S', '0', B_FALSE);
 738         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 739 }
 740 
 741 static void
 742 graph_post_su_online(void)
 743 {
 744         graph_runlevel_changed('S', 1);
 745 }
 746 
 747 static void
 748 graph_post_su_disable(void)
 749 {
 750         graph_runlevel_changed('S', 0);
 751 }
 752 
 753 static void
 754 graph_post_mu_online(void)
 755 {
 756         graph_runlevel_changed('2', 1);
 757 }
 758 
 759 static void
 760 graph_post_mu_disable(void)
 761 {
 762         graph_runlevel_changed('2', 0);
 763 }
 764 
 765 static void
 766 graph_post_mus_online(void)
 767 {
 768         graph_runlevel_changed('3', 1);
 769 }
 770 
 771 static void
 772 graph_post_mus_disable(void)
 773 {
 774         graph_runlevel_changed('3', 0);
 775 }
 776 
 777 static struct special_vertex_info {
 778         const char      *name;
 779         void            (*start_f)(graph_vertex_t *);
 780         void            (*post_online_f)(void);
 781         void            (*post_disable_f)(void);
 782 } special_vertices[] = {
 783         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 784         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 785             graph_post_su_online, graph_post_su_disable },
 786         { SCF_MILESTONE_MULTI_USER, NULL,
 787             graph_post_mu_online, graph_post_mu_disable },
 788         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 789             graph_post_mus_online, graph_post_mus_disable },
 790         { NULL },
 791 };
 792 
 793 
 794 void
 795 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 796 {
 797         switch (e) {
 798         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 799                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 800 
 801                 MUTEX_LOCK(&st->st_load_lock);
 802                 st->st_load_instances++;
 803                 MUTEX_UNLOCK(&st->st_load_lock);
 804                 break;
 805 
 806         case RESTARTER_EVENT_TYPE_ENABLE:
 807                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 808                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 809                     v->gv_state == RESTARTER_STATE_DISABLED ||
 810                     v->gv_state == RESTARTER_STATE_MAINT);
 811                 break;
 812 
 813         case RESTARTER_EVENT_TYPE_DISABLE:
 814         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 815                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 816                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 817                 break;
 818 
 819         case RESTARTER_EVENT_TYPE_STOP:
 820                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 821                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 822                     v->gv_state == RESTARTER_STATE_ONLINE);
 823                 break;
 824 
 825         case RESTARTER_EVENT_TYPE_START:
 826                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 827                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 828                 break;
 829 
 830         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 831         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 832         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 833         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 834         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 835         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 836         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 837         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 838         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 839                 break;
 840 
 841         default:
 842 #ifndef NDEBUG
 843                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 844 #endif
 845                 abort();
 846         }
 847 
 848         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e);
 849 }
 850 
 851 static void
 852 graph_unset_restarter(graph_vertex_t *v)
 853 {
 854         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 855         assert(v->gv_flags & GV_CONFIGURED);
 856 
 857         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 858 
 859         if (v->gv_restarter_id != -1) {
 860                 graph_vertex_t *rv;
 861 
 862                 rv = vertex_get_by_id(v->gv_restarter_id);
 863                 graph_remove_edge(v, rv);
 864         }
 865 
 866         v->gv_restarter_id = -1;
 867         v->gv_restarter_channel = NULL;
 868 }
 869 
 870 /*
 871  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 872  * dgraph otherwise return VERTEX_INUSE.
 873  */
 874 static int
 875 free_if_unrefed(graph_vertex_t *v)
 876 {
 877         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 878 
 879         if (v->gv_refs > 0)
 880                 return (VERTEX_INUSE);
 881 
 882         if (v->gv_type == GVT_SVC &&
 883             uu_list_numnodes(v->gv_dependents) == 0 &&
 884             uu_list_numnodes(v->gv_dependencies) == 0) {
 885                 graph_remove_vertex(v);
 886                 return (VERTEX_REMOVED);
 887         } else if (v->gv_type == GVT_INST &&
 888             (v->gv_flags & GV_CONFIGURED) == 0 &&
 889             uu_list_numnodes(v->gv_dependents) == 1 &&
 890             uu_list_numnodes(v->gv_dependencies) == 0) {
 891                 remove_inst_vertex(v);
 892                 return (VERTEX_REMOVED);
 893         }
 894 
 895         return (VERTEX_INUSE);
 896 }
 897 
 898 static void
 899 delete_depgroup(graph_vertex_t *v)
 900 {
 901         graph_edge_t *e;
 902         graph_vertex_t *dv;
 903 
 904         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 905         assert(v->gv_type == GVT_GROUP);
 906         assert(uu_list_numnodes(v->gv_dependents) == 0);
 907 
 908         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 909                 dv = e->ge_vertex;
 910 
 911                 graph_remove_edge(v, dv);
 912 
 913                 switch (dv->gv_type) {
 914                 case GVT_INST:          /* instance dependency */
 915                 case GVT_SVC:           /* service dependency */
 916                         (void) free_if_unrefed(dv);
 917                         break;
 918 
 919                 case GVT_FILE:          /* file dependency */
 920                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 921                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 922                                 graph_remove_vertex(dv);
 923                         break;
 924 
 925                 default:
 926 #ifndef NDEBUG
 927                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 928                             __LINE__, dv->gv_type);
 929 #endif
 930                         abort();
 931                 }
 932         }
 933 
 934         graph_remove_vertex(v);
 935 }
 936 
 937 static int
 938 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
 939 {
 940         graph_vertex_t *v = ptrs[0];
 941         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
 942         graph_vertex_t *dv;
 943 
 944         dv = e->ge_vertex;
 945 
 946         /*
 947          * We have four possibilities here:
 948          *   - GVT_INST: restarter
 949          *   - GVT_GROUP - GVT_INST: instance dependency
 950          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
 951          *   - GVT_GROUP - GVT_FILE: file dependency
 952          */
 953         switch (dv->gv_type) {
 954         case GVT_INST:  /* restarter */
 955                 assert(dv->gv_id == v->gv_restarter_id);
 956                 if (delete_restarter_dep)
 957                         graph_remove_edge(v, dv);
 958                 break;
 959 
 960         case GVT_GROUP: /* pg dependency */
 961                 graph_remove_edge(v, dv);
 962                 delete_depgroup(dv);
 963                 break;
 964 
 965         case GVT_FILE:
 966                 /* These are currently not direct dependencies */
 967 
 968         default:
 969 #ifndef NDEBUG
 970                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
 971                     dv->gv_type);
 972 #endif
 973                 abort();
 974         }
 975 
 976         return (UU_WALK_NEXT);
 977 }
 978 
 979 static void
 980 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
 981 {
 982         void *ptrs[2];
 983         int r;
 984 
 985         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 986         assert(v->gv_type == GVT_INST);
 987 
 988         ptrs[0] = v;
 989         ptrs[1] = (void *)delete_restarter_dep;
 990 
 991         r = uu_list_walk(v->gv_dependencies,
 992             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
 993         assert(r == 0);
 994 }
 995 
 996 /*
 997  * int graph_insert_vertex_unconfigured()
 998  *   Insert a vertex without sending any restarter events. If the vertex
 999  *   already exists or creation is successful, return a pointer to it in *vp.
1000  *
1001  *   If type is not GVT_GROUP, dt can remain unset.
1002  *
1003  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1004  *   doesn't agree with type, or type doesn't agree with dt).
1005  */
1006 static int
1007 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1008     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1009 {
1010         int r;
1011         int i;
1012 
1013         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1014 
1015         switch (type) {
1016         case GVT_SVC:
1017         case GVT_INST:
1018                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1019                         return (EINVAL);
1020                 break;
1021 
1022         case GVT_FILE:
1023                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1024                         return (EINVAL);
1025                 break;
1026 
1027         case GVT_GROUP:
1028                 if (dt <= 0 || rt < 0)
1029                         return (EINVAL);
1030                 break;
1031 
1032         default:
1033 #ifndef NDEBUG
1034                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1035 #endif
1036                 abort();
1037         }
1038 
1039         *vp = vertex_get_by_name(fmri);
1040         if (*vp != NULL)
1041                 return (EEXIST);
1042 
1043         *vp = graph_add_vertex(fmri);
1044 
1045         (*vp)->gv_type = type;
1046         (*vp)->gv_depgroup = dt;
1047         (*vp)->gv_restart = rt;
1048 
1049         (*vp)->gv_flags = 0;
1050         (*vp)->gv_state = RESTARTER_STATE_NONE;
1051 
1052         for (i = 0; special_vertices[i].name != NULL; ++i) {
1053                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1054                         (*vp)->gv_start_f = special_vertices[i].start_f;
1055                         (*vp)->gv_post_online_f =
1056                             special_vertices[i].post_online_f;
1057                         (*vp)->gv_post_disable_f =
1058                             special_vertices[i].post_disable_f;
1059                         break;
1060                 }
1061         }
1062 
1063         (*vp)->gv_restarter_id = -1;
1064         (*vp)->gv_restarter_channel = 0;
1065 
1066         if (type == GVT_INST) {
1067                 char *sfmri;
1068                 graph_vertex_t *sv;
1069 
1070                 sfmri = inst_fmri_to_svc_fmri(fmri);
1071                 sv = vertex_get_by_name(sfmri);
1072                 if (sv == NULL) {
1073                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1074                             0, &sv);
1075                         assert(r == 0);
1076                 }
1077                 startd_free(sfmri, max_scf_fmri_size);
1078 
1079                 graph_add_edge(sv, *vp);
1080         }
1081 
1082         /*
1083          * If this vertex is in the subgraph, mark it as so, for both
1084          * GVT_INST and GVT_SERVICE verteces.
1085          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1086          * depends on it, in which case it's already been added to the graph
1087          * and marked as in the subgraph (by refresh_vertex()).  If a
1088          * GVT_SERVICE vertex was freshly added (by the code above), it means
1089          * that it has no dependents, and cannot be in the subgraph.
1090          * Regardless of this, we still check that gv_flags includes
1091          * GV_INSUBGRAPH in the event that future behavior causes the above
1092          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1093          */
1094 
1095         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1096 
1097         return (0);
1098 }
1099 
1100 /*
1101  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1102  */
1103 static int
1104 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1105 {
1106         hrtime_t now;
1107 
1108         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1109 
1110         /* cycle detection */
1111         now = gethrtime();
1112 
1113         /* Don't follow exclusions. */
1114         if (!(fv->gv_type == GVT_GROUP &&
1115             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1116                 *pathp = is_path_to(tv, fv);
1117                 if (*pathp)
1118                         return (ELOOP);
1119         }
1120 
1121         dep_cycle_ns += gethrtime() - now;
1122         ++dep_inserts;
1123         now = gethrtime();
1124 
1125         graph_add_edge(fv, tv);
1126 
1127         dep_insert_ns += gethrtime() - now;
1128 
1129         /* Check if the dependency adds the "to" vertex to the subgraph */
1130         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1131 
1132         return (0);
1133 }
1134 
1135 static int
1136 inst_running(graph_vertex_t *v)
1137 {
1138         assert(v->gv_type == GVT_INST);
1139 
1140         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1141             v->gv_state == RESTARTER_STATE_DEGRADED)
1142                 return (1);
1143 
1144         return (0);
1145 }
1146 
1147 /*
1148  * The dependency evaluation functions return
1149  *   1 - dependency satisfied
1150  *   0 - dependency unsatisfied
1151  *   -1 - dependency unsatisfiable (without administrator intervention)
1152  *
1153  * The functions also take a boolean satbility argument.  When true, the
1154  * functions may recurse in order to determine satisfiability.
1155  */
1156 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1157 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1158 
1159 /*
1160  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1161  * is unsatisfiable if any elements are unsatisfiable.
1162  */
1163 static int
1164 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1165 {
1166         graph_edge_t *edge;
1167         int i;
1168         boolean_t any_unsatisfied;
1169 
1170         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1171                 return (1);
1172 
1173         any_unsatisfied = B_FALSE;
1174 
1175         for (edge = uu_list_first(groupv->gv_dependencies);
1176             edge != NULL;
1177             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1178                 i = dependency_satisfied(edge->ge_vertex, satbility);
1179                 if (i == 1)
1180                         continue;
1181 
1182                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1183                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1184                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1185 
1186                 if (!satbility)
1187                         return (0);
1188 
1189                 if (i == -1)
1190                         return (-1);
1191 
1192                 any_unsatisfied = B_TRUE;
1193         }
1194 
1195         return (any_unsatisfied ? 0 : 1);
1196 }
1197 
1198 /*
1199  * A require_any dependency is satisfied if any element is satisfied.  It is
1200  * satisfiable if any element is satisfiable.
1201  */
1202 static int
1203 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1204 {
1205         graph_edge_t *edge;
1206         int s;
1207         boolean_t satisfiable;
1208 
1209         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1210                 return (1);
1211 
1212         satisfiable = B_FALSE;
1213 
1214         for (edge = uu_list_first(groupv->gv_dependencies);
1215             edge != NULL;
1216             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1217                 s = dependency_satisfied(edge->ge_vertex, satbility);
1218 
1219                 if (s == 1)
1220                         return (1);
1221 
1222                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1223                     "require_any(%s): %s is unsatisfi%s.\n",
1224                     groupv->gv_name, edge->ge_vertex->gv_name,
1225                     s == 0 ? "ed" : "able");
1226 
1227                 if (satbility && s == 0)
1228                         satisfiable = B_TRUE;
1229         }
1230 
1231         return (!satbility || satisfiable ? 0 : -1);
1232 }
1233 
1234 /*
1235  * An optional_all dependency only considers elements which are configured,
1236  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1237  * is unsatisfied.
1238  *
1239  * Offline dependencies which are waiting for a dependency to come online are
1240  * unsatisfied.  Offline dependences which cannot possibly come online
1241  * (unsatisfiable) are always considered satisfied.
1242  */
1243 static int
1244 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1245 {
1246         graph_edge_t *edge;
1247         graph_vertex_t *v;
1248         boolean_t any_qualified;
1249         boolean_t any_unsatisfied;
1250         int i;
1251 
1252         any_qualified = B_FALSE;
1253         any_unsatisfied = B_FALSE;
1254 
1255         for (edge = uu_list_first(groupv->gv_dependencies);
1256             edge != NULL;
1257             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1258                 v = edge->ge_vertex;
1259 
1260                 switch (v->gv_type) {
1261                 case GVT_INST:
1262                         /* Skip missing or disabled instances */
1263                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1264                             (GV_CONFIGURED | GV_ENABLED))
1265                                 continue;
1266 
1267                         if (v->gv_state == RESTARTER_STATE_MAINT)
1268                                 continue;
1269 
1270                         if (v->gv_flags & GV_TOOFFLINE)
1271                                 continue;
1272 
1273                         any_qualified = B_TRUE;
1274                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1275                                 /*
1276                                  * For offline dependencies, treat unsatisfiable
1277                                  * as satisfied.
1278                                  */
1279                                 i = dependency_satisfied(v, B_TRUE);
1280                                 if (i == -1)
1281                                         i = 1;
1282                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1283                                 /*
1284                                  * The service is enabled, but hasn't
1285                                  * transitioned out of disabled yet.  Treat it
1286                                  * as unsatisfied (not unsatisfiable).
1287                                  */
1288                                 i = 0;
1289                         } else {
1290                                 i = dependency_satisfied(v, satbility);
1291                         }
1292                         break;
1293 
1294                 case GVT_FILE:
1295                         any_qualified = B_TRUE;
1296                         i = dependency_satisfied(v, satbility);
1297 
1298                         break;
1299 
1300                 case GVT_SVC: {
1301                         boolean_t svc_any_qualified;
1302                         boolean_t svc_satisfied;
1303                         boolean_t svc_satisfiable;
1304                         graph_vertex_t *v2;
1305                         graph_edge_t *e2;
1306 
1307                         svc_any_qualified = B_FALSE;
1308                         svc_satisfied = B_FALSE;
1309                         svc_satisfiable = B_FALSE;
1310 
1311                         for (e2 = uu_list_first(v->gv_dependencies);
1312                             e2 != NULL;
1313                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1314                                 v2 = e2->ge_vertex;
1315                                 assert(v2->gv_type == GVT_INST);
1316 
1317                                 if ((v2->gv_flags &
1318                                     (GV_CONFIGURED | GV_ENABLED)) !=
1319                                     (GV_CONFIGURED | GV_ENABLED))
1320                                         continue;
1321 
1322                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1323                                         continue;
1324 
1325                                 if (v2->gv_flags & GV_TOOFFLINE)
1326                                         continue;
1327 
1328                                 svc_any_qualified = B_TRUE;
1329 
1330                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1331                                         /*
1332                                          * For offline dependencies, treat
1333                                          * unsatisfiable as satisfied.
1334                                          */
1335                                         i = dependency_satisfied(v2, B_TRUE);
1336                                         if (i == -1)
1337                                                 i = 1;
1338                                 } else if (v2->gv_state ==
1339                                     RESTARTER_STATE_DISABLED) {
1340                                         i = 0;
1341                                 } else {
1342                                         i = dependency_satisfied(v2, satbility);
1343                                 }
1344 
1345                                 if (i == 1) {
1346                                         svc_satisfied = B_TRUE;
1347                                         break;
1348                                 }
1349                                 if (i == 0)
1350                                         svc_satisfiable = B_TRUE;
1351                         }
1352 
1353                         if (!svc_any_qualified)
1354                                 continue;
1355                         any_qualified = B_TRUE;
1356                         if (svc_satisfied) {
1357                                 i = 1;
1358                         } else if (svc_satisfiable) {
1359                                 i = 0;
1360                         } else {
1361                                 i = -1;
1362                         }
1363                         break;
1364                 }
1365 
1366                 case GVT_GROUP:
1367                 default:
1368 #ifndef NDEBUG
1369                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1370                             __LINE__, v->gv_type);
1371 #endif
1372                         abort();
1373                 }
1374 
1375                 if (i == 1)
1376                         continue;
1377 
1378                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1379                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1380                     v->gv_name, i == 0 ? "ed" : "able");
1381 
1382                 if (!satbility)
1383                         return (0);
1384                 if (i == -1)
1385                         return (-1);
1386                 any_unsatisfied = B_TRUE;
1387         }
1388 
1389         if (!any_qualified)
1390                 return (1);
1391 
1392         return (any_unsatisfied ? 0 : 1);
1393 }
1394 
1395 /*
1396  * An exclude_all dependency is unsatisfied if any non-service element is
1397  * satisfied or any service instance which is configured, enabled, and not in
1398  * maintenance is satisfied.  Usually when unsatisfied, it is also
1399  * unsatisfiable.
1400  */
1401 #define LOG_EXCLUDE(u, v)                                               \
1402         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1403             "exclude_all(%s): %s is satisfied.\n",                      \
1404             (u)->gv_name, (v)->gv_name)
1405 
1406 /* ARGSUSED */
1407 static int
1408 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1409 {
1410         graph_edge_t *edge, *e2;
1411         graph_vertex_t *v, *v2;
1412 
1413         for (edge = uu_list_first(groupv->gv_dependencies);
1414             edge != NULL;
1415             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1416                 v = edge->ge_vertex;
1417 
1418                 switch (v->gv_type) {
1419                 case GVT_INST:
1420                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1421                                 continue;
1422 
1423                         switch (v->gv_state) {
1424                         case RESTARTER_STATE_ONLINE:
1425                         case RESTARTER_STATE_DEGRADED:
1426                                 LOG_EXCLUDE(groupv, v);
1427                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1428 
1429                         case RESTARTER_STATE_OFFLINE:
1430                         case RESTARTER_STATE_UNINIT:
1431                                 LOG_EXCLUDE(groupv, v);
1432                                 return (0);
1433 
1434                         case RESTARTER_STATE_DISABLED:
1435                         case RESTARTER_STATE_MAINT:
1436                                 continue;
1437 
1438                         default:
1439 #ifndef NDEBUG
1440                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1441                                     __FILE__, __LINE__, v->gv_state);
1442 #endif
1443                                 abort();
1444                         }
1445                         /* NOTREACHED */
1446 
1447                 case GVT_SVC:
1448                         break;
1449 
1450                 case GVT_FILE:
1451                         if (!file_ready(v))
1452                                 continue;
1453                         LOG_EXCLUDE(groupv, v);
1454                         return (-1);
1455 
1456                 case GVT_GROUP:
1457                 default:
1458 #ifndef NDEBUG
1459                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1460                             __LINE__, v->gv_type);
1461 #endif
1462                         abort();
1463                 }
1464 
1465                 /* v represents a service */
1466                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1467                         continue;
1468 
1469                 for (e2 = uu_list_first(v->gv_dependencies);
1470                     e2 != NULL;
1471                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1472                         v2 = e2->ge_vertex;
1473                         assert(v2->gv_type == GVT_INST);
1474 
1475                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1476                                 continue;
1477 
1478                         switch (v2->gv_state) {
1479                         case RESTARTER_STATE_ONLINE:
1480                         case RESTARTER_STATE_DEGRADED:
1481                                 LOG_EXCLUDE(groupv, v2);
1482                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1483 
1484                         case RESTARTER_STATE_OFFLINE:
1485                         case RESTARTER_STATE_UNINIT:
1486                                 LOG_EXCLUDE(groupv, v2);
1487                                 return (0);
1488 
1489                         case RESTARTER_STATE_DISABLED:
1490                         case RESTARTER_STATE_MAINT:
1491                                 continue;
1492 
1493                         default:
1494 #ifndef NDEBUG
1495                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1496                                     __FILE__, __LINE__, v2->gv_type);
1497 #endif
1498                                 abort();
1499                         }
1500                 }
1501         }
1502 
1503         return (1);
1504 }
1505 
1506 /*
1507  * int instance_satisfied()
1508  *   Determine if all the dependencies are satisfied for the supplied instance
1509  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1510  *   without administrator intervention.
1511  */
1512 static int
1513 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1514 {
1515         assert(v->gv_type == GVT_INST);
1516         assert(!inst_running(v));
1517 
1518         return (require_all_satisfied(v, satbility));
1519 }
1520 
1521 /*
1522  * Decide whether v can satisfy a dependency.  v can either be a child of
1523  * a group vertex, or of an instance vertex.
1524  */
1525 static int
1526 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1527 {
1528         switch (v->gv_type) {
1529         case GVT_INST:
1530                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1531                         if (v->gv_flags & GV_DEATHROW) {
1532                                 /*
1533                                  * A dependency on an instance with GV_DEATHROW
1534                                  * flag is always considered as satisfied.
1535                                  */
1536                                 return (1);
1537                         }
1538                         return (-1);
1539                 }
1540 
1541                 /*
1542                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1543                  * to have its dependencies unsatisfiable.
1544                  */
1545                 if (v->gv_flags & GV_TOOFFLINE)
1546                         return (-1);
1547 
1548                 switch (v->gv_state) {
1549                 case RESTARTER_STATE_ONLINE:
1550                 case RESTARTER_STATE_DEGRADED:
1551                         return (1);
1552 
1553                 case RESTARTER_STATE_OFFLINE:
1554                         if (!satbility)
1555                                 return (0);
1556                         return (instance_satisfied(v, satbility) != -1 ?
1557                             0 : -1);
1558 
1559                 case RESTARTER_STATE_DISABLED:
1560                 case RESTARTER_STATE_MAINT:
1561                         return (-1);
1562 
1563                 case RESTARTER_STATE_UNINIT:
1564                         return (0);
1565 
1566                 default:
1567 #ifndef NDEBUG
1568                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1569                             __FILE__, __LINE__, v->gv_state);
1570 #endif
1571                         abort();
1572                         /* NOTREACHED */
1573                 }
1574 
1575         case GVT_SVC:
1576                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1577                         return (-1);
1578                 return (require_any_satisfied(v, satbility));
1579 
1580         case GVT_FILE:
1581                 /* i.e., we assume files will not be automatically generated */
1582                 return (file_ready(v) ? 1 : -1);
1583 
1584         case GVT_GROUP:
1585                 break;
1586 
1587         default:
1588 #ifndef NDEBUG
1589                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1590                     v->gv_type);
1591 #endif
1592                 abort();
1593                 /* NOTREACHED */
1594         }
1595 
1596         switch (v->gv_depgroup) {
1597         case DEPGRP_REQUIRE_ANY:
1598                 return (require_any_satisfied(v, satbility));
1599 
1600         case DEPGRP_REQUIRE_ALL:
1601                 return (require_all_satisfied(v, satbility));
1602 
1603         case DEPGRP_OPTIONAL_ALL:
1604                 return (optional_all_satisfied(v, satbility));
1605 
1606         case DEPGRP_EXCLUDE_ALL:
1607                 return (exclude_all_satisfied(v, satbility));
1608 
1609         default:
1610 #ifndef NDEBUG
1611                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1612                     __LINE__, v->gv_depgroup);
1613 #endif
1614                 abort();
1615         }
1616 }
1617 
1618 void
1619 graph_start_if_satisfied(graph_vertex_t *v)
1620 {
1621         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1622             instance_satisfied(v, B_FALSE) == 1) {
1623                 if (v->gv_start_f == NULL)
1624                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1625                 else
1626                         v->gv_start_f(v);
1627         }
1628 }
1629 
1630 /*
1631  * propagate_satbility()
1632  *
1633  * This function is used when the given vertex changes state in such a way that
1634  * one of its dependents may become unsatisfiable.  This happens when an
1635  * instance transitions between offline -> online, or from !running ->
1636  * maintenance, as well as when an instance is removed from the graph.
1637  *
1638  * We have to walk all the dependents, since optional_all dependencies several
1639  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1640  *
1641  *      +-----+  optional_all  +-----+  require_all  +-----+
1642  *      |  A  |--------------->|  B  |-------------->|  C  |
1643  *      +-----+                +-----+               +-----+
1644  *
1645  *                                              offline -> maintenance
1646  *
1647  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1648  * an optional dependency, what was previously an unsatisfiable situation is now
1649  * satisfied (B will never come online, even though its state hasn't changed).
1650  *
1651  * Note that it's not necessary to continue examining dependents after reaching
1652  * an optional_all dependency.  It's not possible for an optional_all dependency
1653  * to change satisfiability without also coming online, in which case we get a
1654  * start event and propagation continues naturally.  However, it does no harm to
1655  * continue propagating satisfiability (as it is a relatively rare event), and
1656  * keeps the walker code simple and generic.
1657  */
1658 /*ARGSUSED*/
1659 static int
1660 satbility_cb(graph_vertex_t *v, void *arg)
1661 {
1662         if (v->gv_type == GVT_INST)
1663                 graph_start_if_satisfied(v);
1664 
1665         return (UU_WALK_NEXT);
1666 }
1667 
1668 static void
1669 propagate_satbility(graph_vertex_t *v)
1670 {
1671         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1672 }
1673 
1674 static void propagate_stop(graph_vertex_t *, void *);
1675 
1676 /* ARGSUSED */
1677 static void
1678 propagate_start(graph_vertex_t *v, void *arg)
1679 {
1680         switch (v->gv_type) {
1681         case GVT_INST:
1682                 graph_start_if_satisfied(v);
1683                 break;
1684 
1685         case GVT_GROUP:
1686                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1687                         graph_walk_dependents(v, propagate_stop,
1688                             (void *)RERR_RESTART);
1689                         break;
1690                 }
1691                 /* FALLTHROUGH */
1692 
1693         case GVT_SVC:
1694                 graph_walk_dependents(v, propagate_start, NULL);
1695                 break;
1696 
1697         case GVT_FILE:
1698 #ifndef NDEBUG
1699                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1700                     __FILE__, __LINE__);
1701 #endif
1702                 abort();
1703                 /* NOTREACHED */
1704 
1705         default:
1706 #ifndef NDEBUG
1707                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1708                     v->gv_type);
1709 #endif
1710                 abort();
1711         }
1712 }
1713 
1714 static void
1715 propagate_stop(graph_vertex_t *v, void *arg)
1716 {
1717         graph_edge_t *e;
1718         graph_vertex_t *svc;
1719         restarter_error_t err = (restarter_error_t)arg;
1720 
1721         switch (v->gv_type) {
1722         case GVT_INST:
1723                 /* Restarter */
1724                 if (err > RERR_NONE && inst_running(v))
1725                         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1726                 break;
1727 
1728         case GVT_SVC:
1729                 graph_walk_dependents(v, propagate_stop, arg);
1730                 break;
1731 
1732         case GVT_FILE:
1733 #ifndef NDEBUG
1734                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1735                     __FILE__, __LINE__);
1736 #endif
1737                 abort();
1738                 /* NOTREACHED */
1739 
1740         case GVT_GROUP:
1741                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1742                         graph_walk_dependents(v, propagate_start, NULL);
1743                         break;
1744                 }
1745 
1746                 if (err == RERR_NONE || err > v->gv_restart)
1747                         break;
1748 
1749                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1750                 e = uu_list_first(v->gv_dependents);
1751                 svc = e->ge_vertex;
1752 
1753                 if (inst_running(svc))
1754                         vertex_send_event(svc, RESTARTER_EVENT_TYPE_STOP);
1755                 break;
1756 
1757         default:
1758 #ifndef NDEBUG
1759                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1760                     v->gv_type);
1761 #endif
1762                 abort();
1763         }
1764 }
1765 
1766 static void
1767 offline_vertex(graph_vertex_t *v)
1768 {
1769         scf_handle_t *h = libscf_handle_create_bound_loop();
1770         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1771         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1772         restarter_instance_state_t state, next_state;
1773         int r;
1774 
1775         assert(v->gv_type == GVT_INST);
1776 
1777         if (scf_inst == NULL)
1778                 bad_error("safe_scf_instance_create", scf_error());
1779         if (pg == NULL)
1780                 bad_error("safe_scf_pg_create", scf_error());
1781 
1782         /* if the vertex is already going offline, return */
1783 rep_retry:
1784         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1785             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1786                 switch (scf_error()) {
1787                 case SCF_ERROR_CONNECTION_BROKEN:
1788                         libscf_handle_rebind(h);
1789                         goto rep_retry;
1790 
1791                 case SCF_ERROR_NOT_FOUND:
1792                         scf_pg_destroy(pg);
1793                         scf_instance_destroy(scf_inst);
1794                         (void) scf_handle_unbind(h);
1795                         scf_handle_destroy(h);
1796                         return;
1797                 }
1798                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1799                     scf_strerror(scf_error()));
1800         }
1801 
1802         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1803         if (r != 0) {
1804                 switch (scf_error()) {
1805                 case SCF_ERROR_CONNECTION_BROKEN:
1806                         libscf_handle_rebind(h);
1807                         goto rep_retry;
1808 
1809                 case SCF_ERROR_NOT_SET:
1810                 case SCF_ERROR_NOT_FOUND:
1811                         scf_pg_destroy(pg);
1812                         scf_instance_destroy(scf_inst);
1813                         (void) scf_handle_unbind(h);
1814                         scf_handle_destroy(h);
1815                         return;
1816 
1817                 default:
1818                         bad_error("scf_instance_get_pg", scf_error());
1819                 }
1820         } else {
1821                 r = libscf_read_states(pg, &state, &next_state);
1822                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1823                     next_state == RESTARTER_STATE_DISABLED)) {
1824                         log_framework(LOG_DEBUG,
1825                             "%s: instance is already going down.\n",
1826                             v->gv_name);
1827                         scf_pg_destroy(pg);
1828                         scf_instance_destroy(scf_inst);
1829                         (void) scf_handle_unbind(h);
1830                         scf_handle_destroy(h);
1831                         return;
1832                 }
1833         }
1834 
1835         scf_pg_destroy(pg);
1836         scf_instance_destroy(scf_inst);
1837         (void) scf_handle_unbind(h);
1838         scf_handle_destroy(h);
1839 
1840         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1841 }
1842 
1843 /*
1844  * void graph_enable_by_vertex()
1845  *   If admin is non-zero, this is an administrative request for change
1846  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1847  *   a plain DISABLE restarter event.
1848  */
1849 void
1850 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1851 {
1852         graph_vertex_t *v;
1853         int r;
1854 
1855         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1856         assert((vertex->gv_flags & GV_CONFIGURED));
1857 
1858         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1859             (enable ? GV_ENABLED : 0);
1860 
1861         if (enable) {
1862                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1863                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1864                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1865                         /*
1866                          * In case the vertex was notified to go down,
1867                          * but now can return online, clear the _TOOFFLINE
1868                          * and _TODISABLE flags.
1869                          */
1870                         vertex->gv_flags &= ~GV_TOOFFLINE;
1871                         vertex->gv_flags &= ~GV_TODISABLE;
1872 
1873                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1874                 }
1875 
1876                 /*
1877                  * Wait for state update from restarter before sending _START or
1878                  * _STOP.
1879                  */
1880 
1881                 return;
1882         }
1883 
1884         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1885                 return;
1886 
1887         if (!admin) {
1888                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1889 
1890                 /*
1891                  * Wait for state update from restarter before sending _START or
1892                  * _STOP.
1893                  */
1894 
1895                 return;
1896         }
1897 
1898         /*
1899          * If it is a DISABLE event requested by the administrator then we are
1900          * offlining the dependents first.
1901          */
1902 
1903         /*
1904          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1905          * clear the GV_TOOFFLINE bits from all the services because
1906          * other DISABLE events might be handled at the same time.
1907          */
1908         vertex->gv_flags |= GV_TOOFFLINE;
1909 
1910         /* remember which vertex to disable... */
1911         vertex->gv_flags |= GV_TODISABLE;
1912 
1913         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1914             "disabling %s.\n", vertex->gv_name);
1915 
1916         /* set GV_TOOFFLINE for its dependents */
1917         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1918             NULL, 0);
1919         assert(r == 0);
1920 
1921         /* disable the instance now if there is nothing else to offline */
1922         if (insubtree_dependents_down(vertex) == B_TRUE) {
1923                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1924                 return;
1925         }
1926 
1927         /*
1928          * This loop is similar to the one used for the graph reversal shutdown
1929          * and could be improved in term of performance for the subtree reversal
1930          * disable case.
1931          */
1932         for (v = uu_list_first(dgraph); v != NULL;
1933             v = uu_list_next(dgraph, v)) {
1934                 /* skip the vertex we are disabling for now */
1935                 if (v == vertex)
1936                         continue;
1937 
1938                 if (v->gv_type != GVT_INST ||
1939                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1940                     (v->gv_flags & GV_ENABLED) == 0 ||
1941                     (v->gv_flags & GV_TOOFFLINE) == 0)
1942                         continue;
1943 
1944                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1945                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
1946                         /* continue if there is nothing to offline */
1947                         continue;
1948                 }
1949 
1950                 /*
1951                  * Instances which are up need to come down before we're
1952                  * done, but we can only offline the leaves here. An
1953                  * instance is a leaf when all its dependents are down.
1954                  */
1955                 if (insubtree_dependents_down(v) == B_TRUE) {
1956                         log_framework(LOG_DEBUG, "Offlining in-subtree "
1957                             "instance %s for %s.\n",
1958                             v->gv_name, vertex->gv_name);
1959                         offline_vertex(v);
1960                 }
1961         }
1962 }
1963 
1964 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
1965 
1966 /*
1967  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
1968  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
1969  * v is already configured and fmri_arg indicates the current restarter, do
1970  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
1971  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
1972  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
1973  * ECONNABORTED if the repository connection is broken, and ELOOP
1974  * if the dependency would create a cycle.  In the last case, *pathp will
1975  * point to a -1-terminated array of ids which compose the path from v to
1976  * restarter_fmri.
1977  */
1978 int
1979 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
1980     int **pathp)
1981 {
1982         char *restarter_fmri = NULL;
1983         graph_vertex_t *rv;
1984         int err;
1985         int id;
1986 
1987         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1988 
1989         if (fmri_arg[0] != '\0') {
1990                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
1991                 if (err != 0) {
1992                         assert(err == EINVAL);
1993                         return (err);
1994                 }
1995         }
1996 
1997         if (restarter_fmri == NULL ||
1998             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
1999                 if (v->gv_flags & GV_CONFIGURED) {
2000                         if (v->gv_restarter_id == -1) {
2001                                 if (restarter_fmri != NULL)
2002                                         startd_free(restarter_fmri,
2003                                             max_scf_fmri_size);
2004                                 return (0);
2005                         }
2006 
2007                         graph_unset_restarter(v);
2008                 }
2009 
2010                 /* Master restarter, nothing to do. */
2011                 v->gv_restarter_id = -1;
2012                 v->gv_restarter_channel = NULL;
2013                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2014                 return (0);
2015         }
2016 
2017         if (v->gv_flags & GV_CONFIGURED) {
2018                 id = dict_lookup_byname(restarter_fmri);
2019                 if (id != -1 && v->gv_restarter_id == id) {
2020                         startd_free(restarter_fmri, max_scf_fmri_size);
2021                         return (0);
2022                 }
2023 
2024                 graph_unset_restarter(v);
2025         }
2026 
2027         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2028             RERR_NONE, &rv);
2029         startd_free(restarter_fmri, max_scf_fmri_size);
2030         assert(err == 0 || err == EEXIST);
2031 
2032         if (rv->gv_delegate_initialized == 0) {
2033                 rv->gv_delegate_channel = restarter_protocol_init_delegate(
2034                     rv->gv_name);
2035                 rv->gv_delegate_initialized = 1;
2036         }
2037         v->gv_restarter_id = rv->gv_id;
2038         v->gv_restarter_channel = rv->gv_delegate_channel;
2039 
2040         err = graph_insert_dependency(v, rv, pathp);
2041         if (err != 0) {
2042                 assert(err == ELOOP);
2043                 return (ELOOP);
2044         }
2045 
2046         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2047 
2048         if (!(rv->gv_flags & GV_CONFIGURED)) {
2049                 scf_instance_t *inst;
2050 
2051                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2052                 switch (err) {
2053                 case 0:
2054                         err = configure_vertex(rv, inst);
2055                         scf_instance_destroy(inst);
2056                         switch (err) {
2057                         case 0:
2058                         case ECANCELED:
2059                                 break;
2060 
2061                         case ECONNABORTED:
2062                                 return (ECONNABORTED);
2063 
2064                         default:
2065                                 bad_error("configure_vertex", err);
2066                         }
2067                         break;
2068 
2069                 case ECONNABORTED:
2070                         return (ECONNABORTED);
2071 
2072                 case ENOENT:
2073                         break;
2074 
2075                 case ENOTSUP:
2076                         /*
2077                          * The fmri doesn't specify an instance - translate
2078                          * to EINVAL.
2079                          */
2080                         return (EINVAL);
2081 
2082                 case EINVAL:
2083                 default:
2084                         bad_error("libscf_fmri_get_instance", err);
2085                 }
2086         }
2087 
2088         return (0);
2089 }
2090 
2091 
2092 /*
2093  * Add all of the instances of the service named by fmri to the graph.
2094  * Returns
2095  *   0 - success
2096  *   ENOENT - service indicated by fmri does not exist
2097  *
2098  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2099  * otherwise.
2100  */
2101 static int
2102 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2103 {
2104         scf_service_t *svc;
2105         scf_instance_t *inst;
2106         scf_iter_t *iter;
2107         char *inst_fmri;
2108         int ret, r;
2109 
2110         *reboundp = B_FALSE;
2111 
2112         svc = safe_scf_service_create(h);
2113         inst = safe_scf_instance_create(h);
2114         iter = safe_scf_iter_create(h);
2115         inst_fmri = startd_alloc(max_scf_fmri_size);
2116 
2117 rebound:
2118         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2119             SCF_DECODE_FMRI_EXACT) != 0) {
2120                 switch (scf_error()) {
2121                 case SCF_ERROR_CONNECTION_BROKEN:
2122                 default:
2123                         libscf_handle_rebind(h);
2124                         *reboundp = B_TRUE;
2125                         goto rebound;
2126 
2127                 case SCF_ERROR_NOT_FOUND:
2128                         ret = ENOENT;
2129                         goto out;
2130 
2131                 case SCF_ERROR_INVALID_ARGUMENT:
2132                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2133                 case SCF_ERROR_NOT_BOUND:
2134                 case SCF_ERROR_HANDLE_MISMATCH:
2135                         bad_error("scf_handle_decode_fmri", scf_error());
2136                 }
2137         }
2138 
2139         if (scf_iter_service_instances(iter, svc) != 0) {
2140                 switch (scf_error()) {
2141                 case SCF_ERROR_CONNECTION_BROKEN:
2142                 default:
2143                         libscf_handle_rebind(h);
2144                         *reboundp = B_TRUE;
2145                         goto rebound;
2146 
2147                 case SCF_ERROR_DELETED:
2148                         ret = ENOENT;
2149                         goto out;
2150 
2151                 case SCF_ERROR_HANDLE_MISMATCH:
2152                 case SCF_ERROR_NOT_BOUND:
2153                 case SCF_ERROR_NOT_SET:
2154                         bad_error("scf_iter_service_instances", scf_error());
2155                 }
2156         }
2157 
2158         for (;;) {
2159                 r = scf_iter_next_instance(iter, inst);
2160                 if (r == 0)
2161                         break;
2162                 if (r != 1) {
2163                         switch (scf_error()) {
2164                         case SCF_ERROR_CONNECTION_BROKEN:
2165                         default:
2166                                 libscf_handle_rebind(h);
2167                                 *reboundp = B_TRUE;
2168                                 goto rebound;
2169 
2170                         case SCF_ERROR_DELETED:
2171                                 ret = ENOENT;
2172                                 goto out;
2173 
2174                         case SCF_ERROR_HANDLE_MISMATCH:
2175                         case SCF_ERROR_NOT_BOUND:
2176                         case SCF_ERROR_NOT_SET:
2177                         case SCF_ERROR_INVALID_ARGUMENT:
2178                                 bad_error("scf_iter_next_instance",
2179                                     scf_error());
2180                         }
2181                 }
2182 
2183                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2184                     0) {
2185                         switch (scf_error()) {
2186                         case SCF_ERROR_CONNECTION_BROKEN:
2187                                 libscf_handle_rebind(h);
2188                                 *reboundp = B_TRUE;
2189                                 goto rebound;
2190 
2191                         case SCF_ERROR_DELETED:
2192                                 continue;
2193 
2194                         case SCF_ERROR_NOT_BOUND:
2195                         case SCF_ERROR_NOT_SET:
2196                                 bad_error("scf_instance_to_fmri", scf_error());
2197                         }
2198                 }
2199 
2200                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2201                 switch (r) {
2202                 case 0:
2203                 case ECANCELED:
2204                         break;
2205 
2206                 case EEXIST:
2207                         continue;
2208 
2209                 case ECONNABORTED:
2210                         libscf_handle_rebind(h);
2211                         *reboundp = B_TRUE;
2212                         goto rebound;
2213 
2214                 case EINVAL:
2215                 default:
2216                         bad_error("dgraph_add_instance", r);
2217                 }
2218         }
2219 
2220         ret = 0;
2221 
2222 out:
2223         startd_free(inst_fmri, max_scf_fmri_size);
2224         scf_iter_destroy(iter);
2225         scf_instance_destroy(inst);
2226         scf_service_destroy(svc);
2227         return (ret);
2228 }
2229 
2230 struct depfmri_info {
2231         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2232         gv_type_t       type;           /* type of dependency */
2233         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2234         const char      *pg_name;       /* Name of dependency pg */
2235         scf_handle_t    *h;
2236         int             err;            /* return error code */
2237         int             **pathp;        /* return circular dependency path */
2238 };
2239 
2240 /*
2241  * Find or create a vertex for fmri and make info->v depend on it.
2242  * Returns
2243  *   0 - success
2244  *   nonzero - failure
2245  *
2246  * On failure, sets info->err to
2247  *   EINVAL - fmri is invalid
2248  *            fmri does not match info->type
2249  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2250  *           will point to an array of the ids of the members of the cycle.
2251  *   ECONNABORTED - repository connection was broken
2252  *   ECONNRESET - succeeded, but repository connection was reset
2253  */
2254 static int
2255 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2256 {
2257         int err;
2258         graph_vertex_t *depgroup_v, *v;
2259         char *fmri_copy, *cfmri;
2260         size_t fmri_copy_sz;
2261         const char *scope, *service, *instance, *pg;
2262         scf_instance_t *inst;
2263         boolean_t rebound;
2264 
2265         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2266 
2267         /* Get or create vertex for FMRI */
2268         depgroup_v = info->v;
2269 
2270         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2271                 if (info->type != GVT_FILE) {
2272                         log_framework(LOG_NOTICE,
2273                             "FMRI \"%s\" is not allowed for the \"%s\" "
2274                             "dependency's type of instance %s.\n", fmri,
2275                             info->pg_name, info->inst_fmri);
2276                         return (info->err = EINVAL);
2277                 }
2278 
2279                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2280                     RERR_NONE, &v);
2281                 switch (err) {
2282                 case 0:
2283                         break;
2284 
2285                 case EEXIST:
2286                         assert(v->gv_type == GVT_FILE);
2287                         break;
2288 
2289                 case EINVAL:            /* prevented above */
2290                 default:
2291                         bad_error("graph_insert_vertex_unconfigured", err);
2292                 }
2293         } else {
2294                 if (info->type != GVT_INST) {
2295                         log_framework(LOG_NOTICE,
2296                             "FMRI \"%s\" is not allowed for the \"%s\" "
2297                             "dependency's type of instance %s.\n", fmri,
2298                             info->pg_name, info->inst_fmri);
2299                         return (info->err = EINVAL);
2300                 }
2301 
2302                 /*
2303                  * We must canonify fmri & add a vertex for it.
2304                  */
2305                 fmri_copy_sz = strlen(fmri) + 1;
2306                 fmri_copy = startd_alloc(fmri_copy_sz);
2307                 (void) strcpy(fmri_copy, fmri);
2308 
2309                 /* Determine if the FMRI is a property group or instance */
2310                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2311                     &instance, &pg, NULL) != 0) {
2312                         startd_free(fmri_copy, fmri_copy_sz);
2313                         log_framework(LOG_NOTICE,
2314                             "Dependency \"%s\" of %s has invalid FMRI "
2315                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2316                             fmri);
2317                         return (info->err = EINVAL);
2318                 }
2319 
2320                 if (service == NULL || pg != NULL) {
2321                         startd_free(fmri_copy, fmri_copy_sz);
2322                         log_framework(LOG_NOTICE,
2323                             "Dependency \"%s\" of %s does not designate a "
2324                             "service or instance.\n", info->pg_name,
2325                             info->inst_fmri);
2326                         return (info->err = EINVAL);
2327                 }
2328 
2329                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2330                         cfmri = uu_msprintf("svc:/%s%s%s",
2331                             service, instance ? ":" : "", instance ? instance :
2332                             "");
2333                 } else {
2334                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2335                             scope, service, instance ? ":" : "", instance ?
2336                             instance : "");
2337                 }
2338 
2339                 startd_free(fmri_copy, fmri_copy_sz);
2340 
2341                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2342                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2343                     RERR_NONE, &v);
2344                 uu_free(cfmri);
2345                 switch (err) {
2346                 case 0:
2347                         break;
2348 
2349                 case EEXIST:
2350                         /* Verify v. */
2351                         if (instance != NULL)
2352                                 assert(v->gv_type == GVT_INST);
2353                         else
2354                                 assert(v->gv_type == GVT_SVC);
2355                         break;
2356 
2357                 default:
2358                         bad_error("graph_insert_vertex_unconfigured", err);
2359                 }
2360         }
2361 
2362         /* Add dependency from depgroup_v to new vertex */
2363         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2364         switch (info->err) {
2365         case 0:
2366                 break;
2367 
2368         case ELOOP:
2369                 return (ELOOP);
2370 
2371         default:
2372                 bad_error("graph_insert_dependency", info->err);
2373         }
2374 
2375         /* This must be after we insert the dependency, to avoid looping. */
2376         switch (v->gv_type) {
2377         case GVT_INST:
2378                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2379                         break;
2380 
2381                 inst = safe_scf_instance_create(info->h);
2382 
2383                 rebound = B_FALSE;
2384 
2385 rebound:
2386                 err = libscf_lookup_instance(v->gv_name, inst);
2387                 switch (err) {
2388                 case 0:
2389                         err = configure_vertex(v, inst);
2390                         switch (err) {
2391                         case 0:
2392                         case ECANCELED:
2393                                 break;
2394 
2395                         case ECONNABORTED:
2396                                 libscf_handle_rebind(info->h);
2397                                 rebound = B_TRUE;
2398                                 goto rebound;
2399 
2400                         default:
2401                                 bad_error("configure_vertex", err);
2402                         }
2403                         break;
2404 
2405                 case ENOENT:
2406                         break;
2407 
2408                 case ECONNABORTED:
2409                         libscf_handle_rebind(info->h);
2410                         rebound = B_TRUE;
2411                         goto rebound;
2412 
2413                 case EINVAL:
2414                 case ENOTSUP:
2415                 default:
2416                         bad_error("libscf_fmri_get_instance", err);
2417                 }
2418 
2419                 scf_instance_destroy(inst);
2420 
2421                 if (rebound)
2422                         return (info->err = ECONNRESET);
2423                 break;
2424 
2425         case GVT_SVC:
2426                 (void) add_service(v->gv_name, info->h, &rebound);
2427                 if (rebound)
2428                         return (info->err = ECONNRESET);
2429         }
2430 
2431         return (0);
2432 }
2433 
2434 struct deppg_info {
2435         graph_vertex_t  *v;             /* GVT_INST vertex */
2436         int             err;            /* return error */
2437         int             **pathp;        /* return circular dependency path */
2438 };
2439 
2440 /*
2441  * Make info->v depend on a new GVT_GROUP node for this property group,
2442  * and then call process_dependency_fmri() for the values of the entity
2443  * property.  Return 0 on success, or if something goes wrong return nonzero
2444  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2445  * process_dependency_fmri().
2446  */
2447 static int
2448 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2449 {
2450         scf_handle_t *h;
2451         depgroup_type_t deptype;
2452         restarter_error_t rerr;
2453         struct depfmri_info linfo;
2454         char *fmri, *pg_name;
2455         size_t fmri_sz;
2456         graph_vertex_t *depgrp;
2457         scf_property_t *prop;
2458         int err;
2459         int empty;
2460         scf_error_t scferr;
2461         ssize_t len;
2462 
2463         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2464 
2465         h = scf_pg_handle(pg);
2466 
2467         pg_name = startd_alloc(max_scf_name_size);
2468 
2469         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2470         if (len < 0) {
2471                 startd_free(pg_name, max_scf_name_size);
2472                 switch (scf_error()) {
2473                 case SCF_ERROR_CONNECTION_BROKEN:
2474                 default:
2475                         return (info->err = ECONNABORTED);
2476 
2477                 case SCF_ERROR_DELETED:
2478                         return (info->err = 0);
2479 
2480                 case SCF_ERROR_NOT_SET:
2481                         bad_error("scf_pg_get_name", scf_error());
2482                 }
2483         }
2484 
2485         /*
2486          * Skip over empty dependency groups.  Since dependency property
2487          * groups are updated atomically, they are either empty or
2488          * fully populated.
2489          */
2490         empty = depgroup_empty(h, pg);
2491         if (empty < 0) {
2492                 log_error(LOG_INFO,
2493                     "Error reading dependency group \"%s\" of %s: %s\n",
2494                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2495                 startd_free(pg_name, max_scf_name_size);
2496                 return (info->err = EINVAL);
2497 
2498         } else if (empty == 1) {
2499                 log_framework(LOG_DEBUG,
2500                     "Ignoring empty dependency group \"%s\" of %s\n",
2501                     pg_name, info->v->gv_name);
2502                 startd_free(pg_name, max_scf_name_size);
2503                 return (info->err = 0);
2504         }
2505 
2506         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2507         fmri = startd_alloc(fmri_sz);
2508 
2509         (void) snprintf(fmri, max_scf_name_size, "%s>%s", info->v->gv_name,
2510             pg_name);
2511 
2512         /* Validate the pg before modifying the graph */
2513         deptype = depgroup_read_grouping(h, pg);
2514         if (deptype == DEPGRP_UNSUPPORTED) {
2515                 log_error(LOG_INFO,
2516                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2517                     pg_name, info->v->gv_name);
2518                 startd_free(fmri, fmri_sz);
2519                 startd_free(pg_name, max_scf_name_size);
2520                 return (info->err = EINVAL);
2521         }
2522 
2523         rerr = depgroup_read_restart(h, pg);
2524         if (rerr == RERR_UNSUPPORTED) {
2525                 log_error(LOG_INFO,
2526                     "Dependency \"%s\" of %s has an unknown restart_on value."
2527                     "\n", pg_name, info->v->gv_name);
2528                 startd_free(fmri, fmri_sz);
2529                 startd_free(pg_name, max_scf_name_size);
2530                 return (info->err = EINVAL);
2531         }
2532 
2533         prop = safe_scf_property_create(h);
2534 
2535         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2536                 scferr = scf_error();
2537                 scf_property_destroy(prop);
2538                 if (scferr == SCF_ERROR_DELETED) {
2539                         startd_free(fmri, fmri_sz);
2540                         startd_free(pg_name, max_scf_name_size);
2541                         return (info->err = 0);
2542                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2543                         startd_free(fmri, fmri_sz);
2544                         startd_free(pg_name, max_scf_name_size);
2545                         return (info->err = ECONNABORTED);
2546                 }
2547 
2548                 log_error(LOG_INFO,
2549                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2550                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2551 
2552                 startd_free(fmri, fmri_sz);
2553                 startd_free(pg_name, max_scf_name_size);
2554 
2555                 return (info->err = EINVAL);
2556         }
2557 
2558         /* Create depgroup vertex for pg */
2559         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2560             rerr, &depgrp);
2561         assert(err == 0);
2562         startd_free(fmri, fmri_sz);
2563 
2564         /* Add dependency from inst vertex to new vertex */
2565         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2566         /* ELOOP can't happen because this should be a new vertex */
2567         assert(err == 0);
2568 
2569         linfo.v = depgrp;
2570         linfo.type = depgroup_read_scheme(h, pg);
2571         linfo.inst_fmri = info->v->gv_name;
2572         linfo.pg_name = pg_name;
2573         linfo.h = h;
2574         linfo.err = 0;
2575         linfo.pathp = info->pathp;
2576         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2577             &linfo);
2578 
2579         scf_property_destroy(prop);
2580         startd_free(pg_name, max_scf_name_size);
2581 
2582         switch (err) {
2583         case 0:
2584         case EINTR:
2585                 return (info->err = linfo.err);
2586 
2587         case ECONNABORTED:
2588         case EINVAL:
2589                 return (info->err = err);
2590 
2591         case ECANCELED:
2592                 return (info->err = 0);
2593 
2594         case ECONNRESET:
2595                 return (info->err = ECONNABORTED);
2596 
2597         default:
2598                 bad_error("walk_property_astrings", err);
2599                 /* NOTREACHED */
2600         }
2601 }
2602 
2603 /*
2604  * Build the dependency info for v from the repository.  Returns 0 on success,
2605  * ECONNABORTED on repository disconnection, EINVAL if the repository
2606  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2607  * In the last case, *pathp will point to a -1-terminated array of ids which
2608  * constitute the rest of the dependency cycle.
2609  */
2610 static int
2611 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2612 {
2613         struct deppg_info info;
2614         int err;
2615         uint_t old_configured;
2616 
2617         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2618 
2619         /*
2620          * Mark the vertex as configured during dependency insertion to avoid
2621          * dependency cycles (which can appear in the graph if one of the
2622          * vertices is an exclusion-group).
2623          */
2624         old_configured = v->gv_flags & GV_CONFIGURED;
2625         v->gv_flags |= GV_CONFIGURED;
2626 
2627         info.err = 0;
2628         info.v = v;
2629         info.pathp = pathp;
2630 
2631         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2632             &info);
2633 
2634         if (!old_configured)
2635                 v->gv_flags &= ~GV_CONFIGURED;
2636 
2637         switch (err) {
2638         case 0:
2639         case EINTR:
2640                 return (info.err);
2641 
2642         case ECONNABORTED:
2643                 return (ECONNABORTED);
2644 
2645         case ECANCELED:
2646                 /* Should get delete event, so return 0. */
2647                 return (0);
2648 
2649         default:
2650                 bad_error("walk_dependency_pgs", err);
2651                 /* NOTREACHED */
2652         }
2653 }
2654 
2655 
2656 static void
2657 handle_cycle(const char *fmri, int *path)
2658 {
2659         const char *cp;
2660         size_t sz;
2661 
2662         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2663 
2664         path_to_str(path, (char **)&cp, &sz);
2665 
2666         log_error(LOG_ERR, "Transitioning %s to maintenance "
2667             "because it completes a dependency cycle (see svcs -xv for "
2668             "details):\n%s", fmri ? fmri : "?", cp);
2669 
2670         startd_free((void *)cp, sz);
2671 }
2672 
2673 /*
2674  * Increment the vertex's reference count to prevent the vertex removal
2675  * from the dgraph.
2676  */
2677 static void
2678 vertex_ref(graph_vertex_t *v)
2679 {
2680         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2681 
2682         v->gv_refs++;
2683 }
2684 
2685 /*
2686  * Decrement the vertex's reference count and remove the vertex from
2687  * the dgraph when possible.
2688  *
2689  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2690  * return VERTEX_INUSE.
2691  */
2692 static int
2693 vertex_unref(graph_vertex_t *v)
2694 {
2695         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2696         assert(v->gv_refs > 0);
2697 
2698         v->gv_refs--;
2699 
2700         return (free_if_unrefed(v));
2701 }
2702 
2703 /*
2704  * When run on the dependencies of a vertex, populates list with
2705  * graph_edge_t's which point to the service vertices or the instance
2706  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2707  *
2708  * Increment the vertex's reference count once the vertex is inserted
2709  * in the list. The vertex won't be able to be deleted from the dgraph
2710  * while it is referenced.
2711  */
2712 static int
2713 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2714 {
2715         graph_vertex_t *v = e->ge_vertex;
2716         graph_edge_t *new;
2717         int r;
2718 
2719         switch (v->gv_type) {
2720         case GVT_INST:
2721         case GVT_SVC:
2722                 break;
2723 
2724         case GVT_GROUP:
2725                 r = uu_list_walk(v->gv_dependencies,
2726                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2727                 assert(r == 0);
2728                 return (UU_WALK_NEXT);
2729 
2730         case GVT_FILE:
2731                 return (UU_WALK_NEXT);
2732 
2733         default:
2734 #ifndef NDEBUG
2735                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2736                     __LINE__, v->gv_type);
2737 #endif
2738                 abort();
2739         }
2740 
2741         new = startd_alloc(sizeof (*new));
2742         new->ge_vertex = v;
2743         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2744         r = uu_list_insert_before(list, NULL, new);
2745         assert(r == 0);
2746 
2747         /*
2748          * Because we are inserting the vertex in a list, we don't want
2749          * the vertex to be freed while the list is in use. In order to
2750          * achieve that, increment the vertex's reference count.
2751          */
2752         vertex_ref(v);
2753 
2754         return (UU_WALK_NEXT);
2755 }
2756 
2757 static boolean_t
2758 should_be_in_subgraph(graph_vertex_t *v)
2759 {
2760         graph_edge_t *e;
2761 
2762         if (v == milestone)
2763                 return (B_TRUE);
2764 
2765         /*
2766          * v is in the subgraph if any of its dependents are in the subgraph.
2767          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2768          * count if we're enabled.
2769          */
2770         for (e = uu_list_first(v->gv_dependents);
2771             e != NULL;
2772             e = uu_list_next(v->gv_dependents, e)) {
2773                 graph_vertex_t *dv = e->ge_vertex;
2774 
2775                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2776                         continue;
2777 
2778                 /*
2779                  * Don't include instances that are optional and disabled.
2780                  */
2781                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2782 
2783                         int in = 0;
2784                         graph_edge_t *ee;
2785 
2786                         for (ee = uu_list_first(dv->gv_dependents);
2787                             ee != NULL;
2788                             ee = uu_list_next(dv->gv_dependents, ee)) {
2789 
2790                                 graph_vertex_t *ddv = e->ge_vertex;
2791 
2792                                 if (ddv->gv_type == GVT_GROUP &&
2793                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2794                                         continue;
2795 
2796                                 if (ddv->gv_type == GVT_GROUP &&
2797                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2798                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2799                                         continue;
2800 
2801                                 in = 1;
2802                         }
2803                         if (!in)
2804                                 continue;
2805                 }
2806                 if (v->gv_type == GVT_INST &&
2807                     dv->gv_type == GVT_GROUP &&
2808                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2809                     !(v->gv_flags & GV_ENBLD_NOOVR))
2810                         continue;
2811 
2812                 /* Don't include excluded services and instances */
2813                 if (dv->gv_type == GVT_GROUP &&
2814                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2815                         continue;
2816 
2817                 return (B_TRUE);
2818         }
2819 
2820         return (B_FALSE);
2821 }
2822 
2823 /*
2824  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2825  * any bits change, manipulate the repository appropriately.  Returns 0 or
2826  * ECONNABORTED.
2827  */
2828 static int
2829 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2830 {
2831         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2832         boolean_t new;
2833         graph_edge_t *e;
2834         scf_instance_t *inst;
2835         int ret = 0, r;
2836 
2837         assert(milestone != NULL && milestone != MILESTONE_NONE);
2838 
2839         new = should_be_in_subgraph(v);
2840 
2841         if (new == old)
2842                 return (0);
2843 
2844         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2845             "Removing %s from the subgraph.\n", v->gv_name);
2846 
2847         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2848             (new ? GV_INSUBGRAPH : 0);
2849 
2850         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2851                 int err;
2852 
2853 get_inst:
2854                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2855                 if (err != 0) {
2856                         switch (err) {
2857                         case ECONNABORTED:
2858                                 libscf_handle_rebind(h);
2859                                 ret = ECONNABORTED;
2860                                 goto get_inst;
2861 
2862                         case ENOENT:
2863                                 break;
2864 
2865                         case EINVAL:
2866                         case ENOTSUP:
2867                         default:
2868                                 bad_error("libscf_fmri_get_instance", err);
2869                         }
2870                 } else {
2871                         const char *f;
2872 
2873                         if (new) {
2874                                 err = libscf_delete_enable_ovr(inst);
2875                                 f = "libscf_delete_enable_ovr";
2876                         } else {
2877                                 err = libscf_set_enable_ovr(inst, 0);
2878                                 f = "libscf_set_enable_ovr";
2879                         }
2880                         scf_instance_destroy(inst);
2881                         switch (err) {
2882                         case 0:
2883                         case ECANCELED:
2884                                 break;
2885 
2886                         case ECONNABORTED:
2887                                 libscf_handle_rebind(h);
2888                                 /*
2889                                  * We must continue so the graph is updated,
2890                                  * but we must return ECONNABORTED so any
2891                                  * libscf state held by any callers is reset.
2892                                  */
2893                                 ret = ECONNABORTED;
2894                                 goto get_inst;
2895 
2896                         case EROFS:
2897                         case EPERM:
2898                                 log_error(LOG_WARNING,
2899                                     "Could not set %s/%s for %s: %s.\n",
2900                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2901                                     v->gv_name, strerror(err));
2902                                 break;
2903 
2904                         default:
2905                                 bad_error(f, err);
2906                         }
2907                 }
2908         }
2909 
2910         for (e = uu_list_first(v->gv_dependencies);
2911             e != NULL;
2912             e = uu_list_next(v->gv_dependencies, e)) {
2913                 r = eval_subgraph(e->ge_vertex, h);
2914                 if (r != 0) {
2915                         assert(r == ECONNABORTED);
2916                         ret = ECONNABORTED;
2917                 }
2918         }
2919 
2920         return (ret);
2921 }
2922 
2923 /*
2924  * Delete the (property group) dependencies of v & create new ones based on
2925  * inst.  If doing so would create a cycle, log a message and put the instance
2926  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2927  * ECONNABORTED.
2928  */
2929 int
2930 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2931 {
2932         int err;
2933         int *path;
2934         char *fmri;
2935         int r;
2936         scf_handle_t *h = scf_instance_handle(inst);
2937         uu_list_t *old_deps;
2938         int ret = 0;
2939         graph_edge_t *e;
2940         graph_vertex_t *vv;
2941 
2942         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2943         assert(v->gv_type == GVT_INST);
2944 
2945         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
2946 
2947         if (milestone > MILESTONE_NONE) {
2948                 /*
2949                  * In case some of v's dependencies are being deleted we must
2950                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
2951                  * after the new dependencies are in place.
2952                  */
2953                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
2954 
2955                 err = uu_list_walk(v->gv_dependencies,
2956                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
2957                 assert(err == 0);
2958         }
2959 
2960         delete_instance_dependencies(v, B_FALSE);
2961 
2962         err = set_dependencies(v, inst, &path);
2963         switch (err) {
2964         case 0:
2965                 break;
2966 
2967         case ECONNABORTED:
2968                 ret = err;
2969                 goto out;
2970 
2971         case EINVAL:
2972         case ELOOP:
2973                 r = libscf_instance_get_fmri(inst, &fmri);
2974                 switch (r) {
2975                 case 0:
2976                         break;
2977 
2978                 case ECONNABORTED:
2979                         ret = ECONNABORTED;
2980                         goto out;
2981 
2982                 case ECANCELED:
2983                         ret = 0;
2984                         goto out;
2985 
2986                 default:
2987                         bad_error("libscf_instance_get_fmri", r);
2988                 }
2989 
2990                 if (err == EINVAL) {
2991                         log_error(LOG_ERR, "Transitioning %s "
2992                             "to maintenance due to misconfiguration.\n",
2993                             fmri ? fmri : "?");
2994                         vertex_send_event(v,
2995                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
2996                 } else {
2997                         handle_cycle(fmri, path);
2998                         vertex_send_event(v,
2999                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3000                 }
3001                 startd_free(fmri, max_scf_fmri_size);
3002                 ret = 0;
3003                 goto out;
3004 
3005         default:
3006                 bad_error("set_dependencies", err);
3007         }
3008 
3009         if (milestone > MILESTONE_NONE) {
3010                 boolean_t aborted = B_FALSE;
3011 
3012                 for (e = uu_list_first(old_deps);
3013                     e != NULL;
3014                     e = uu_list_next(old_deps, e)) {
3015                         vv = e->ge_vertex;
3016 
3017                         if (vertex_unref(vv) == VERTEX_INUSE &&
3018                             eval_subgraph(vv, h) == ECONNABORTED)
3019                                 aborted = B_TRUE;
3020                 }
3021 
3022                 for (e = uu_list_first(v->gv_dependencies);
3023                     e != NULL;
3024                     e = uu_list_next(v->gv_dependencies, e)) {
3025                         if (eval_subgraph(e->ge_vertex, h) ==
3026                             ECONNABORTED)
3027                                 aborted = B_TRUE;
3028                 }
3029 
3030                 if (aborted) {
3031                         ret = ECONNABORTED;
3032                         goto out;
3033                 }
3034         }
3035 
3036         graph_start_if_satisfied(v);
3037 
3038         ret = 0;
3039 
3040 out:
3041         if (milestone > MILESTONE_NONE) {
3042                 void *cookie = NULL;
3043 
3044                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3045                         startd_free(e, sizeof (*e));
3046 
3047                 uu_list_destroy(old_deps);
3048         }
3049 
3050         return (ret);
3051 }
3052 
3053 /*
3054  * Set up v according to inst.  That is, make sure it depends on its
3055  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3056  * the restarter, and send ENABLE or DISABLE as appropriate.
3057  *
3058  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3059  * ECANCELED if inst is deleted.
3060  */
3061 static int
3062 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3063 {
3064         scf_handle_t *h;
3065         scf_propertygroup_t *pg;
3066         scf_snapshot_t *snap;
3067         char *restarter_fmri = startd_alloc(max_scf_value_size);
3068         int enabled, enabled_ovr;
3069         int err;
3070         int *path;
3071         int deathrow;
3072 
3073         restarter_fmri[0] = '\0';
3074 
3075         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3076         assert(v->gv_type == GVT_INST);
3077         assert((v->gv_flags & GV_CONFIGURED) == 0);
3078 
3079         /* GV_INSUBGRAPH should already be set properly. */
3080         assert(should_be_in_subgraph(v) ==
3081             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3082 
3083         /*
3084          * If the instance fmri is in the deathrow list then set the
3085          * GV_DEATHROW flag on the vertex and create and set to true the
3086          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3087          * repository for this instance fmri.
3088          */
3089         if ((v->gv_flags & GV_DEATHROW) ||
3090             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3091                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3092                         /*
3093                          * Set flag GV_DEATHROW, create and set to true
3094                          * the SCF_PROPERTY_DEATHROW property in the
3095                          * non-persistent repository for this instance fmri.
3096                          */
3097                         v->gv_flags |= GV_DEATHROW;
3098 
3099                         switch (err = libscf_set_deathrow(inst, 1)) {
3100                         case 0:
3101                                 break;
3102 
3103                         case ECONNABORTED:
3104                         case ECANCELED:
3105                                 startd_free(restarter_fmri, max_scf_value_size);
3106                                 return (err);
3107 
3108                         case EROFS:
3109                                 log_error(LOG_WARNING, "Could not set %s/%s "
3110                                     "for deathrow %s: %s.\n",
3111                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3112                                     v->gv_name, strerror(err));
3113                                 break;
3114 
3115                         case EPERM:
3116                                 uu_die("Permission denied.\n");
3117                                 /* NOTREACHED */
3118 
3119                         default:
3120                                 bad_error("libscf_set_deathrow", err);
3121                         }
3122                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3123                             v->gv_name);
3124                 }
3125                 startd_free(restarter_fmri, max_scf_value_size);
3126                 return (0);
3127         }
3128 
3129         h = scf_instance_handle(inst);
3130 
3131         /*
3132          * Using a temporary deathrow boolean property, set through
3133          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3134          * because deathrow_fini() may already have been called, and in case
3135          * of a refresh, GV_DEATHROW may need to be set again.
3136          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3137          * has a temporary boolean property named 'deathrow' valued true
3138          * in a property group 'deathrow', -1 or 0 in all other cases.
3139          */
3140         err = libscf_get_deathrow(h, inst, &deathrow);
3141         switch (err) {
3142         case 0:
3143                 break;
3144 
3145         case ECONNABORTED:
3146         case ECANCELED:
3147                 startd_free(restarter_fmri, max_scf_value_size);
3148                 return (err);
3149 
3150         default:
3151                 bad_error("libscf_get_deathrow", err);
3152         }
3153 
3154         if (deathrow == 1) {
3155                 v->gv_flags |= GV_DEATHROW;
3156                 startd_free(restarter_fmri, max_scf_value_size);
3157                 return (0);
3158         }
3159 
3160         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3161 
3162         /*
3163          * If the instance does not have a restarter property group,
3164          * initialize its state to uninitialized/none, in case the restarter
3165          * is not enabled.
3166          */
3167         pg = safe_scf_pg_create(h);
3168 
3169         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3170                 instance_data_t idata;
3171                 uint_t count = 0, msecs = ALLOC_DELAY;
3172 
3173                 switch (scf_error()) {
3174                 case SCF_ERROR_NOT_FOUND:
3175                         break;
3176 
3177                 case SCF_ERROR_CONNECTION_BROKEN:
3178                 default:
3179                         scf_pg_destroy(pg);
3180                         return (ECONNABORTED);
3181 
3182                 case SCF_ERROR_DELETED:
3183                         scf_pg_destroy(pg);
3184                         return (ECANCELED);
3185 
3186                 case SCF_ERROR_NOT_SET:
3187                         bad_error("scf_instance_get_pg", scf_error());
3188                 }
3189 
3190                 switch (err = libscf_instance_get_fmri(inst,
3191                     (char **)&idata.i_fmri)) {
3192                 case 0:
3193                         break;
3194 
3195                 case ECONNABORTED:
3196                 case ECANCELED:
3197                         scf_pg_destroy(pg);
3198                         return (err);
3199 
3200                 default:
3201                         bad_error("libscf_instance_get_fmri", err);
3202                 }
3203 
3204                 idata.i_state = RESTARTER_STATE_NONE;
3205                 idata.i_next_state = RESTARTER_STATE_NONE;
3206 
3207 init_state:
3208                 switch (err = _restarter_commit_states(h, &idata,
3209                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE, NULL)) {
3210                 case 0:
3211                         break;
3212 
3213                 case ENOMEM:
3214                         ++count;
3215                         if (count < ALLOC_RETRY) {
3216                                 (void) poll(NULL, 0, msecs);
3217                                 msecs *= ALLOC_DELAY_MULT;
3218                                 goto init_state;
3219                         }
3220 
3221                         uu_die("Insufficient memory.\n");
3222                         /* NOTREACHED */
3223 
3224                 case ECONNABORTED:
3225                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3226                         scf_pg_destroy(pg);
3227                         return (ECONNABORTED);
3228 
3229                 case ENOENT:
3230                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3231                         scf_pg_destroy(pg);
3232                         return (ECANCELED);
3233 
3234                 case EPERM:
3235                 case EACCES:
3236                 case EROFS:
3237                         log_error(LOG_NOTICE, "Could not initialize state for "
3238                             "%s: %s.\n", idata.i_fmri, strerror(err));
3239                         break;
3240 
3241                 case EINVAL:
3242                 default:
3243                         bad_error("_restarter_commit_states", err);
3244                 }
3245 
3246                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3247         }
3248 
3249         scf_pg_destroy(pg);
3250 
3251         if (milestone != NULL) {
3252                 /*
3253                  * Make sure the enable-override is set properly before we
3254                  * read whether we should be enabled.
3255                  */
3256                 if (milestone == MILESTONE_NONE ||
3257                     !(v->gv_flags & GV_INSUBGRAPH)) {
3258                         /*
3259                          * This might seem unjustified after the milestone
3260                          * transition has completed (non_subgraph_svcs == 0),
3261                          * but it's important because when we boot to
3262                          * a milestone, we set the milestone before populating
3263                          * the graph, and all of the new non-subgraph services
3264                          * need to be disabled here.
3265                          */
3266                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3267                         case 0:
3268                                 break;
3269 
3270                         case ECONNABORTED:
3271                         case ECANCELED:
3272                                 return (err);
3273 
3274                         case EROFS:
3275                                 log_error(LOG_WARNING,
3276                                     "Could not set %s/%s for %s: %s.\n",
3277                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3278                                     v->gv_name, strerror(err));
3279                                 break;
3280 
3281                         case EPERM:
3282                                 uu_die("Permission denied.\n");
3283                                 /* NOTREACHED */
3284 
3285                         default:
3286                                 bad_error("libscf_set_enable_ovr", err);
3287                         }
3288                 } else {
3289                         assert(v->gv_flags & GV_INSUBGRAPH);
3290                         switch (err = libscf_delete_enable_ovr(inst)) {
3291                         case 0:
3292                                 break;
3293 
3294                         case ECONNABORTED:
3295                         case ECANCELED:
3296                                 return (err);
3297 
3298                         case EPERM:
3299                                 uu_die("Permission denied.\n");
3300                                 /* NOTREACHED */
3301 
3302                         default:
3303                                 bad_error("libscf_delete_enable_ovr", err);
3304                         }
3305                 }
3306         }
3307 
3308         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3309             &enabled_ovr, &restarter_fmri);
3310         switch (err) {
3311         case 0:
3312                 break;
3313 
3314         case ECONNABORTED:
3315         case ECANCELED:
3316                 startd_free(restarter_fmri, max_scf_value_size);
3317                 return (err);
3318 
3319         case ENOENT:
3320                 log_framework(LOG_DEBUG,
3321                     "Ignoring %s because it has no general property group.\n",
3322                     v->gv_name);
3323                 startd_free(restarter_fmri, max_scf_value_size);
3324                 return (0);
3325 
3326         default:
3327                 bad_error("libscf_get_basic_instance_data", err);
3328         }
3329 
3330         if (enabled == -1) {
3331                 startd_free(restarter_fmri, max_scf_value_size);
3332                 return (0);
3333         }
3334 
3335         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3336             (enabled ? GV_ENBLD_NOOVR : 0);
3337 
3338         if (enabled_ovr != -1)
3339                 enabled = enabled_ovr;
3340 
3341         v->gv_state = RESTARTER_STATE_UNINIT;
3342 
3343         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3344         scf_snapshot_destroy(snap);
3345 
3346         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3347         err = graph_change_restarter(v, restarter_fmri, h, &path);
3348         if (err != 0) {
3349                 instance_data_t idata;
3350                 uint_t count = 0, msecs = ALLOC_DELAY;
3351                 const char *reason;
3352 
3353                 if (err == ECONNABORTED) {
3354                         startd_free(restarter_fmri, max_scf_value_size);
3355                         return (err);
3356                 }
3357 
3358                 assert(err == EINVAL || err == ELOOP);
3359 
3360                 if (err == EINVAL) {
3361                         log_framework(LOG_ERR, emsg_invalid_restarter,
3362                             v->gv_name);
3363                         reason = "invalid_restarter";
3364                 } else {
3365                         handle_cycle(v->gv_name, path);
3366                         reason = "dependency_cycle";
3367                 }
3368 
3369                 startd_free(restarter_fmri, max_scf_value_size);
3370 
3371                 /*
3372                  * We didn't register the instance with the restarter, so we
3373                  * must set maintenance mode ourselves.
3374                  */
3375                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3376                 if (err != 0) {
3377                         assert(err == ECONNABORTED || err == ECANCELED);
3378                         return (err);
3379                 }
3380 
3381                 idata.i_state = RESTARTER_STATE_NONE;
3382                 idata.i_next_state = RESTARTER_STATE_NONE;
3383 
3384 set_maint:
3385                 switch (err = _restarter_commit_states(h, &idata,
3386                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, reason)) {
3387                 case 0:
3388                         break;
3389 
3390                 case ENOMEM:
3391                         ++count;
3392                         if (count < ALLOC_RETRY) {
3393                                 (void) poll(NULL, 0, msecs);
3394                                 msecs *= ALLOC_DELAY_MULT;
3395                                 goto set_maint;
3396                         }
3397 
3398                         uu_die("Insufficient memory.\n");
3399                         /* NOTREACHED */
3400 
3401                 case ECONNABORTED:
3402                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3403                         return (ECONNABORTED);
3404 
3405                 case ENOENT:
3406                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3407                         return (ECANCELED);
3408 
3409                 case EPERM:
3410                 case EACCES:
3411                 case EROFS:
3412                         log_error(LOG_NOTICE, "Could not initialize state for "
3413                             "%s: %s.\n", idata.i_fmri, strerror(err));
3414                         break;
3415 
3416                 case EINVAL:
3417                 default:
3418                         bad_error("_restarter_commit_states", err);
3419                 }
3420 
3421                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3422 
3423                 v->gv_state = RESTARTER_STATE_MAINT;
3424 
3425                 goto out;
3426         }
3427         startd_free(restarter_fmri, max_scf_value_size);
3428 
3429         /* Add all the other dependencies. */
3430         err = refresh_vertex(v, inst);
3431         if (err != 0) {
3432                 assert(err == ECONNABORTED);
3433                 return (err);
3434         }
3435 
3436 out:
3437         v->gv_flags |= GV_CONFIGURED;
3438 
3439         graph_enable_by_vertex(v, enabled, 0);
3440 
3441         return (0);
3442 }
3443 
3444 static void
3445 do_uadmin(void)
3446 {
3447         int fd, left;
3448         struct statvfs vfs;
3449 
3450         const char * const resetting = "/etc/svc/volatile/resetting";
3451 
3452         fd = creat(resetting, 0777);
3453         if (fd >= 0)
3454                 startd_close(fd);
3455         else
3456                 uu_warn("Could not create \"%s\"", resetting);
3457 
3458         /* Kill dhcpagent if we're not using nfs for root */
3459         if ((statvfs("/", &vfs) == 0) &&
3460             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3461                 (void) system("/usr/bin/pkill -x -u 0 dhcpagent");
3462 
3463         (void) system("/usr/sbin/killall");
3464         left = 5;
3465         while (left > 0)
3466                 left = sleep(left);
3467 
3468         (void) system("/usr/sbin/killall 9");
3469         left = 10;
3470         while (left > 0)
3471                 left = sleep(left);
3472 
3473         sync();
3474         sync();
3475         sync();
3476 
3477         (void) system("/sbin/umountall -l");
3478         (void) system("/sbin/umount /tmp >/dev/null 2>&1");
3479         (void) system("/sbin/umount /var/adm >/dev/null 2>&1");
3480         (void) system("/sbin/umount /var/run >/dev/null 2>&1");
3481         (void) system("/sbin/umount /var >/dev/null 2>&1");
3482         (void) system("/sbin/umount /usr >/dev/null 2>&1");
3483 
3484         uu_warn("The system is down.\n");
3485 
3486         (void) uadmin(A_SHUTDOWN, halting, NULL);
3487         uu_warn("uadmin() failed");
3488 
3489         if (remove(resetting) != 0 && errno != ENOENT)
3490                 uu_warn("Could not remove \"%s\"", resetting);
3491 }
3492 
3493 /*
3494  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3495  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3496  */
3497 boolean_t
3498 can_come_up(void)
3499 {
3500         int i;
3501 
3502         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3503 
3504         /*
3505          * If we are booting to single user (boot -s),
3506          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3507          * spawns sulogin after single-user is online (see specials.c).
3508          */
3509         i = (booting_to_single_user ? 0 : 1);
3510 
3511         for (; up_svcs[i] != NULL; ++i) {
3512                 if (up_svcs_p[i] == NULL) {
3513                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3514 
3515                         if (up_svcs_p[i] == NULL)
3516                                 continue;
3517                 }
3518 
3519                 /*
3520                  * Ignore unconfigured services (the ones that have been
3521                  * mentioned in a dependency from other services, but do
3522                  * not exist in the repository).  Services which exist
3523                  * in the repository but don't have general/enabled
3524                  * property will be also ignored.
3525                  */
3526                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3527                         continue;
3528 
3529                 switch (up_svcs_p[i]->gv_state) {
3530                 case RESTARTER_STATE_ONLINE:
3531                 case RESTARTER_STATE_DEGRADED:
3532                         /*
3533                          * Deactivate verbose boot once a login service has been
3534                          * reached.
3535                          */
3536                         st->st_log_login_reached = 1;
3537                         /*FALLTHROUGH*/
3538                 case RESTARTER_STATE_UNINIT:
3539                         return (B_TRUE);
3540 
3541                 case RESTARTER_STATE_OFFLINE:
3542                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3543                                 return (B_TRUE);
3544                         log_framework(LOG_DEBUG,
3545                             "can_come_up(): %s is unsatisfiable.\n",
3546                             up_svcs_p[i]->gv_name);
3547                         continue;
3548 
3549                 case RESTARTER_STATE_DISABLED:
3550                 case RESTARTER_STATE_MAINT:
3551                         log_framework(LOG_DEBUG,
3552                             "can_come_up(): %s is in state %s.\n",
3553                             up_svcs_p[i]->gv_name,
3554                             instance_state_str[up_svcs_p[i]->gv_state]);
3555                         continue;
3556 
3557                 default:
3558 #ifndef NDEBUG
3559                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3560                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3561 #endif
3562                         abort();
3563                 }
3564         }
3565 
3566         /*
3567          * In the seed repository, console-login is unsatisfiable because
3568          * services are missing.  To behave correctly in that case we don't want
3569          * to return false until manifest-import is online.
3570          */
3571 
3572         if (manifest_import_p == NULL) {
3573                 manifest_import_p = vertex_get_by_name(manifest_import);
3574 
3575                 if (manifest_import_p == NULL)
3576                         return (B_FALSE);
3577         }
3578 
3579         switch (manifest_import_p->gv_state) {
3580         case RESTARTER_STATE_ONLINE:
3581         case RESTARTER_STATE_DEGRADED:
3582         case RESTARTER_STATE_DISABLED:
3583         case RESTARTER_STATE_MAINT:
3584                 break;
3585 
3586         case RESTARTER_STATE_OFFLINE:
3587                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3588                         break;
3589                 /* FALLTHROUGH */
3590 
3591         case RESTARTER_STATE_UNINIT:
3592                 return (B_TRUE);
3593         }
3594 
3595         return (B_FALSE);
3596 }
3597 
3598 /*
3599  * Runs sulogin.  Returns
3600  *   0 - success
3601  *   EALREADY - sulogin is already running
3602  *   EBUSY - console-login is running
3603  */
3604 static int
3605 run_sulogin(const char *msg)
3606 {
3607         graph_vertex_t *v;
3608 
3609         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3610 
3611         if (sulogin_running)
3612                 return (EALREADY);
3613 
3614         v = vertex_get_by_name(console_login_fmri);
3615         if (v != NULL && inst_running(v))
3616                 return (EBUSY);
3617 
3618         sulogin_running = B_TRUE;
3619 
3620         MUTEX_UNLOCK(&dgraph_lock);
3621 
3622         fork_sulogin(B_FALSE, msg);
3623 
3624         MUTEX_LOCK(&dgraph_lock);
3625 
3626         sulogin_running = B_FALSE;
3627 
3628         if (console_login_ready) {
3629                 v = vertex_get_by_name(console_login_fmri);
3630 
3631                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE &&
3632                     !inst_running(v)) {
3633                         if (v->gv_start_f == NULL)
3634                                 vertex_send_event(v,
3635                                     RESTARTER_EVENT_TYPE_START);
3636                         else
3637                                 v->gv_start_f(v);
3638                 }
3639 
3640                 console_login_ready = B_FALSE;
3641         }
3642 
3643         return (0);
3644 }
3645 
3646 /*
3647  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3648  * keeps sulogin from stepping on console-login's toes.
3649  */
3650 /* ARGSUSED */
3651 static void *
3652 sulogin_thread(void *unused)
3653 {
3654         MUTEX_LOCK(&dgraph_lock);
3655 
3656         assert(sulogin_thread_running);
3657 
3658         do {
3659                 (void) run_sulogin("Console login service(s) cannot run\n");
3660         } while (!can_come_up());
3661 
3662         sulogin_thread_running = B_FALSE;
3663         MUTEX_UNLOCK(&dgraph_lock);
3664 
3665         return (NULL);
3666 }
3667 
3668 /* ARGSUSED */
3669 void *
3670 single_user_thread(void *unused)
3671 {
3672         uint_t left;
3673         scf_handle_t *h;
3674         scf_instance_t *inst;
3675         scf_property_t *prop;
3676         scf_value_t *val;
3677         const char *msg;
3678         char *buf;
3679         int r;
3680 
3681         MUTEX_LOCK(&single_user_thread_lock);
3682         single_user_thread_count++;
3683 
3684         if (!booting_to_single_user) {
3685                 /*
3686                  * From rcS.sh: Look for ttymon, in.telnetd, in.rlogind and
3687                  * processes in their process groups so they can be terminated.
3688                  */
3689                 (void) fputs("svc.startd: Killing user processes: ", stdout);
3690                 (void) system("/usr/sbin/killall");
3691                 (void) system("/usr/sbin/killall 9");
3692                 (void) system("/usr/bin/pkill -TERM -v -u 0,1");
3693 
3694                 left = 5;
3695                 while (left > 0)
3696                         left = sleep(left);
3697 
3698                 (void) system("/usr/bin/pkill -KILL -v -u 0,1");
3699                 (void) puts("done.");
3700         }
3701 
3702         if (go_single_user_mode || booting_to_single_user) {
3703                 msg = "SINGLE USER MODE\n";
3704         } else {
3705                 assert(go_to_level1);
3706 
3707                 fork_rc_script('1', "start", B_TRUE);
3708 
3709                 uu_warn("The system is ready for administration.\n");
3710 
3711                 msg = "";
3712         }
3713 
3714         MUTEX_UNLOCK(&single_user_thread_lock);
3715 
3716         for (;;) {
3717                 MUTEX_LOCK(&dgraph_lock);
3718                 r = run_sulogin(msg);
3719                 MUTEX_UNLOCK(&dgraph_lock);
3720                 if (r == 0)
3721                         break;
3722 
3723                 assert(r == EALREADY || r == EBUSY);
3724 
3725                 left = 3;
3726                 while (left > 0)
3727                         left = sleep(left);
3728         }
3729 
3730         MUTEX_LOCK(&single_user_thread_lock);
3731 
3732         /*
3733          * If another single user thread has started, let it finish changing
3734          * the run level.
3735          */
3736         if (single_user_thread_count > 1) {
3737                 single_user_thread_count--;
3738                 MUTEX_UNLOCK(&single_user_thread_lock);
3739                 return (NULL);
3740         }
3741 
3742         h = libscf_handle_create_bound_loop();
3743         inst = scf_instance_create(h);
3744         prop = safe_scf_property_create(h);
3745         val = safe_scf_value_create(h);
3746         buf = startd_alloc(max_scf_fmri_size);
3747 
3748 lookup:
3749         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3750             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3751                 switch (scf_error()) {
3752                 case SCF_ERROR_NOT_FOUND:
3753                         r = libscf_create_self(h);
3754                         if (r == 0)
3755                                 goto lookup;
3756                         assert(r == ECONNABORTED);
3757                         /* FALLTHROUGH */
3758 
3759                 case SCF_ERROR_CONNECTION_BROKEN:
3760                         libscf_handle_rebind(h);
3761                         goto lookup;
3762 
3763                 case SCF_ERROR_INVALID_ARGUMENT:
3764                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3765                 case SCF_ERROR_NOT_BOUND:
3766                 case SCF_ERROR_HANDLE_MISMATCH:
3767                 default:
3768                         bad_error("scf_handle_decode_fmri", scf_error());
3769                 }
3770         }
3771 
3772         MUTEX_LOCK(&dgraph_lock);
3773 
3774         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3775             SCF_PROPERTY_MILESTONE);
3776         switch (r) {
3777         case 0:
3778         case ECANCELED:
3779                 break;
3780 
3781         case ECONNABORTED:
3782                 MUTEX_UNLOCK(&dgraph_lock);
3783                 libscf_handle_rebind(h);
3784                 goto lookup;
3785 
3786         case EPERM:
3787         case EACCES:
3788         case EROFS:
3789                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3790                     "%s.\n", strerror(r));
3791                 break;
3792 
3793         default:
3794                 bad_error("scf_instance_delete_prop", r);
3795         }
3796 
3797         MUTEX_UNLOCK(&dgraph_lock);
3798 
3799         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3800         switch (r) {
3801         case ECANCELED:
3802         case ENOENT:
3803         case EINVAL:
3804                 (void) strcpy(buf, "all");
3805                 /* FALLTHROUGH */
3806 
3807         case 0:
3808                 uu_warn("Returning to milestone %s.\n", buf);
3809                 break;
3810 
3811         case ECONNABORTED:
3812                 libscf_handle_rebind(h);
3813                 goto lookup;
3814 
3815         default:
3816                 bad_error("libscf_get_milestone", r);
3817         }
3818 
3819         r = dgraph_set_milestone(buf, h, B_FALSE);
3820         switch (r) {
3821         case 0:
3822         case ECONNRESET:
3823         case EALREADY:
3824         case EINVAL:
3825         case ENOENT:
3826                 break;
3827 
3828         default:
3829                 bad_error("dgraph_set_milestone", r);
3830         }
3831 
3832         /*
3833          * See graph_runlevel_changed().
3834          */
3835         MUTEX_LOCK(&dgraph_lock);
3836         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
3837         MUTEX_UNLOCK(&dgraph_lock);
3838 
3839         startd_free(buf, max_scf_fmri_size);
3840         scf_value_destroy(val);
3841         scf_property_destroy(prop);
3842         scf_instance_destroy(inst);
3843         scf_handle_destroy(h);
3844 
3845         /*
3846          * We'll give ourselves 3 seconds to respond to all of the enablings
3847          * that setting the milestone should have created before checking
3848          * whether to run sulogin.
3849          */
3850         left = 3;
3851         while (left > 0)
3852                 left = sleep(left);
3853 
3854         MUTEX_LOCK(&dgraph_lock);
3855         /*
3856          * Clearing these variables will allow the sulogin thread to run.  We
3857          * check here in case there aren't any more state updates anytime soon.
3858          */
3859         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
3860         if (!sulogin_thread_running && !can_come_up()) {
3861                 (void) startd_thread_create(sulogin_thread, NULL);
3862                 sulogin_thread_running = B_TRUE;
3863         }
3864         MUTEX_UNLOCK(&dgraph_lock);
3865         single_user_thread_count--;
3866         MUTEX_UNLOCK(&single_user_thread_lock);
3867         return (NULL);
3868 }
3869 
3870 
3871 /*
3872  * Dependency graph operations API.  These are handle-independent thread-safe
3873  * graph manipulation functions which are the entry points for the event
3874  * threads below.
3875  */
3876 
3877 /*
3878  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
3879  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
3880  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
3881  * Fetch whether the instance should be enabled from inst and send _ENABLE or
3882  * _DISABLE as appropriate.  Finally rummage through inst's dependency
3883  * property groups and add vertices and edges as appropriate.  If anything
3884  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
3885  * instance in maintenance.  Don't send _START or _STOP until we get a state
3886  * update in case we're being restarted and the service is already running.
3887  *
3888  * To support booting to a milestone, we must also make sure all dependencies
3889  * encountered are configured, if they exist in the repository.
3890  *
3891  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
3892  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
3893  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
3894  */
3895 int
3896 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
3897     boolean_t lock_graph)
3898 {
3899         graph_vertex_t *v;
3900         int err;
3901 
3902         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
3903                 return (0);
3904 
3905         /* Check for a vertex for inst_fmri. */
3906         if (lock_graph) {
3907                 MUTEX_LOCK(&dgraph_lock);
3908         } else {
3909                 assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3910         }
3911 
3912         v = vertex_get_by_name(inst_fmri);
3913 
3914         if (v != NULL) {
3915                 assert(v->gv_type == GVT_INST);
3916 
3917                 if (v->gv_flags & GV_CONFIGURED) {
3918                         if (lock_graph)
3919                                 MUTEX_UNLOCK(&dgraph_lock);
3920                         return (EEXIST);
3921                 }
3922         } else {
3923                 /* Add the vertex. */
3924                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
3925                     RERR_NONE, &v);
3926                 if (err != 0) {
3927                         assert(err == EINVAL);
3928                         if (lock_graph)
3929                                 MUTEX_UNLOCK(&dgraph_lock);
3930                         return (EINVAL);
3931                 }
3932         }
3933 
3934         err = configure_vertex(v, inst);
3935 
3936         if (lock_graph)
3937                 MUTEX_UNLOCK(&dgraph_lock);
3938 
3939         return (err);
3940 }
3941 
3942 /*
3943  * Locate the vertex for this property group's instance.  If it doesn't exist
3944  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
3945  * the restarter for the instance, and if it has changed, send
3946  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
3947  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
3948  * the new restarter.  Then fetch whether the instance should be enabled, and
3949  * if it is different from what we had, or if we changed the restarter, send
3950  * the appropriate _ENABLE or _DISABLE command.
3951  *
3952  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
3953  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
3954  * deleted, or -1 if the instance's general property group is deleted or if
3955  * its enabled property is misconfigured.
3956  */
3957 static int
3958 dgraph_update_general(scf_propertygroup_t *pg)
3959 {
3960         scf_handle_t *h;
3961         scf_instance_t *inst;
3962         char *fmri;
3963         char *restarter_fmri;
3964         graph_vertex_t *v;
3965         int err;
3966         int enabled, enabled_ovr;
3967         int oldflags;
3968 
3969         /* Find the vertex for this service */
3970         h = scf_pg_handle(pg);
3971 
3972         inst = safe_scf_instance_create(h);
3973 
3974         if (scf_pg_get_parent_instance(pg, inst) != 0) {
3975                 switch (scf_error()) {
3976                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3977                         return (ENOTSUP);
3978 
3979                 case SCF_ERROR_CONNECTION_BROKEN:
3980                 default:
3981                         return (ECONNABORTED);
3982 
3983                 case SCF_ERROR_DELETED:
3984                         return (0);
3985 
3986                 case SCF_ERROR_NOT_SET:
3987                         bad_error("scf_pg_get_parent_instance", scf_error());
3988                 }
3989         }
3990 
3991         err = libscf_instance_get_fmri(inst, &fmri);
3992         switch (err) {
3993         case 0:
3994                 break;
3995 
3996         case ECONNABORTED:
3997                 scf_instance_destroy(inst);
3998                 return (ECONNABORTED);
3999 
4000         case ECANCELED:
4001                 scf_instance_destroy(inst);
4002                 return (0);
4003 
4004         default:
4005                 bad_error("libscf_instance_get_fmri", err);
4006         }
4007 
4008         log_framework(LOG_DEBUG,
4009             "Graph engine: Reloading general properties for %s.\n", fmri);
4010 
4011         MUTEX_LOCK(&dgraph_lock);
4012 
4013         v = vertex_get_by_name(fmri);
4014         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4015                 /* Will get the up-to-date properties. */
4016                 MUTEX_UNLOCK(&dgraph_lock);
4017                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4018                 startd_free(fmri, max_scf_fmri_size);
4019                 scf_instance_destroy(inst);
4020                 return (err == ECANCELED ? 0 : err);
4021         }
4022 
4023         /* Read enabled & restarter from repository. */
4024         restarter_fmri = startd_alloc(max_scf_value_size);
4025         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4026             &enabled_ovr, &restarter_fmri);
4027         if (err != 0 || enabled == -1) {
4028                 MUTEX_UNLOCK(&dgraph_lock);
4029                 scf_instance_destroy(inst);
4030                 startd_free(fmri, max_scf_fmri_size);
4031 
4032                 switch (err) {
4033                 case ENOENT:
4034                 case 0:
4035                         startd_free(restarter_fmri, max_scf_value_size);
4036                         return (-1);
4037 
4038                 case ECONNABORTED:
4039                 case ECANCELED:
4040                         startd_free(restarter_fmri, max_scf_value_size);
4041                         return (err);
4042 
4043                 default:
4044                         bad_error("libscf_get_basic_instance_data", err);
4045                 }
4046         }
4047 
4048         oldflags = v->gv_flags;
4049         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4050             (enabled ? GV_ENBLD_NOOVR : 0);
4051 
4052         if (enabled_ovr != -1)
4053                 enabled = enabled_ovr;
4054 
4055         /*
4056          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4057          * subgraph.
4058          */
4059         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4060                 (void) eval_subgraph(v, h);
4061 
4062         scf_instance_destroy(inst);
4063 
4064         /* Ignore restarter change for now. */
4065 
4066         startd_free(restarter_fmri, max_scf_value_size);
4067         startd_free(fmri, max_scf_fmri_size);
4068 
4069         /*
4070          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4071          * restarter didn't change and the enabled value didn't change, but
4072          * that's not easy to check and improbable anyway, so we'll just do
4073          * this.
4074          */
4075         graph_enable_by_vertex(v, enabled, 1);
4076 
4077         MUTEX_UNLOCK(&dgraph_lock);
4078 
4079         return (0);
4080 }
4081 
4082 /*
4083  * Delete all of the property group dependencies of v, update inst's running
4084  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4085  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4086  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4087  * the same for v's dependents.
4088  *
4089  * Returns
4090  *   0 - success
4091  *   ECONNABORTED - repository connection broken
4092  *   ECANCELED - inst was deleted
4093  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4094  *   -1 - libscf_snapshots_refresh() failed
4095  */
4096 static int
4097 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4098 {
4099         int r;
4100         int enabled;
4101 
4102         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4103         assert(v->gv_type == GVT_INST);
4104 
4105         /* Only refresh services with valid general/enabled properties. */
4106         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4107             v->gv_name, &enabled, NULL, NULL);
4108         switch (r) {
4109         case 0:
4110                 break;
4111 
4112         case ECONNABORTED:
4113         case ECANCELED:
4114                 return (r);
4115 
4116         case ENOENT:
4117                 log_framework(LOG_DEBUG,
4118                     "Ignoring %s because it has no general property group.\n",
4119                     v->gv_name);
4120                 return (EINVAL);
4121 
4122         default:
4123                 bad_error("libscf_get_basic_instance_data", r);
4124         }
4125 
4126         if (enabled == -1)
4127                 return (EINVAL);
4128 
4129         r = libscf_snapshots_refresh(inst, v->gv_name);
4130         if (r != 0) {
4131                 if (r != -1)
4132                         bad_error("libscf_snapshots_refresh", r);
4133 
4134                 /* error logged */
4135                 return (r);
4136         }
4137 
4138         r = refresh_vertex(v, inst);
4139         if (r != 0 && r != ECONNABORTED)
4140                 bad_error("refresh_vertex", r);
4141         return (r);
4142 }
4143 
4144 /*
4145  * Returns true only if none of this service's dependents are 'up' -- online
4146  * or degraded (offline is considered down in this situation). This function
4147  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4148  */
4149 static boolean_t
4150 insubtree_dependents_down(graph_vertex_t *v)
4151 {
4152         graph_vertex_t *vv;
4153         graph_edge_t *e;
4154 
4155         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4156 
4157         for (e = uu_list_first(v->gv_dependents); e != NULL;
4158             e = uu_list_next(v->gv_dependents, e)) {
4159                 vv = e->ge_vertex;
4160                 if (vv->gv_type == GVT_INST) {
4161                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4162                                 continue;
4163 
4164                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4165                                 continue;
4166 
4167                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4168                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4169                                 return (B_FALSE);
4170                 } else {
4171                         /*
4172                          * For dependency groups or service vertices, keep
4173                          * traversing to see if instances are running.
4174                          */
4175                         if (insubtree_dependents_down(vv) == B_FALSE)
4176                                 return (B_FALSE);
4177                 }
4178         }
4179 
4180         return (B_TRUE);
4181 }
4182 
4183 /*
4184  * Returns true only if none of this service's dependents are 'up' -- online,
4185  * degraded, or offline.
4186  */
4187 static int
4188 is_nonsubgraph_leaf(graph_vertex_t *v)
4189 {
4190         graph_vertex_t *vv;
4191         graph_edge_t *e;
4192 
4193         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4194 
4195         for (e = uu_list_first(v->gv_dependents);
4196             e != NULL;
4197             e = uu_list_next(v->gv_dependents, e)) {
4198 
4199                 vv = e->ge_vertex;
4200                 if (vv->gv_type == GVT_INST) {
4201                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4202                                 continue;
4203 
4204                         if (vv->gv_flags & GV_INSUBGRAPH)
4205                                 continue;
4206 
4207                         if (up_state(vv->gv_state))
4208                                 return (0);
4209                 } else {
4210                         /*
4211                          * For dependency group or service vertices, keep
4212                          * traversing to see if instances are running.
4213                          *
4214                          * We should skip exclude_all dependencies otherwise
4215                          * the vertex will never be considered as a leaf
4216                          * if the dependent is offline. The main reason for
4217                          * this is that disable_nonsubgraph_leaves() skips
4218                          * exclusion dependencies.
4219                          */
4220                         if (vv->gv_type == GVT_GROUP &&
4221                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4222                                 continue;
4223 
4224                         if (!is_nonsubgraph_leaf(vv))
4225                                 return (0);
4226                 }
4227         }
4228 
4229         return (1);
4230 }
4231 
4232 /*
4233  * Disable v temporarily.  Attempt to do this by setting its enabled override
4234  * property in the repository.  If that fails, send a _DISABLE command.
4235  * Returns 0 on success and ECONNABORTED if the repository connection is
4236  * broken.
4237  */
4238 static int
4239 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4240 {
4241         const char * const emsg = "Could not temporarily disable %s because "
4242             "%s.  Will stop service anyways.  Repository status for the "
4243             "service may be inaccurate.\n";
4244         const char * const emsg_cbroken =
4245             "the repository connection was broken";
4246 
4247         scf_instance_t *inst;
4248         int r;
4249 
4250         inst = scf_instance_create(h);
4251         if (inst == NULL) {
4252                 char buf[100];
4253 
4254                 (void) snprintf(buf, sizeof (buf),
4255                     "scf_instance_create() failed (%s)",
4256                     scf_strerror(scf_error()));
4257                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4258 
4259                 graph_enable_by_vertex(v, 0, 0);
4260                 return (0);
4261         }
4262 
4263         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4264             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4265         if (r != 0) {
4266                 switch (scf_error()) {
4267                 case SCF_ERROR_CONNECTION_BROKEN:
4268                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4269                         graph_enable_by_vertex(v, 0, 0);
4270                         return (ECONNABORTED);
4271 
4272                 case SCF_ERROR_NOT_FOUND:
4273                         return (0);
4274 
4275                 case SCF_ERROR_HANDLE_MISMATCH:
4276                 case SCF_ERROR_INVALID_ARGUMENT:
4277                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4278                 case SCF_ERROR_NOT_BOUND:
4279                 default:
4280                         bad_error("scf_handle_decode_fmri",
4281                             scf_error());
4282                 }
4283         }
4284 
4285         r = libscf_set_enable_ovr(inst, 0);
4286         switch (r) {
4287         case 0:
4288                 scf_instance_destroy(inst);
4289                 return (0);
4290 
4291         case ECANCELED:
4292                 scf_instance_destroy(inst);
4293                 return (0);
4294 
4295         case ECONNABORTED:
4296                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4297                 graph_enable_by_vertex(v, 0, 0);
4298                 return (ECONNABORTED);
4299 
4300         case EPERM:
4301                 log_error(LOG_WARNING, emsg, v->gv_name,
4302                     "the repository denied permission");
4303                 graph_enable_by_vertex(v, 0, 0);
4304                 return (0);
4305 
4306         case EROFS:
4307                 log_error(LOG_WARNING, emsg, v->gv_name,
4308                     "the repository is read-only");
4309                 graph_enable_by_vertex(v, 0, 0);
4310                 return (0);
4311 
4312         default:
4313                 bad_error("libscf_set_enable_ovr", r);
4314                 /* NOTREACHED */
4315         }
4316 }
4317 
4318 /*
4319  * Of the transitive instance dependencies of v, offline those which are
4320  * in the subtree and which are leaves (i.e., have no dependents which are
4321  * "up").
4322  */
4323 void
4324 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4325 {
4326         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4327 
4328         /* If v isn't an instance, recurse on its dependencies. */
4329         if (v->gv_type != GVT_INST) {
4330                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4331                 return;
4332         }
4333 
4334         /*
4335          * If v is not in the subtree, so should all of its dependencies,
4336          * so do nothing.
4337          */
4338         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4339                 return;
4340 
4341         /* If v isn't a leaf because it's already down, recurse. */
4342         if (!up_state(v->gv_state)) {
4343                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4344                 return;
4345         }
4346 
4347         /* if v is a leaf, offline it or disable it if it's the last one */
4348         if (insubtree_dependents_down(v) == B_TRUE) {
4349                 if (v->gv_flags & GV_TODISABLE)
4350                         vertex_send_event(v,
4351                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4352                 else
4353                         offline_vertex(v);
4354         }
4355 }
4356 
4357 void
4358 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4359 {
4360         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4361 }
4362 
4363 
4364 /*
4365  * Of the transitive instance dependencies of v, disable those which are not
4366  * in the subgraph and which are leaves (i.e., have no dependents which are
4367  * "up").
4368  */
4369 static void
4370 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4371 {
4372         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4373 
4374         /*
4375          * We must skip exclusion dependencies because they are allowed to
4376          * complete dependency cycles.  This is correct because A's exclusion
4377          * dependency on B doesn't bear on the order in which they should be
4378          * stopped.  Indeed, the exclusion dependency should guarantee that
4379          * they are never online at the same time.
4380          */
4381         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4382                 return;
4383 
4384         /* If v isn't an instance, recurse on its dependencies. */
4385         if (v->gv_type != GVT_INST)
4386                 goto recurse;
4387 
4388         if ((v->gv_flags & GV_CONFIGURED) == 0)
4389                 /*
4390                  * Unconfigured instances should have no dependencies, but in
4391                  * case they ever get them,
4392                  */
4393                 goto recurse;
4394 
4395         /*
4396          * If v is in the subgraph, so should all of its dependencies, so do
4397          * nothing.
4398          */
4399         if (v->gv_flags & GV_INSUBGRAPH)
4400                 return;
4401 
4402         /* If v isn't a leaf because it's already down, recurse. */
4403         if (!up_state(v->gv_state))
4404                 goto recurse;
4405 
4406         /* If v is disabled but not down yet, be patient. */
4407         if ((v->gv_flags & GV_ENABLED) == 0)
4408                 return;
4409 
4410         /* If v is a leaf, disable it. */
4411         if (is_nonsubgraph_leaf(v))
4412                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4413 
4414         return;
4415 
4416 recurse:
4417         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4418 }
4419 
4420 /*
4421  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4422  * Otherwise set its state to state.  If the instance has entered a state
4423  * which requires automatic action, take it (Uninitialized: do
4424  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4425  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4426  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4427  * _START.  Online, Degraded: if the instance wasn't running, update its start
4428  * snapshot.  Maintenance: no action.)
4429  *
4430  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4431  */
4432 static int
4433 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4434     restarter_instance_state_t state, restarter_error_t serr)
4435 {
4436         graph_vertex_t *v;
4437         int err = 0;
4438         restarter_instance_state_t old_state;
4439 
4440         MUTEX_LOCK(&dgraph_lock);
4441 
4442         v = vertex_get_by_name(inst_name);
4443         if (v == NULL) {
4444                 MUTEX_UNLOCK(&dgraph_lock);
4445                 return (ENOENT);
4446         }
4447 
4448         assert(v->gv_type == GVT_INST);
4449 
4450         switch (state) {
4451         case RESTARTER_STATE_UNINIT:
4452         case RESTARTER_STATE_DISABLED:
4453         case RESTARTER_STATE_OFFLINE:
4454         case RESTARTER_STATE_ONLINE:
4455         case RESTARTER_STATE_DEGRADED:
4456         case RESTARTER_STATE_MAINT:
4457                 break;
4458 
4459         default:
4460                 MUTEX_UNLOCK(&dgraph_lock);
4461                 return (EINVAL);
4462         }
4463 
4464         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4465             instance_state_str[v->gv_state], instance_state_str[state]);
4466 
4467         old_state = v->gv_state;
4468         v->gv_state = state;
4469 
4470         err = gt_transition(h, v, serr, old_state);
4471 
4472         MUTEX_UNLOCK(&dgraph_lock);
4473         return (err);
4474 }
4475 
4476 /*
4477  * Handle state changes during milestone shutdown.  See
4478  * dgraph_set_milestone().  If the repository connection is broken,
4479  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4480  * the vertex anyway.
4481  */
4482 int
4483 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4484     restarter_instance_state_t old_state)
4485 {
4486         int was_up, now_up;
4487         int ret = 0;
4488 
4489         assert(v->gv_type == GVT_INST);
4490 
4491         /* Don't care if we're not going to a milestone. */
4492         if (milestone == NULL)
4493                 return (0);
4494 
4495         /* Don't care if we already finished coming down. */
4496         if (non_subgraph_svcs == 0)
4497                 return (0);
4498 
4499         /* Don't care if the service is in the subgraph. */
4500         if (v->gv_flags & GV_INSUBGRAPH)
4501                 return (0);
4502 
4503         /*
4504          * Update non_subgraph_svcs.  It is the number of non-subgraph
4505          * services which are in online, degraded, or offline.
4506          */
4507 
4508         was_up = up_state(old_state);
4509         now_up = up_state(v->gv_state);
4510 
4511         if (!was_up && now_up) {
4512                 ++non_subgraph_svcs;
4513         } else if (was_up && !now_up) {
4514                 --non_subgraph_svcs;
4515 
4516                 if (non_subgraph_svcs == 0) {
4517                         if (halting != -1) {
4518                                 do_uadmin();
4519                         } else if (go_single_user_mode || go_to_level1) {
4520                                 (void) startd_thread_create(single_user_thread,
4521                                     NULL);
4522                         }
4523                         return (0);
4524                 }
4525         }
4526 
4527         /* If this service is a leaf, it should be disabled. */
4528         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4529                 int r;
4530 
4531                 r = disable_service_temporarily(v, h);
4532                 switch (r) {
4533                 case 0:
4534                         break;
4535 
4536                 case ECONNABORTED:
4537                         ret = ECONNABORTED;
4538                         break;
4539 
4540                 default:
4541                         bad_error("disable_service_temporarily", r);
4542                 }
4543         }
4544 
4545         /*
4546          * If the service just came down, propagate the disable to the newly
4547          * exposed leaves.
4548          */
4549         if (was_up && !now_up)
4550                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4551                     (void *)h);
4552 
4553         return (ret);
4554 }
4555 
4556 /*
4557  * Decide whether to start up an sulogin thread after a service is
4558  * finished changing state.  Only need to do the full can_come_up()
4559  * evaluation if an instance is changing state, we're not halfway through
4560  * loading the thread, and we aren't shutting down or going to the single
4561  * user milestone.
4562  */
4563 void
4564 graph_transition_sulogin(restarter_instance_state_t state,
4565     restarter_instance_state_t old_state)
4566 {
4567         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4568 
4569         if (state != old_state && st->st_load_complete &&
4570             !go_single_user_mode && !go_to_level1 &&
4571             halting == -1) {
4572                 if (!sulogin_thread_running && !can_come_up()) {
4573                         (void) startd_thread_create(sulogin_thread, NULL);
4574                         sulogin_thread_running = B_TRUE;
4575                 }
4576         }
4577 }
4578 
4579 /*
4580  * Propagate a start, stop event, or a satisfiability event.
4581  *
4582  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4583  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4584  * full dependent graph to check for newly satisfied nodes.  This is
4585  * necessary for cases when non-direct dependents may be effected but direct
4586  * dependents may not (e.g. for optional_all evaluations, see the
4587  * propagate_satbility() comments).
4588  *
4589  * PROPAGATE_SAT should be used whenever a non-running service moves into
4590  * a state which can satisfy optional dependencies, like disabled or
4591  * maintenance.
4592  */
4593 void
4594 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4595     restarter_error_t rerr)
4596 {
4597         if (type == PROPAGATE_STOP) {
4598                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4599         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4600                 graph_walk_dependents(v, propagate_start, NULL);
4601 
4602                 if (type == PROPAGATE_SAT)
4603                         propagate_satbility(v);
4604         } else {
4605 #ifndef NDEBUG
4606                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4607                     __LINE__, type);
4608 #endif
4609                 abort();
4610         }
4611 }
4612 
4613 /*
4614  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4615  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4616  * all property group dependencies, and the dependency on the restarter,
4617  * disposing of vertices as appropriate.  If other vertices depend on this
4618  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4619  * returns 0.
4620  */
4621 static int
4622 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4623 {
4624         graph_vertex_t *v;
4625         graph_edge_t *e;
4626         uu_list_t *old_deps;
4627         int err;
4628 
4629         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4630 
4631         MUTEX_LOCK(&dgraph_lock);
4632 
4633         v = vertex_get_by_name(fmri);
4634         if (v == NULL) {
4635                 MUTEX_UNLOCK(&dgraph_lock);
4636                 return (0);
4637         }
4638 
4639         /* Send restarter delete event. */
4640         if (v->gv_flags & GV_CONFIGURED)
4641                 graph_unset_restarter(v);
4642 
4643         if (milestone > MILESTONE_NONE) {
4644                 /*
4645                  * Make a list of v's current dependencies so we can
4646                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4647                  * are removed.
4648                  */
4649                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4650 
4651                 err = uu_list_walk(v->gv_dependencies,
4652                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4653                 assert(err == 0);
4654         }
4655 
4656         delete_instance_dependencies(v, B_TRUE);
4657 
4658         /*
4659          * Deleting an instance can both satisfy and unsatisfy dependencies,
4660          * depending on their type.  First propagate the stop as a RERR_RESTART
4661          * event -- deletion isn't a fault, just a normal stop.  This gives
4662          * dependent services the chance to do a clean shutdown.  Then, mark
4663          * the service as unconfigured and propagate the start event for the
4664          * optional_all dependencies that might have become satisfied.
4665          */
4666         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
4667 
4668         v->gv_flags &= ~GV_CONFIGURED;
4669         v->gv_flags &= ~GV_DEATHROW;
4670 
4671         graph_walk_dependents(v, propagate_start, NULL);
4672         propagate_satbility(v);
4673 
4674         /*
4675          * If there are no (non-service) dependents, the vertex can be
4676          * completely removed.
4677          */
4678         if (v != milestone && v->gv_refs == 0 &&
4679             uu_list_numnodes(v->gv_dependents) == 1)
4680                 remove_inst_vertex(v);
4681 
4682         if (milestone > MILESTONE_NONE) {
4683                 void *cookie = NULL;
4684 
4685                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
4686                         v = e->ge_vertex;
4687 
4688                         if (vertex_unref(v) == VERTEX_INUSE)
4689                                 while (eval_subgraph(v, h) == ECONNABORTED)
4690                                         libscf_handle_rebind(h);
4691 
4692                         startd_free(e, sizeof (*e));
4693                 }
4694 
4695                 uu_list_destroy(old_deps);
4696         }
4697 
4698         MUTEX_UNLOCK(&dgraph_lock);
4699 
4700         return (0);
4701 }
4702 
4703 /*
4704  * Return the eventual (maybe current) milestone in the form of a
4705  * legacy runlevel.
4706  */
4707 static char
4708 target_milestone_as_runlevel()
4709 {
4710         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4711 
4712         if (milestone == NULL)
4713                 return ('3');
4714         else if (milestone == MILESTONE_NONE)
4715                 return ('0');
4716 
4717         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
4718                 return ('2');
4719         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
4720                 return ('S');
4721         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
4722                 return ('3');
4723 
4724 #ifndef NDEBUG
4725         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
4726             __FILE__, __LINE__, milestone->gv_name);
4727 #endif
4728         abort();
4729         /* NOTREACHED */
4730 }
4731 
4732 static struct {
4733         char    rl;
4734         int     sig;
4735 } init_sigs[] = {
4736         { 'S', SIGBUS },
4737         { '0', SIGINT },
4738         { '1', SIGQUIT },
4739         { '2', SIGILL },
4740         { '3', SIGTRAP },
4741         { '4', SIGIOT },
4742         { '5', SIGEMT },
4743         { '6', SIGFPE },
4744         { 0, 0 }
4745 };
4746 
4747 static void
4748 signal_init(char rl)
4749 {
4750         pid_t init_pid;
4751         int i;
4752 
4753         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4754 
4755         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
4756             sizeof (init_pid)) != sizeof (init_pid)) {
4757                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
4758                 return;
4759         }
4760 
4761         for (i = 0; init_sigs[i].rl != 0; ++i)
4762                 if (init_sigs[i].rl == rl)
4763                         break;
4764 
4765         if (init_sigs[i].rl != 0) {
4766                 if (kill(init_pid, init_sigs[i].sig) != 0) {
4767                         switch (errno) {
4768                         case EPERM:
4769                         case ESRCH:
4770                                 log_error(LOG_NOTICE, "Could not signal init: "
4771                                     "%s.\n", strerror(errno));
4772                                 break;
4773 
4774                         case EINVAL:
4775                         default:
4776                                 bad_error("kill", errno);
4777                         }
4778                 }
4779         }
4780 }
4781 
4782 /*
4783  * This is called when one of the major milestones changes state, or when
4784  * init is signalled and tells us it was told to change runlevel.  We wait
4785  * to reach the milestone because this allows /etc/inittab entries to retain
4786  * some boot ordering: historically, entries could place themselves before/after
4787  * the running of /sbin/rcX scripts but we can no longer make the
4788  * distinction because the /sbin/rcX scripts no longer exist as punctuation
4789  * marks in /etc/inittab.
4790  *
4791  * Also, we only trigger an update when we reach the eventual target
4792  * milestone: without this, an /etc/inittab entry marked only for
4793  * runlevel 2 would be executed for runlevel 3, which is not how
4794  * /etc/inittab entries work.
4795  *
4796  * If we're single user coming online, then we set utmpx to the target
4797  * runlevel so that legacy scripts can work as expected.
4798  */
4799 static void
4800 graph_runlevel_changed(char rl, int online)
4801 {
4802         char trl;
4803 
4804         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4805 
4806         trl = target_milestone_as_runlevel();
4807 
4808         if (online) {
4809                 if (rl == trl) {
4810                         current_runlevel = trl;
4811                         signal_init(trl);
4812                 } else if (rl == 'S') {
4813                         /*
4814                          * At boot, set the entry early for the benefit of the
4815                          * legacy init scripts.
4816                          */
4817                         utmpx_set_runlevel(trl, 'S', B_FALSE);
4818                 }
4819         } else {
4820                 if (rl == '3' && trl == '2') {
4821                         current_runlevel = trl;
4822                         signal_init(trl);
4823                 } else if (rl == '2' && trl == 'S') {
4824                         current_runlevel = trl;
4825                         signal_init(trl);
4826                 }
4827         }
4828 }
4829 
4830 /*
4831  * Move to a backwards-compatible runlevel by executing the appropriate
4832  * /etc/rc?.d/K* scripts and/or setting the milestone.
4833  *
4834  * Returns
4835  *   0 - success
4836  *   ECONNRESET - success, but handle was reset
4837  *   ECONNABORTED - repository connection broken
4838  *   ECANCELED - pg was deleted
4839  */
4840 static int
4841 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
4842 {
4843         char rl;
4844         scf_handle_t *h;
4845         int r;
4846         const char *ms = NULL;  /* what to commit as options/milestone */
4847         boolean_t rebound = B_FALSE;
4848         int mark_rl = 0;
4849 
4850         const char * const stop = "stop";
4851 
4852         r = libscf_extract_runlevel(prop, &rl);
4853         switch (r) {
4854         case 0:
4855                 break;
4856 
4857         case ECONNABORTED:
4858         case ECANCELED:
4859                 return (r);
4860 
4861         case EINVAL:
4862         case ENOENT:
4863                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
4864                     "ignoring.\n");
4865                 /* delete the bad property */
4866                 goto nolock_out;
4867 
4868         default:
4869                 bad_error("libscf_extract_runlevel", r);
4870         }
4871 
4872         switch (rl) {
4873         case 's':
4874                 rl = 'S';
4875                 /* FALLTHROUGH */
4876 
4877         case 'S':
4878         case '2':
4879         case '3':
4880                 /*
4881                  * These cases cause a milestone change, so
4882                  * graph_runlevel_changed() will eventually deal with
4883                  * signalling init.
4884                  */
4885                 break;
4886 
4887         case '0':
4888         case '1':
4889         case '4':
4890         case '5':
4891         case '6':
4892                 mark_rl = 1;
4893                 break;
4894 
4895         default:
4896                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
4897                 ms = NULL;
4898                 goto nolock_out;
4899         }
4900 
4901         h = scf_pg_handle(pg);
4902 
4903         MUTEX_LOCK(&dgraph_lock);
4904 
4905         /*
4906          * Since this triggers no milestone changes, force it by hand.
4907          */
4908         if (current_runlevel == '4' && rl == '3')
4909                 mark_rl = 1;
4910 
4911         /*
4912          * 1. If we are here after an "init X":
4913          *
4914          * init X
4915          *      init/lscf_set_runlevel()
4916          *              process_pg_event()
4917          *              dgraph_set_runlevel()
4918          *
4919          * then we haven't passed through graph_runlevel_changed() yet,
4920          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
4921          * In consequence, if 'rl' is lower than 'current_runlevel', we change
4922          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
4923          * past this test.
4924          *
4925          * 2. On the other hand, if we are here after a "svcadm milestone":
4926          *
4927          * svcadm milestone X
4928          *      dgraph_set_milestone()
4929          *              handle_graph_update_event()
4930          *              dgraph_set_instance_state()
4931          *              graph_post_X_[online|offline]()
4932          *              graph_runlevel_changed()
4933          *              signal_init()
4934          *                      init/lscf_set_runlevel()
4935          *                              process_pg_event()
4936          *                              dgraph_set_runlevel()
4937          *
4938          * then we already passed through graph_runlevel_changed() (by the way
4939          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
4940          * and already be equal to 'rl' so we are going to return immediately
4941          * from dgraph_set_runlevel() without changing the system runlevel and
4942          * without executing the /etc/rc?.d/K* scripts.
4943          */
4944         if (rl == current_runlevel) {
4945                 ms = NULL;
4946                 goto out;
4947         }
4948 
4949         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
4950 
4951         /*
4952          * Make sure stop rc scripts see the new settings via who -r.
4953          */
4954         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
4955 
4956         /*
4957          * Some run levels don't have a direct correspondence to any
4958          * milestones, so we have to signal init directly.
4959          */
4960         if (mark_rl) {
4961                 current_runlevel = rl;
4962                 signal_init(rl);
4963         }
4964 
4965         switch (rl) {
4966         case 'S':
4967                 uu_warn("The system is coming down for administration.  "
4968                     "Please wait.\n");
4969                 fork_rc_script(rl, stop, B_FALSE);
4970                 ms = single_user_fmri;
4971                 go_single_user_mode = B_TRUE;
4972                 break;
4973 
4974         case '0':
4975                 fork_rc_script(rl, stop, B_TRUE);
4976                 halting = AD_HALT;
4977                 goto uadmin;
4978 
4979         case '5':
4980                 fork_rc_script(rl, stop, B_TRUE);
4981                 halting = AD_POWEROFF;
4982                 goto uadmin;
4983 
4984         case '6':
4985                 fork_rc_script(rl, stop, B_TRUE);
4986                 halting = AD_BOOT;
4987                 goto uadmin;
4988 
4989 uadmin:
4990                 uu_warn("The system is coming down.  Please wait.\n");
4991                 ms = "none";
4992 
4993                 /*
4994                  * We can't wait until all services are offline since this
4995                  * thread is responsible for taking them offline.  Instead we
4996                  * set halting to the second argument for uadmin() and call
4997                  * do_uadmin() from dgraph_set_instance_state() when
4998                  * appropriate.
4999                  */
5000                 break;
5001 
5002         case '1':
5003                 if (current_runlevel != 'S') {
5004                         uu_warn("Changing to state 1.\n");
5005                         fork_rc_script(rl, stop, B_FALSE);
5006                 } else {
5007                         uu_warn("The system is coming up for administration.  "
5008                             "Please wait.\n");
5009                 }
5010                 ms = single_user_fmri;
5011                 go_to_level1 = B_TRUE;
5012                 break;
5013 
5014         case '2':
5015                 if (current_runlevel == '3' || current_runlevel == '4')
5016                         fork_rc_script(rl, stop, B_FALSE);
5017                 ms = multi_user_fmri;
5018                 break;
5019 
5020         case '3':
5021         case '4':
5022                 ms = "all";
5023                 break;
5024 
5025         default:
5026 #ifndef NDEBUG
5027                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5028                     __FILE__, __LINE__, rl, rl);
5029 #endif
5030                 abort();
5031         }
5032 
5033 out:
5034         MUTEX_UNLOCK(&dgraph_lock);
5035 
5036 nolock_out:
5037         switch (r = libscf_clear_runlevel(pg, ms)) {
5038         case 0:
5039                 break;
5040 
5041         case ECONNABORTED:
5042                 libscf_handle_rebind(h);
5043                 rebound = B_TRUE;
5044                 goto nolock_out;
5045 
5046         case ECANCELED:
5047                 break;
5048 
5049         case EPERM:
5050         case EACCES:
5051         case EROFS:
5052                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5053                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5054                 break;
5055 
5056         default:
5057                 bad_error("libscf_clear_runlevel", r);
5058         }
5059 
5060         return (rebound ? ECONNRESET : 0);
5061 }
5062 
5063 /*
5064  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5065  * to the instances that are supposed to go offline during an
5066  * administrative disable operation.
5067  */
5068 static int
5069 mark_subtree(graph_edge_t *e, void *arg)
5070 {
5071         graph_vertex_t *v;
5072         int r;
5073 
5074         v = e->ge_vertex;
5075 
5076         /* If it's already in the subgraph, skip. */
5077         if (v->gv_flags & GV_TOOFFLINE)
5078                 return (UU_WALK_NEXT);
5079 
5080         switch (v->gv_type) {
5081         case GVT_INST:
5082                 /* If the instance is already disabled, skip it. */
5083                 if (!(v->gv_flags & GV_ENABLED))
5084                         return (UU_WALK_NEXT);
5085 
5086                 v->gv_flags |= GV_TOOFFLINE;
5087                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5088                 break;
5089         case GVT_GROUP:
5090                 /*
5091                  * Skip all excluded and optional_all dependencies and decide
5092                  * whether to offline the service based on restart_on attribute.
5093                  */
5094                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL ||
5095                     v->gv_depgroup == DEPGRP_OPTIONAL_ALL ||
5096                     v->gv_restart < RERR_RESTART)
5097                         return (UU_WALK_NEXT);
5098                 break;
5099         }
5100 
5101         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5102             0);
5103         assert(r == 0);
5104         return (UU_WALK_NEXT);
5105 }
5106 
5107 static int
5108 mark_subgraph(graph_edge_t *e, void *arg)
5109 {
5110         graph_vertex_t *v;
5111         int r;
5112         int optional = (int)arg;
5113 
5114         v = e->ge_vertex;
5115 
5116         /* If it's already in the subgraph, skip. */
5117         if (v->gv_flags & GV_INSUBGRAPH)
5118                 return (UU_WALK_NEXT);
5119 
5120         /*
5121          * Keep track if walk has entered an optional dependency group
5122          */
5123         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5124                 optional = 1;
5125         }
5126         /*
5127          * Quit if we are in an optional dependency group and the instance
5128          * is disabled
5129          */
5130         if (optional && (v->gv_type == GVT_INST) &&
5131             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5132                 return (UU_WALK_NEXT);
5133 
5134         v->gv_flags |= GV_INSUBGRAPH;
5135 
5136         /* Skip all excluded dependencies. */
5137         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5138                 return (UU_WALK_NEXT);
5139 
5140         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5141             (void *)optional, 0);
5142         assert(r == 0);
5143         return (UU_WALK_NEXT);
5144 }
5145 
5146 /*
5147  * Bring down all services which are not dependencies of fmri.  The
5148  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5149  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5150  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5151  * could consider "offline" to be down, and refrain from sending start
5152  * commands for such services, but that's not strictly necessary, so we'll
5153  * decline to intrude on the state machine.  It would probably confuse users
5154  * anyway.
5155  *
5156  * The services should be brought down in reverse-dependency order, so we
5157  * can't do it all at once here.  We initiate by override-disabling the leaves
5158  * of the dependency tree -- those services which are up but have no
5159  * dependents which are up.  When they come down,
5160  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5161  * exposed leaves.  Perseverance will ensure completion.
5162  *
5163  * Sometimes we need to take action when the transition is complete, like
5164  * start sulogin or halt the system.  To tell when we're done, we initialize
5165  * non_subgraph_svcs here to be the number of services which need to come
5166  * down.  As each does, we decrement the counter.  When it hits zero, we take
5167  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5168  *
5169  * In case we're coming up, we also remove any enable-overrides for the
5170  * services which are dependencies of fmri.
5171  *
5172  * If norepository is true, the function will not change the repository.
5173  *
5174  * The decision to change the system run level in accordance with the milestone
5175  * is taken in dgraph_set_runlevel().
5176  *
5177  * Returns
5178  *   0 - success
5179  *   ECONNRESET - success, but handle was rebound
5180  *   EINVAL - fmri is invalid (error is logged)
5181  *   EALREADY - the milestone is already set to fmri
5182  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5183  */
5184 static int
5185 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5186 {
5187         const char *cfmri, *fs;
5188         graph_vertex_t *nm, *v;
5189         int ret = 0, r;
5190         scf_instance_t *inst;
5191         boolean_t isall, isnone, rebound = B_FALSE;
5192 
5193         /* Validate fmri */
5194         isall = (strcmp(fmri, "all") == 0);
5195         isnone = (strcmp(fmri, "none") == 0);
5196 
5197         if (!isall && !isnone) {
5198                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5199                         goto reject;
5200 
5201                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5202                     strcmp(cfmri, multi_user_fmri) != 0 &&
5203                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5204                         startd_free((void *)cfmri, max_scf_fmri_size);
5205 reject:
5206                         log_framework(LOG_WARNING,
5207                             "Rejecting request for invalid milestone \"%s\".\n",
5208                             fmri);
5209                         return (EINVAL);
5210                 }
5211         }
5212 
5213         inst = safe_scf_instance_create(h);
5214 
5215         MUTEX_LOCK(&dgraph_lock);
5216 
5217         if (milestone == NULL) {
5218                 if (isall) {
5219                         log_framework(LOG_DEBUG,
5220                             "Milestone already set to all.\n");
5221                         ret = EALREADY;
5222                         goto out;
5223                 }
5224         } else if (milestone == MILESTONE_NONE) {
5225                 if (isnone) {
5226                         log_framework(LOG_DEBUG,
5227                             "Milestone already set to none.\n");
5228                         ret = EALREADY;
5229                         goto out;
5230                 }
5231         } else {
5232                 if (!isall && !isnone &&
5233                     strcmp(cfmri, milestone->gv_name) == 0) {
5234                         log_framework(LOG_DEBUG,
5235                             "Milestone already set to %s.\n", cfmri);
5236                         ret = EALREADY;
5237                         goto out;
5238                 }
5239         }
5240 
5241         if (!isall && !isnone) {
5242                 nm = vertex_get_by_name(cfmri);
5243                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5244                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5245                             "because no such service exists.\n", cfmri);
5246                         ret = ENOENT;
5247                         goto out;
5248                 }
5249         }
5250 
5251         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5252 
5253         /*
5254          * Set milestone, removing the old one if this was the last reference.
5255          */
5256         if (milestone > MILESTONE_NONE)
5257                 (void) vertex_unref(milestone);
5258 
5259         if (isall)
5260                 milestone = NULL;
5261         else if (isnone)
5262                 milestone = MILESTONE_NONE;
5263         else {
5264                 milestone = nm;
5265                 /* milestone should count as a reference */
5266                 vertex_ref(milestone);
5267         }
5268 
5269         /* Clear all GV_INSUBGRAPH bits. */
5270         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5271                 v->gv_flags &= ~GV_INSUBGRAPH;
5272 
5273         if (!isall && !isnone) {
5274                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5275                 milestone->gv_flags |= GV_INSUBGRAPH;
5276 
5277                 r = uu_list_walk(milestone->gv_dependencies,
5278                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5279                 assert(r == 0);
5280         }
5281 
5282         /* Un-override services in the subgraph & override-disable the rest. */
5283         if (norepository)
5284                 goto out;
5285 
5286         non_subgraph_svcs = 0;
5287         for (v = uu_list_first(dgraph);
5288             v != NULL;
5289             v = uu_list_next(dgraph, v)) {
5290                 if (v->gv_type != GVT_INST ||
5291                     (v->gv_flags & GV_CONFIGURED) == 0)
5292                         continue;
5293 
5294 again:
5295                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5296                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5297                 if (r != 0) {
5298                         switch (scf_error()) {
5299                         case SCF_ERROR_CONNECTION_BROKEN:
5300                         default:
5301                                 libscf_handle_rebind(h);
5302                                 rebound = B_TRUE;
5303                                 goto again;
5304 
5305                         case SCF_ERROR_NOT_FOUND:
5306                                 continue;
5307 
5308                         case SCF_ERROR_HANDLE_MISMATCH:
5309                         case SCF_ERROR_INVALID_ARGUMENT:
5310                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5311                         case SCF_ERROR_NOT_BOUND:
5312                                 bad_error("scf_handle_decode_fmri",
5313                                     scf_error());
5314                         }
5315                 }
5316 
5317                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5318                         r = libscf_delete_enable_ovr(inst);
5319                         fs = "libscf_delete_enable_ovr";
5320                 } else {
5321                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5322 
5323                         /*
5324                          * Services which are up need to come down before
5325                          * we're done, but we can only disable the leaves
5326                          * here.
5327                          */
5328 
5329                         if (up_state(v->gv_state))
5330                                 ++non_subgraph_svcs;
5331 
5332                         /* If it's already disabled, don't bother. */
5333                         if ((v->gv_flags & GV_ENABLED) == 0)
5334                                 continue;
5335 
5336                         if (!is_nonsubgraph_leaf(v))
5337                                 continue;
5338 
5339                         r = libscf_set_enable_ovr(inst, 0);
5340                         fs = "libscf_set_enable_ovr";
5341                 }
5342                 switch (r) {
5343                 case 0:
5344                 case ECANCELED:
5345                         break;
5346 
5347                 case ECONNABORTED:
5348                         libscf_handle_rebind(h);
5349                         rebound = B_TRUE;
5350                         goto again;
5351 
5352                 case EPERM:
5353                 case EROFS:
5354                         log_error(LOG_WARNING,
5355                             "Could not set %s/%s for %s: %s.\n",
5356                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5357                             v->gv_name, strerror(r));
5358                         break;
5359 
5360                 default:
5361                         bad_error(fs, r);
5362                 }
5363         }
5364 
5365         if (halting != -1) {
5366                 if (non_subgraph_svcs > 1)
5367                         uu_warn("%d system services are now being stopped.\n",
5368                             non_subgraph_svcs);
5369                 else if (non_subgraph_svcs == 1)
5370                         uu_warn("One system service is now being stopped.\n");
5371                 else if (non_subgraph_svcs == 0)
5372                         do_uadmin();
5373         }
5374 
5375         ret = rebound ? ECONNRESET : 0;
5376 
5377 out:
5378         MUTEX_UNLOCK(&dgraph_lock);
5379         if (!isall && !isnone)
5380                 startd_free((void *)cfmri, max_scf_fmri_size);
5381         scf_instance_destroy(inst);
5382         return (ret);
5383 }
5384 
5385 
5386 /*
5387  * Returns 0, ECONNABORTED, or EINVAL.
5388  */
5389 static int
5390 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5391 {
5392         int r;
5393 
5394         switch (e->gpe_type) {
5395         case GRAPH_UPDATE_RELOAD_GRAPH:
5396                 log_error(LOG_WARNING,
5397                     "graph_event: reload graph unimplemented\n");
5398                 break;
5399 
5400         case GRAPH_UPDATE_STATE_CHANGE: {
5401                 protocol_states_t *states = e->gpe_data;
5402 
5403                 switch (r = dgraph_set_instance_state(h, e->gpe_inst,
5404                     states->ps_state, states->ps_err)) {
5405                 case 0:
5406                 case ENOENT:
5407                         break;
5408 
5409                 case ECONNABORTED:
5410                         return (ECONNABORTED);
5411 
5412                 case EINVAL:
5413                 default:
5414 #ifndef NDEBUG
5415                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5416                             "failed with unexpected error %d at %s:%d.\n", r,
5417                             __FILE__, __LINE__);
5418 #endif
5419                         abort();
5420                 }
5421 
5422                 startd_free(states, sizeof (protocol_states_t));
5423                 break;
5424         }
5425 
5426         default:
5427                 log_error(LOG_WARNING,
5428                     "graph_event_loop received an unknown event: %d\n",
5429                     e->gpe_type);
5430                 break;
5431         }
5432 
5433         return (0);
5434 }
5435 
5436 /*
5437  * graph_event_thread()
5438  *    Wait for state changes from the restarters.
5439  */
5440 /*ARGSUSED*/
5441 void *
5442 graph_event_thread(void *unused)
5443 {
5444         scf_handle_t *h;
5445         int err;
5446 
5447         h = libscf_handle_create_bound_loop();
5448 
5449         /*CONSTCOND*/
5450         while (1) {
5451                 graph_protocol_event_t *e;
5452 
5453                 MUTEX_LOCK(&gu->gu_lock);
5454 
5455                 while (gu->gu_wakeup == 0)
5456                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5457 
5458                 gu->gu_wakeup = 0;
5459 
5460                 while ((e = graph_event_dequeue()) != NULL) {
5461                         MUTEX_LOCK(&e->gpe_lock);
5462                         MUTEX_UNLOCK(&gu->gu_lock);
5463 
5464                         while ((err = handle_graph_update_event(h, e)) ==
5465                             ECONNABORTED)
5466                                 libscf_handle_rebind(h);
5467 
5468                         if (err == 0)
5469                                 graph_event_release(e);
5470                         else
5471                                 graph_event_requeue(e);
5472 
5473                         MUTEX_LOCK(&gu->gu_lock);
5474                 }
5475 
5476                 MUTEX_UNLOCK(&gu->gu_lock);
5477         }
5478 
5479         /*
5480          * Unreachable for now -- there's currently no graceful cleanup
5481          * called on exit().
5482          */
5483         MUTEX_UNLOCK(&gu->gu_lock);
5484         scf_handle_destroy(h);
5485         return (NULL);
5486 }
5487 
5488 static void
5489 set_initial_milestone(scf_handle_t *h)
5490 {
5491         scf_instance_t *inst;
5492         char *fmri, *cfmri;
5493         size_t sz;
5494         int r;
5495 
5496         inst = safe_scf_instance_create(h);
5497         fmri = startd_alloc(max_scf_fmri_size);
5498 
5499         /*
5500          * If -m milestone= was specified, we want to set options_ovr/milestone
5501          * to it.  Otherwise we want to read what the milestone should be set
5502          * to.  Either way we need our inst.
5503          */
5504 get_self:
5505         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5506             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5507                 switch (scf_error()) {
5508                 case SCF_ERROR_CONNECTION_BROKEN:
5509                         libscf_handle_rebind(h);
5510                         goto get_self;
5511 
5512                 case SCF_ERROR_NOT_FOUND:
5513                         if (st->st_subgraph != NULL &&
5514                             st->st_subgraph[0] != '\0') {
5515                                 sz = strlcpy(fmri, st->st_subgraph,
5516                                     max_scf_fmri_size);
5517                                 assert(sz < max_scf_fmri_size);
5518                         } else {
5519                                 fmri[0] = '\0';
5520                         }
5521                         break;
5522 
5523                 case SCF_ERROR_INVALID_ARGUMENT:
5524                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5525                 case SCF_ERROR_HANDLE_MISMATCH:
5526                 default:
5527                         bad_error("scf_handle_decode_fmri", scf_error());
5528                 }
5529         } else {
5530                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5531                         scf_propertygroup_t *pg;
5532 
5533                         pg = safe_scf_pg_create(h);
5534 
5535                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5536                         assert(sz < max_scf_fmri_size);
5537 
5538                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5539                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5540                             pg);
5541                         switch (r) {
5542                         case 0:
5543                                 break;
5544 
5545                         case ECONNABORTED:
5546                                 libscf_handle_rebind(h);
5547                                 goto get_self;
5548 
5549                         case EPERM:
5550                         case EACCES:
5551                         case EROFS:
5552                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5553                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5554                                     SCF_PROPERTY_MILESTONE, strerror(r));
5555                                 /* FALLTHROUGH */
5556 
5557                         case ECANCELED:
5558                                 sz = strlcpy(fmri, st->st_subgraph,
5559                                     max_scf_fmri_size);
5560                                 assert(sz < max_scf_fmri_size);
5561                                 break;
5562 
5563                         default:
5564                                 bad_error("libscf_inst_get_or_add_pg", r);
5565                         }
5566 
5567                         r = libscf_clear_runlevel(pg, fmri);
5568                         switch (r) {
5569                         case 0:
5570                                 break;
5571 
5572                         case ECONNABORTED:
5573                                 libscf_handle_rebind(h);
5574                                 goto get_self;
5575 
5576                         case EPERM:
5577                         case EACCES:
5578                         case EROFS:
5579                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5580                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5581                                     SCF_PROPERTY_MILESTONE, strerror(r));
5582                                 /* FALLTHROUGH */
5583 
5584                         case ECANCELED:
5585                                 sz = strlcpy(fmri, st->st_subgraph,
5586                                     max_scf_fmri_size);
5587                                 assert(sz < max_scf_fmri_size);
5588                                 break;
5589 
5590                         default:
5591                                 bad_error("libscf_clear_runlevel", r);
5592                         }
5593 
5594                         scf_pg_destroy(pg);
5595                 } else {
5596                         scf_property_t *prop;
5597                         scf_value_t *val;
5598 
5599                         prop = safe_scf_property_create(h);
5600                         val = safe_scf_value_create(h);
5601 
5602                         r = libscf_get_milestone(inst, prop, val, fmri,
5603                             max_scf_fmri_size);
5604                         switch (r) {
5605                         case 0:
5606                                 break;
5607 
5608                         case ECONNABORTED:
5609                                 libscf_handle_rebind(h);
5610                                 goto get_self;
5611 
5612                         case EINVAL:
5613                                 log_error(LOG_WARNING, "Milestone property is "
5614                                     "misconfigured.  Defaulting to \"all\".\n");
5615                                 /* FALLTHROUGH */
5616 
5617                         case ECANCELED:
5618                         case ENOENT:
5619                                 fmri[0] = '\0';
5620                                 break;
5621 
5622                         default:
5623                                 bad_error("libscf_get_milestone", r);
5624                         }
5625 
5626                         scf_value_destroy(val);
5627                         scf_property_destroy(prop);
5628                 }
5629         }
5630 
5631         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5632                 goto out;
5633 
5634         if (strcmp(fmri, "none") != 0) {
5635 retry:
5636                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5637                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5638                         switch (scf_error()) {
5639                         case SCF_ERROR_INVALID_ARGUMENT:
5640                                 log_error(LOG_WARNING,
5641                                     "Requested milestone \"%s\" is invalid.  "
5642                                     "Reverting to \"all\".\n", fmri);
5643                                 goto out;
5644 
5645                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5646                                 log_error(LOG_WARNING, "Requested milestone "
5647                                     "\"%s\" does not specify an instance.  "
5648                                     "Reverting to \"all\".\n", fmri);
5649                                 goto out;
5650 
5651                         case SCF_ERROR_CONNECTION_BROKEN:
5652                                 libscf_handle_rebind(h);
5653                                 goto retry;
5654 
5655                         case SCF_ERROR_NOT_FOUND:
5656                                 log_error(LOG_WARNING, "Requested milestone "
5657                                     "\"%s\" not in repository.  Reverting to "
5658                                     "\"all\".\n", fmri);
5659                                 goto out;
5660 
5661                         case SCF_ERROR_HANDLE_MISMATCH:
5662                         default:
5663                                 bad_error("scf_handle_decode_fmri",
5664                                     scf_error());
5665                         }
5666                 }
5667 
5668                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
5669                 assert(r == 0);
5670 
5671                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
5672                 startd_free(cfmri, max_scf_fmri_size);
5673                 switch (r) {
5674                 case 0:
5675                         break;
5676 
5677                 case ECONNABORTED:
5678                         goto retry;
5679 
5680                 case EINVAL:
5681                         log_error(LOG_WARNING,
5682                             "Requested milestone \"%s\" is invalid.  "
5683                             "Reverting to \"all\".\n", fmri);
5684                         goto out;
5685 
5686                 case ECANCELED:
5687                         log_error(LOG_WARNING,
5688                             "Requested milestone \"%s\" not "
5689                             "in repository.  Reverting to \"all\".\n",
5690                             fmri);
5691                         goto out;
5692 
5693                 case EEXIST:
5694                 default:
5695                         bad_error("dgraph_add_instance", r);
5696                 }
5697         }
5698 
5699         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
5700 
5701         r = dgraph_set_milestone(fmri, h, B_FALSE);
5702         switch (r) {
5703         case 0:
5704         case ECONNRESET:
5705         case EALREADY:
5706                 break;
5707 
5708         case EINVAL:
5709         case ENOENT:
5710         default:
5711                 bad_error("dgraph_set_milestone", r);
5712         }
5713 
5714 out:
5715         startd_free(fmri, max_scf_fmri_size);
5716         scf_instance_destroy(inst);
5717 }
5718 
5719 void
5720 set_restart_milestone(scf_handle_t *h)
5721 {
5722         scf_instance_t *inst;
5723         scf_property_t *prop;
5724         scf_value_t *val;
5725         char *fmri;
5726         int r;
5727 
5728         inst = safe_scf_instance_create(h);
5729 
5730 get_self:
5731         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
5732             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5733                 switch (scf_error()) {
5734                 case SCF_ERROR_CONNECTION_BROKEN:
5735                         libscf_handle_rebind(h);
5736                         goto get_self;
5737 
5738                 case SCF_ERROR_NOT_FOUND:
5739                         break;
5740 
5741                 case SCF_ERROR_INVALID_ARGUMENT:
5742                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5743                 case SCF_ERROR_HANDLE_MISMATCH:
5744                 default:
5745                         bad_error("scf_handle_decode_fmri", scf_error());
5746                 }
5747 
5748                 scf_instance_destroy(inst);
5749                 return;
5750         }
5751 
5752         prop = safe_scf_property_create(h);
5753         val = safe_scf_value_create(h);
5754         fmri = startd_alloc(max_scf_fmri_size);
5755 
5756         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
5757         switch (r) {
5758         case 0:
5759                 break;
5760 
5761         case ECONNABORTED:
5762                 libscf_handle_rebind(h);
5763                 goto get_self;
5764 
5765         case ECANCELED:
5766         case ENOENT:
5767         case EINVAL:
5768                 goto out;
5769 
5770         default:
5771                 bad_error("libscf_get_milestone", r);
5772         }
5773 
5774         r = dgraph_set_milestone(fmri, h, B_TRUE);
5775         switch (r) {
5776         case 0:
5777         case ECONNRESET:
5778         case EALREADY:
5779         case EINVAL:
5780         case ENOENT:
5781                 break;
5782 
5783         default:
5784                 bad_error("dgraph_set_milestone", r);
5785         }
5786 
5787 out:
5788         startd_free(fmri, max_scf_fmri_size);
5789         scf_value_destroy(val);
5790         scf_property_destroy(prop);
5791         scf_instance_destroy(inst);
5792 }
5793 
5794 /*
5795  * void *graph_thread(void *)
5796  *
5797  * Graph management thread.
5798  */
5799 /*ARGSUSED*/
5800 void *
5801 graph_thread(void *arg)
5802 {
5803         scf_handle_t *h;
5804         int err;
5805 
5806         h = libscf_handle_create_bound_loop();
5807 
5808         if (st->st_initial)
5809                 set_initial_milestone(h);
5810 
5811         MUTEX_LOCK(&dgraph_lock);
5812         initial_milestone_set = B_TRUE;
5813         err = pthread_cond_broadcast(&initial_milestone_cv);
5814         assert(err == 0);
5815         MUTEX_UNLOCK(&dgraph_lock);
5816 
5817         libscf_populate_graph(h);
5818 
5819         if (!st->st_initial)
5820                 set_restart_milestone(h);
5821 
5822         MUTEX_LOCK(&st->st_load_lock);
5823         st->st_load_complete = 1;
5824         (void) pthread_cond_broadcast(&st->st_load_cv);
5825         MUTEX_UNLOCK(&st->st_load_lock);
5826 
5827         MUTEX_LOCK(&dgraph_lock);
5828         /*
5829          * Now that we've set st_load_complete we need to check can_come_up()
5830          * since if we booted to a milestone, then there won't be any more
5831          * state updates.
5832          */
5833         if (!go_single_user_mode && !go_to_level1 &&
5834             halting == -1) {
5835                 if (!sulogin_thread_running && !can_come_up()) {
5836                         (void) startd_thread_create(sulogin_thread, NULL);
5837                         sulogin_thread_running = B_TRUE;
5838                 }
5839         }
5840         MUTEX_UNLOCK(&dgraph_lock);
5841 
5842         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
5843 
5844         /*CONSTCOND*/
5845         while (1) {
5846                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
5847                     &gu->gu_freeze_lock);
5848         }
5849 
5850         /*
5851          * Unreachable for now -- there's currently no graceful cleanup
5852          * called on exit().
5853          */
5854         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
5855         scf_handle_destroy(h);
5856 
5857         return (NULL);
5858 }
5859 
5860 
5861 /*
5862  * int next_action()
5863  *   Given an array of timestamps 'a' with 'num' elements, find the
5864  *   lowest non-zero timestamp and return its index. If there are no
5865  *   non-zero elements, return -1.
5866  */
5867 static int
5868 next_action(hrtime_t *a, int num)
5869 {
5870         hrtime_t t = 0;
5871         int i = 0, smallest = -1;
5872 
5873         for (i = 0; i < num; i++) {
5874                 if (t == 0) {
5875                         t = a[i];
5876                         smallest = i;
5877                 } else if (a[i] != 0 && a[i] < t) {
5878                         t = a[i];
5879                         smallest = i;
5880                 }
5881         }
5882 
5883         if (t == 0)
5884                 return (-1);
5885         else
5886                 return (smallest);
5887 }
5888 
5889 /*
5890  * void process_actions()
5891  *   Process actions requested by the administrator. Possibilities include:
5892  *   refresh, restart, maintenance mode off, maintenance mode on,
5893  *   maintenance mode immediate, and degraded.
5894  *
5895  *   The set of pending actions is represented in the repository as a
5896  *   per-instance property group, with each action being a single property
5897  *   in that group.  This property group is converted to an array, with each
5898  *   action type having an array slot.  The actions in the array at the
5899  *   time process_actions() is called are acted on in the order of the
5900  *   timestamp (which is the value stored in the slot).  A value of zero
5901  *   indicates that there is no pending action of the type associated with
5902  *   a particular slot.
5903  *
5904  *   Sending an action event multiple times before the restarter has a
5905  *   chance to process that action will force it to be run at the last
5906  *   timestamp where it appears in the ordering.
5907  *
5908  *   Turning maintenance mode on trumps all other actions.
5909  *
5910  *   Returns 0 or ECONNABORTED.
5911  */
5912 static int
5913 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
5914 {
5915         scf_property_t *prop = NULL;
5916         scf_value_t *val = NULL;
5917         scf_type_t type;
5918         graph_vertex_t *vertex;
5919         admin_action_t a;
5920         int i, ret = 0, r;
5921         hrtime_t action_ts[NACTIONS];
5922         char *inst_name;
5923 
5924         r = libscf_instance_get_fmri(inst, &inst_name);
5925         switch (r) {
5926         case 0:
5927                 break;
5928 
5929         case ECONNABORTED:
5930                 return (ECONNABORTED);
5931 
5932         case ECANCELED:
5933                 return (0);
5934 
5935         default:
5936                 bad_error("libscf_instance_get_fmri", r);
5937         }
5938 
5939         MUTEX_LOCK(&dgraph_lock);
5940 
5941         vertex = vertex_get_by_name(inst_name);
5942         if (vertex == NULL) {
5943                 MUTEX_UNLOCK(&dgraph_lock);
5944                 startd_free(inst_name, max_scf_fmri_size);
5945                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
5946                     "The instance must have been removed.\n", inst_name);
5947                 return (0);
5948         }
5949 
5950         prop = safe_scf_property_create(h);
5951         val = safe_scf_value_create(h);
5952 
5953         for (i = 0; i < NACTIONS; i++) {
5954                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
5955                         switch (scf_error()) {
5956                         case SCF_ERROR_CONNECTION_BROKEN:
5957                         default:
5958                                 ret = ECONNABORTED;
5959                                 goto out;
5960 
5961                         case SCF_ERROR_DELETED:
5962                                 goto out;
5963 
5964                         case SCF_ERROR_NOT_FOUND:
5965                                 action_ts[i] = 0;
5966                                 continue;
5967 
5968                         case SCF_ERROR_HANDLE_MISMATCH:
5969                         case SCF_ERROR_INVALID_ARGUMENT:
5970                         case SCF_ERROR_NOT_SET:
5971                                 bad_error("scf_pg_get_property", scf_error());
5972                         }
5973                 }
5974 
5975                 if (scf_property_type(prop, &type) != 0) {
5976                         switch (scf_error()) {
5977                         case SCF_ERROR_CONNECTION_BROKEN:
5978                         default:
5979                                 ret = ECONNABORTED;
5980                                 goto out;
5981 
5982                         case SCF_ERROR_DELETED:
5983                                 action_ts[i] = 0;
5984                                 continue;
5985 
5986                         case SCF_ERROR_NOT_SET:
5987                                 bad_error("scf_property_type", scf_error());
5988                         }
5989                 }
5990 
5991                 if (type != SCF_TYPE_INTEGER) {
5992                         action_ts[i] = 0;
5993                         continue;
5994                 }
5995 
5996                 if (scf_property_get_value(prop, val) != 0) {
5997                         switch (scf_error()) {
5998                         case SCF_ERROR_CONNECTION_BROKEN:
5999                         default:
6000                                 ret = ECONNABORTED;
6001                                 goto out;
6002 
6003                         case SCF_ERROR_DELETED:
6004                                 goto out;
6005 
6006                         case SCF_ERROR_NOT_FOUND:
6007                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6008                                 action_ts[i] = 0;
6009                                 continue;
6010 
6011                         case SCF_ERROR_NOT_SET:
6012                         case SCF_ERROR_PERMISSION_DENIED:
6013                                 bad_error("scf_property_get_value",
6014                                     scf_error());
6015                         }
6016                 }
6017 
6018                 r = scf_value_get_integer(val, &action_ts[i]);
6019                 assert(r == 0);
6020         }
6021 
6022         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6023         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6024             action_ts[ADMIN_EVENT_MAINT_ON]) {
6025                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6026                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6027 
6028                 vertex_send_event(vertex, admin_events[a]);
6029                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6030                 switch (r) {
6031                 case 0:
6032                 case EACCES:
6033                         break;
6034 
6035                 case ECONNABORTED:
6036                         ret = ECONNABORTED;
6037                         goto out;
6038 
6039                 case EPERM:
6040                         uu_die("Insufficient privilege.\n");
6041                         /* NOTREACHED */
6042 
6043                 default:
6044                         bad_error("libscf_unset_action", r);
6045                 }
6046         }
6047 
6048         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6049                 log_framework(LOG_DEBUG,
6050                     "Graph: processing %s action for %s.\n", admin_actions[a],
6051                     inst_name);
6052 
6053                 if (a == ADMIN_EVENT_REFRESH) {
6054                         r = dgraph_refresh_instance(vertex, inst);
6055                         switch (r) {
6056                         case 0:
6057                         case ECANCELED:
6058                         case EINVAL:
6059                         case -1:
6060                                 break;
6061 
6062                         case ECONNABORTED:
6063                                 /* pg & inst are reset now, so just return. */
6064                                 ret = ECONNABORTED;
6065                                 goto out;
6066 
6067                         default:
6068                                 bad_error("dgraph_refresh_instance", r);
6069                         }
6070                 }
6071 
6072                 vertex_send_event(vertex, admin_events[a]);
6073 
6074                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6075                 switch (r) {
6076                 case 0:
6077                 case EACCES:
6078                         break;
6079 
6080                 case ECONNABORTED:
6081                         ret = ECONNABORTED;
6082                         goto out;
6083 
6084                 case EPERM:
6085                         uu_die("Insufficient privilege.\n");
6086                         /* NOTREACHED */
6087 
6088                 default:
6089                         bad_error("libscf_unset_action", r);
6090                 }
6091 
6092                 action_ts[a] = 0;
6093         }
6094 
6095 out:
6096         MUTEX_UNLOCK(&dgraph_lock);
6097 
6098         scf_property_destroy(prop);
6099         scf_value_destroy(val);
6100         startd_free(inst_name, max_scf_fmri_size);
6101         return (ret);
6102 }
6103 
6104 /*
6105  * inst and pg_name are scratch space, and are unset on entry.
6106  * Returns
6107  *   0 - success
6108  *   ECONNRESET - success, but repository handle rebound
6109  *   ECONNABORTED - repository connection broken
6110  */
6111 static int
6112 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6113     char *pg_name)
6114 {
6115         int r;
6116         scf_property_t *prop;
6117         scf_value_t *val;
6118         char *fmri;
6119         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6120 
6121         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6122                 switch (scf_error()) {
6123                 case SCF_ERROR_CONNECTION_BROKEN:
6124                 default:
6125                         return (ECONNABORTED);
6126 
6127                 case SCF_ERROR_DELETED:
6128                         return (0);
6129 
6130                 case SCF_ERROR_NOT_SET:
6131                         bad_error("scf_pg_get_name", scf_error());
6132                 }
6133         }
6134 
6135         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6136             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6137                 r = dgraph_update_general(pg);
6138                 switch (r) {
6139                 case 0:
6140                 case ENOTSUP:
6141                 case ECANCELED:
6142                         return (0);
6143 
6144                 case ECONNABORTED:
6145                         return (ECONNABORTED);
6146 
6147                 case -1:
6148                         /* Error should have been logged. */
6149                         return (0);
6150 
6151                 default:
6152                         bad_error("dgraph_update_general", r);
6153                 }
6154         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6155                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6156                         switch (scf_error()) {
6157                         case SCF_ERROR_CONNECTION_BROKEN:
6158                                 return (ECONNABORTED);
6159 
6160                         case SCF_ERROR_DELETED:
6161                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6162                                 /* Ignore commands on services. */
6163                                 return (0);
6164 
6165                         case SCF_ERROR_NOT_BOUND:
6166                         case SCF_ERROR_HANDLE_MISMATCH:
6167                         case SCF_ERROR_NOT_SET:
6168                         default:
6169                                 bad_error("scf_pg_get_parent_instance",
6170                                     scf_error());
6171                         }
6172                 }
6173 
6174                 return (process_actions(h, pg, inst));
6175         }
6176 
6177         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6178             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6179                 return (0);
6180 
6181         /*
6182          * We only care about the options[_ovr] property groups of our own
6183          * instance, so get the fmri and compare.  Plus, once we know it's
6184          * correct, if the repository connection is broken we know exactly what
6185          * property group we were operating on, and can look it up again.
6186          */
6187         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6188                 switch (scf_error()) {
6189                 case SCF_ERROR_CONNECTION_BROKEN:
6190                         return (ECONNABORTED);
6191 
6192                 case SCF_ERROR_DELETED:
6193                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6194                         return (0);
6195 
6196                 case SCF_ERROR_HANDLE_MISMATCH:
6197                 case SCF_ERROR_NOT_BOUND:
6198                 case SCF_ERROR_NOT_SET:
6199                 default:
6200                         bad_error("scf_pg_get_parent_instance",
6201                             scf_error());
6202                 }
6203         }
6204 
6205         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6206         case 0:
6207                 break;
6208 
6209         case ECONNABORTED:
6210                 return (ECONNABORTED);
6211 
6212         case ECANCELED:
6213                 return (0);
6214 
6215         default:
6216                 bad_error("libscf_instance_get_fmri", r);
6217         }
6218 
6219         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6220                 startd_free(fmri, max_scf_fmri_size);
6221                 return (0);
6222         }
6223 
6224         prop = safe_scf_property_create(h);
6225         val = safe_scf_value_create(h);
6226 
6227         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6228                 /* See if we need to set the runlevel. */
6229                 /* CONSTCOND */
6230                 if (0) {
6231 rebind_pg:
6232                         libscf_handle_rebind(h);
6233                         rebound = B_TRUE;
6234 
6235                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6236                         switch (r) {
6237                         case 0:
6238                                 break;
6239 
6240                         case ECONNABORTED:
6241                                 goto rebind_pg;
6242 
6243                         case ENOENT:
6244                                 goto out;
6245 
6246                         case EINVAL:
6247                         case ENOTSUP:
6248                                 bad_error("libscf_lookup_instance", r);
6249                         }
6250 
6251                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6252                                 switch (scf_error()) {
6253                                 case SCF_ERROR_DELETED:
6254                                 case SCF_ERROR_NOT_FOUND:
6255                                         goto out;
6256 
6257                                 case SCF_ERROR_CONNECTION_BROKEN:
6258                                         goto rebind_pg;
6259 
6260                                 case SCF_ERROR_HANDLE_MISMATCH:
6261                                 case SCF_ERROR_NOT_BOUND:
6262                                 case SCF_ERROR_NOT_SET:
6263                                 case SCF_ERROR_INVALID_ARGUMENT:
6264                                 default:
6265                                         bad_error("scf_instance_get_pg",
6266                                             scf_error());
6267                                 }
6268                         }
6269                 }
6270 
6271                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6272                         r = dgraph_set_runlevel(pg, prop);
6273                         switch (r) {
6274                         case ECONNRESET:
6275                                 rebound = B_TRUE;
6276                                 rebind_inst = B_TRUE;
6277                                 /* FALLTHROUGH */
6278 
6279                         case 0:
6280                                 break;
6281 
6282                         case ECONNABORTED:
6283                                 goto rebind_pg;
6284 
6285                         case ECANCELED:
6286                                 goto out;
6287 
6288                         default:
6289                                 bad_error("dgraph_set_runlevel", r);
6290                         }
6291                 } else {
6292                         switch (scf_error()) {
6293                         case SCF_ERROR_CONNECTION_BROKEN:
6294                         default:
6295                                 goto rebind_pg;
6296 
6297                         case SCF_ERROR_DELETED:
6298                                 goto out;
6299 
6300                         case SCF_ERROR_NOT_FOUND:
6301                                 break;
6302 
6303                         case SCF_ERROR_INVALID_ARGUMENT:
6304                         case SCF_ERROR_HANDLE_MISMATCH:
6305                         case SCF_ERROR_NOT_BOUND:
6306                         case SCF_ERROR_NOT_SET:
6307                                 bad_error("scf_pg_get_property", scf_error());
6308                         }
6309                 }
6310         }
6311 
6312         if (rebind_inst) {
6313 lookup_inst:
6314                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6315                 switch (r) {
6316                 case 0:
6317                         break;
6318 
6319                 case ECONNABORTED:
6320                         libscf_handle_rebind(h);
6321                         rebound = B_TRUE;
6322                         goto lookup_inst;
6323 
6324                 case ENOENT:
6325                         goto out;
6326 
6327                 case EINVAL:
6328                 case ENOTSUP:
6329                         bad_error("libscf_lookup_instance", r);
6330                 }
6331         }
6332 
6333         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6334         switch (r) {
6335         case 0:
6336                 break;
6337 
6338         case ECONNABORTED:
6339                 libscf_handle_rebind(h);
6340                 rebound = B_TRUE;
6341                 goto lookup_inst;
6342 
6343         case EINVAL:
6344                 log_error(LOG_NOTICE,
6345                     "%s/%s property of %s is misconfigured.\n", pg_name,
6346                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6347                 /* FALLTHROUGH */
6348 
6349         case ECANCELED:
6350         case ENOENT:
6351                 (void) strcpy(fmri, "all");
6352                 break;
6353 
6354         default:
6355                 bad_error("libscf_get_milestone", r);
6356         }
6357 
6358         r = dgraph_set_milestone(fmri, h, B_FALSE);
6359         switch (r) {
6360         case 0:
6361         case ECONNRESET:
6362         case EALREADY:
6363                 break;
6364 
6365         case EINVAL:
6366                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6367                 break;
6368 
6369         case ENOENT:
6370                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6371                 break;
6372 
6373         default:
6374                 bad_error("dgraph_set_milestone", r);
6375         }
6376 
6377 out:
6378         startd_free(fmri, max_scf_fmri_size);
6379         scf_value_destroy(val);
6380         scf_property_destroy(prop);
6381 
6382         return (rebound ? ECONNRESET : 0);
6383 }
6384 
6385 /*
6386  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6387  * instance fmri or if 'fmri' matches the 'general' property group of an
6388  * instance (or the 'general/enabled' property).
6389  *
6390  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6391  */
6392 static void
6393 process_delete(char *fmri, scf_handle_t *h)
6394 {
6395         char *lfmri, *end_inst_fmri;
6396         const char *inst_name = NULL;
6397         const char *pg_name = NULL;
6398         const char *prop_name = NULL;
6399 
6400         lfmri = safe_strdup(fmri);
6401 
6402         /* Determine if the FMRI is a property group or instance */
6403         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6404             &prop_name) != SCF_SUCCESS) {
6405                 log_error(LOG_WARNING,
6406                     "Received invalid FMRI \"%s\" from repository server.\n",
6407                     fmri);
6408         } else if (inst_name != NULL && pg_name == NULL) {
6409                 (void) dgraph_remove_instance(fmri, h);
6410         } else if (inst_name != NULL && pg_name != NULL) {
6411                 /*
6412                  * If we're deleting the 'general' property group or
6413                  * 'general/enabled' property then the whole instance
6414                  * must be removed from the dgraph.
6415                  */
6416                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6417                         free(lfmri);
6418                         return;
6419                 }
6420 
6421                 if (prop_name != NULL &&
6422                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6423                         free(lfmri);
6424                         return;
6425                 }
6426 
6427                 /*
6428                  * Because the instance has already been deleted from the
6429                  * repository, we cannot use any scf_ functions to retrieve
6430                  * the instance FMRI however we can easily reconstruct it
6431                  * manually.
6432                  */
6433                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6434                 if (end_inst_fmri == NULL)
6435                         bad_error("process_delete", 0);
6436 
6437                 end_inst_fmri[0] = '\0';
6438 
6439                 (void) dgraph_remove_instance(fmri, h);
6440         }
6441 
6442         free(lfmri);
6443 }
6444 
6445 /*ARGSUSED*/
6446 void *
6447 repository_event_thread(void *unused)
6448 {
6449         scf_handle_t *h;
6450         scf_propertygroup_t *pg;
6451         scf_instance_t *inst;
6452         char *fmri = startd_alloc(max_scf_fmri_size);
6453         char *pg_name = startd_alloc(max_scf_value_size);
6454         int r;
6455 
6456         h = libscf_handle_create_bound_loop();
6457 
6458         pg = safe_scf_pg_create(h);
6459         inst = safe_scf_instance_create(h);
6460 
6461 retry:
6462         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6463                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6464                         libscf_handle_rebind(h);
6465                 } else {
6466                         log_error(LOG_WARNING,
6467                             "Couldn't set up repository notification "
6468                             "for property group type %s: %s\n",
6469                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6470 
6471                         (void) sleep(1);
6472                 }
6473 
6474                 goto retry;
6475         }
6476 
6477         /*CONSTCOND*/
6478         while (1) {
6479                 ssize_t res;
6480 
6481                 /* Note: fmri is only set on delete events. */
6482                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6483                 if (res < 0) {
6484                         libscf_handle_rebind(h);
6485                         goto retry;
6486                 } else if (res == 0) {
6487                         /*
6488                          * property group modified.  inst and pg_name are
6489                          * pre-allocated scratch space.
6490                          */
6491                         if (scf_pg_update(pg) < 0) {
6492                                 switch (scf_error()) {
6493                                 case SCF_ERROR_DELETED:
6494                                         continue;
6495 
6496                                 case SCF_ERROR_CONNECTION_BROKEN:
6497                                         log_error(LOG_WARNING,
6498                                             "Lost repository event due to "
6499                                             "disconnection.\n");
6500                                         libscf_handle_rebind(h);
6501                                         goto retry;
6502 
6503                                 case SCF_ERROR_NOT_BOUND:
6504                                 case SCF_ERROR_NOT_SET:
6505                                 default:
6506                                         bad_error("scf_pg_update", scf_error());
6507                                 }
6508                         }
6509 
6510                         r = process_pg_event(h, pg, inst, pg_name);
6511                         switch (r) {
6512                         case 0:
6513                                 break;
6514 
6515                         case ECONNABORTED:
6516                                 log_error(LOG_WARNING, "Lost repository event "
6517                                     "due to disconnection.\n");
6518                                 libscf_handle_rebind(h);
6519                                 /* FALLTHROUGH */
6520 
6521                         case ECONNRESET:
6522                                 goto retry;
6523 
6524                         default:
6525                                 bad_error("process_pg_event", r);
6526                         }
6527                 } else {
6528                         /*
6529                          * Service, instance, or pg deleted.
6530                          * Don't trust fmri on return.
6531                          */
6532                         process_delete(fmri, h);
6533                 }
6534         }
6535 
6536         /*NOTREACHED*/
6537         return (NULL);
6538 }
6539 
6540 void
6541 graph_engine_start()
6542 {
6543         int err;
6544 
6545         (void) startd_thread_create(graph_thread, NULL);
6546 
6547         MUTEX_LOCK(&dgraph_lock);
6548         while (!initial_milestone_set) {
6549                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6550                 assert(err == 0);
6551         }
6552         MUTEX_UNLOCK(&dgraph_lock);
6553 
6554         (void) startd_thread_create(repository_event_thread, NULL);
6555         (void) startd_thread_create(graph_event_thread, NULL);
6556 }