1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /*
  27  * graph.c - master restarter graph engine
  28  *
  29  *   The graph engine keeps a dependency graph of all service instances on the
  30  *   system, as recorded in the repository.  It decides when services should
  31  *   be brought up or down based on service states and dependencies and sends
  32  *   commands to restarters to effect any changes.  It also executes
  33  *   administrator commands sent by svcadm via the repository.
  34  *
  35  *   The graph is stored in uu_list_t *dgraph and its vertices are
  36  *   graph_vertex_t's, each of which has a name and an integer id unique to
  37  *   its name (see dict.c).  A vertex's type attribute designates the type
  38  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  39  *   service objects (since service instances may depend on another service,
  40  *   rather than service instance), GVT_FILE for files (which services may
  41  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  42  *   vertices are necessary because dependency lists may have particular
  43  *   grouping types (require any, require all, optional, or exclude) and
  44  *   event-propagation characteristics.
  45  *
  46  *   The initial graph is built by libscf_populate_graph() invoking
  47  *   dgraph_add_instance() for each instance in the repository.  The function
  48  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  49  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  50  *   The resulting web of vertices & edges associated with an instance's vertex
  51  *   includes
  52  *
  53  *     - an edge from the GVT_SVC vertex for the instance's service
  54  *
  55  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  56  *       restarter is not svc.startd
  57  *
  58  *     - edges from other GVT_INST vertices if the instance is a restarter
  59  *
  60  *     - for each dependency property group in the instance's "running"
  61  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  62  *       instance and the name of the property group
  63  *
  64  *     - for each value of the "entities" property in each dependency property
  65  *       group, an edge from the corresponding GVT_GROUP vertex to a
  66  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  67  *
  68  *     - edges from GVT_GROUP vertices for each dependent instance
  69  *
  70  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  71  *   there are problems, or if a service is mentioned in a dependency but does
  72  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  73  *
  74  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  75  *   See restarter.c for more information.
  76  *
  77  *   The properties of an instance fall into two classes: immediate and
  78  *   snapshotted.  Immediate properties should have an immediate effect when
  79  *   changed.  Snapshotted properties should be read from a snapshot, so they
  80  *   only change when the snapshot changes.  The immediate properties used by
  81  *   the graph engine are general/enabled, general/restarter, and the properties
  82  *   in the restarter_actions property group.  Since they are immediate, they
  83  *   are not read out of a snapshot.  The snapshotted properties used by the
  84  *   graph engine are those in the property groups with type "dependency" and
  85  *   are read out of the "running" snapshot.  The "running" snapshot is created
  86  *   by the the graph engine as soon as possible, and it is updated, along with
  87  *   in-core copies of the data (dependency information for the graph engine) on
  88  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  89  *   updates the "start" snapshot from the "running" snapshot whenever a service
  90  *   comes online.
  91  *
  92  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  93  *   the dependents first before shutting down the requested service.
  94  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  95  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  96  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  97  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
  98  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
  99  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 100  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 101  *   exposed leaves. We do the same until we reach the last leaf (the one with
 102  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 103  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 104  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 105  *   continue to propagate the offline event to the vertex's dependencies.
 106  */
 107 
 108 #include <sys/uadmin.h>
 109 #include <sys/wait.h>
 110 
 111 #include <assert.h>
 112 #include <errno.h>
 113 #include <fcntl.h>
 114 #include <libscf.h>
 115 #include <libscf_priv.h>
 116 #include <libuutil.h>
 117 #include <locale.h>
 118 #include <poll.h>
 119 #include <pthread.h>
 120 #include <signal.h>
 121 #include <stddef.h>
 122 #include <stdio.h>
 123 #include <stdlib.h>
 124 #include <string.h>
 125 #include <strings.h>
 126 #include <sys/statvfs.h>
 127 #include <sys/uadmin.h>
 128 #include <zone.h>
 129 
 130 #include "startd.h"
 131 #include "protocol.h"
 132 
 133 
 134 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 135 
 136 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 137 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 138 
 139 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 140 #define VERTEX_INUSE    1       /* vertex is still in use */
 141 
 142 /*
 143  * Services in these states are not considered 'down' by the
 144  * milestone/shutdown code.
 145  */
 146 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 147         (state) == RESTARTER_STATE_DEGRADED || \
 148         (state) == RESTARTER_STATE_OFFLINE)
 149 
 150 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 151 static uu_list_t *dgraph;
 152 static pthread_mutex_t dgraph_lock;
 153 
 154 /*
 155  * milestone indicates the current subgraph.  When NULL, it is the entire
 156  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 157  * services on which the target vertex depends.
 158  */
 159 static graph_vertex_t *milestone = NULL;
 160 static boolean_t initial_milestone_set = B_FALSE;
 161 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 162 
 163 /* protected by dgraph_lock */
 164 static boolean_t sulogin_thread_running = B_FALSE;
 165 static boolean_t sulogin_running = B_FALSE;
 166 static boolean_t console_login_ready = B_FALSE;
 167 
 168 /* Number of services to come down to complete milestone transition. */
 169 static uint_t non_subgraph_svcs;
 170 
 171 /*
 172  * These variables indicate what should be done when we reach the milestone
 173  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 174  * dgraph_set_instance_state().
 175  */
 176 static int halting = -1;
 177 static boolean_t go_single_user_mode = B_FALSE;
 178 static boolean_t go_to_level1 = B_FALSE;
 179 
 180 /*
 181  * Tracks when we started halting.
 182  */
 183 static time_t halting_time = 0;
 184 
 185 /*
 186  * This tracks the legacy runlevel to ensure we signal init and manage
 187  * utmpx entries correctly.
 188  */
 189 static char current_runlevel = '\0';
 190 
 191 /* Number of single user threads currently running */
 192 static pthread_mutex_t single_user_thread_lock;
 193 static int single_user_thread_count = 0;
 194 
 195 /* Statistics for dependency cycle-checking */
 196 static u_longlong_t dep_inserts = 0;
 197 static u_longlong_t dep_cycle_ns = 0;
 198 static u_longlong_t dep_insert_ns = 0;
 199 
 200 
 201 static const char * const emsg_invalid_restarter =
 202         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 203         "(see 'svcs -xv' for details).\n";
 204 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 205 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 206 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 207 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 208 
 209 
 210 /*
 211  * These services define the system being "up".  If none of them can come
 212  * online, then we will run sulogin on the console.  Note that the install ones
 213  * are for the miniroot and when installing CDs after the first.  can_come_up()
 214  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 215  * started by dgraph_set_instance_state() or single_user_thread().
 216  *
 217  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 218  * entry, which is only used when booting_to_single_user (boot -s) is set.
 219  * This is because when doing a "boot -s", sulogin is started from specials.c
 220  * after milestone/single-user comes online, for backwards compatibility.
 221  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 222  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 223  */
 224 static const char * const up_svcs[] = {
 225         SCF_MILESTONE_SINGLE_USER,
 226         CONSOLE_LOGIN_FMRI,
 227         "svc:/system/install-setup:default",
 228         "svc:/system/install:default",
 229         NULL
 230 };
 231 
 232 /* This array must have an element for each non-NULL element of up_svcs[]. */
 233 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 234 
 235 /* These are for seed repository magic.  See can_come_up(). */
 236 static const char * const manifest_import =
 237         "svc:/system/manifest-import:default";
 238 static graph_vertex_t *manifest_import_p = NULL;
 239 
 240 
 241 static char target_milestone_as_runlevel(void);
 242 static void graph_runlevel_changed(char rl, int online);
 243 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 244 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 245 static int mark_subtree(graph_edge_t *, void *);
 246 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 247 
 248 /*
 249  * graph_vertex_compare()
 250  *      This function can compare either int *id or * graph_vertex_t *gv
 251  *      values, as the vertex id is always the first element of a
 252  *      graph_vertex structure.
 253  */
 254 /* ARGSUSED */
 255 static int
 256 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 257 {
 258         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 259         int rc_id = *(int *)rc_arg;
 260 
 261         if (lc_id > rc_id)
 262                 return (1);
 263         if (lc_id < rc_id)
 264                 return (-1);
 265         return (0);
 266 }
 267 
 268 void
 269 graph_init()
 270 {
 271         graph_edge_pool = startd_list_pool_create("graph_edges",
 272             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 273             UU_LIST_POOL_DEBUG);
 274         assert(graph_edge_pool != NULL);
 275 
 276         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 277             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 278             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 279         assert(graph_vertex_pool != NULL);
 280 
 281         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 282         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 283         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 284         assert(dgraph != NULL);
 285 
 286         if (!st->st_initial)
 287                 current_runlevel = utmpx_get_runlevel();
 288 
 289         log_framework(LOG_DEBUG, "Initialized graph\n");
 290 }
 291 
 292 static graph_vertex_t *
 293 vertex_get_by_name(const char *name)
 294 {
 295         int id;
 296 
 297         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 298 
 299         id = dict_lookup_byname(name);
 300         if (id == -1)
 301                 return (NULL);
 302 
 303         return (uu_list_find(dgraph, &id, NULL, NULL));
 304 }
 305 
 306 static graph_vertex_t *
 307 vertex_get_by_id(int id)
 308 {
 309         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 310 
 311         if (id == -1)
 312                 return (NULL);
 313 
 314         return (uu_list_find(dgraph, &id, NULL, NULL));
 315 }
 316 
 317 /*
 318  * Creates a new vertex with the given name, adds it to the graph, and returns
 319  * a pointer to it.  The graph lock must be held by this thread on entry.
 320  */
 321 static graph_vertex_t *
 322 graph_add_vertex(const char *name)
 323 {
 324         int id;
 325         graph_vertex_t *v;
 326         void *p;
 327         uu_list_index_t idx;
 328 
 329         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 330 
 331         id = dict_insert(name);
 332 
 333         v = startd_zalloc(sizeof (*v));
 334 
 335         v->gv_id = id;
 336 
 337         v->gv_name = startd_alloc(strlen(name) + 1);
 338         (void) strcpy(v->gv_name, name);
 339 
 340         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 341         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 342 
 343         p = uu_list_find(dgraph, &id, NULL, &idx);
 344         assert(p == NULL);
 345 
 346         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 347         uu_list_insert(dgraph, v, idx);
 348 
 349         return (v);
 350 }
 351 
 352 /*
 353  * Removes v from the graph and frees it.  The graph should be locked by this
 354  * thread, and v should have no edges associated with it.
 355  */
 356 static void
 357 graph_remove_vertex(graph_vertex_t *v)
 358 {
 359         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 360 
 361         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 362         assert(uu_list_numnodes(v->gv_dependents) == 0);
 363         assert(v->gv_refs == 0);
 364 
 365         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 366         uu_list_destroy(v->gv_dependencies);
 367         uu_list_destroy(v->gv_dependents);
 368         uu_list_remove(dgraph, v);
 369 
 370         startd_free(v, sizeof (graph_vertex_t));
 371 }
 372 
 373 static void
 374 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 375 {
 376         graph_edge_t *e, *re;
 377         int r;
 378 
 379         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 380 
 381         e = startd_alloc(sizeof (graph_edge_t));
 382         re = startd_alloc(sizeof (graph_edge_t));
 383 
 384         e->ge_parent = fv;
 385         e->ge_vertex = tv;
 386 
 387         re->ge_parent = tv;
 388         re->ge_vertex = fv;
 389 
 390         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 391         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 392         assert(r == 0);
 393 
 394         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 395         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 396         assert(r == 0);
 397 }
 398 
 399 static void
 400 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 401 {
 402         graph_edge_t *e;
 403 
 404         for (e = uu_list_first(v->gv_dependencies);
 405             e != NULL;
 406             e = uu_list_next(v->gv_dependencies, e)) {
 407                 if (e->ge_vertex == dv) {
 408                         uu_list_remove(v->gv_dependencies, e);
 409                         startd_free(e, sizeof (graph_edge_t));
 410                         break;
 411                 }
 412         }
 413 
 414         for (e = uu_list_first(dv->gv_dependents);
 415             e != NULL;
 416             e = uu_list_next(dv->gv_dependents, e)) {
 417                 if (e->ge_vertex == v) {
 418                         uu_list_remove(dv->gv_dependents, e);
 419                         startd_free(e, sizeof (graph_edge_t));
 420                         break;
 421                 }
 422         }
 423 }
 424 
 425 static void
 426 remove_inst_vertex(graph_vertex_t *v)
 427 {
 428         graph_edge_t *e;
 429         graph_vertex_t *sv;
 430         int i;
 431 
 432         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 433         assert(uu_list_numnodes(v->gv_dependents) == 1);
 434         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 435         assert(v->gv_refs == 0);
 436         assert((v->gv_flags & GV_CONFIGURED) == 0);
 437 
 438         e = uu_list_first(v->gv_dependents);
 439         sv = e->ge_vertex;
 440         graph_remove_edge(sv, v);
 441 
 442         for (i = 0; up_svcs[i] != NULL; ++i) {
 443                 if (up_svcs_p[i] == v)
 444                         up_svcs_p[i] = NULL;
 445         }
 446 
 447         if (manifest_import_p == v)
 448                 manifest_import_p = NULL;
 449 
 450         graph_remove_vertex(v);
 451 
 452         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 453             uu_list_numnodes(sv->gv_dependents) == 0 &&
 454             sv->gv_refs == 0)
 455                 graph_remove_vertex(sv);
 456 }
 457 
 458 static void
 459 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 460     void *arg)
 461 {
 462         graph_edge_t *e;
 463 
 464         for (e = uu_list_first(v->gv_dependents);
 465             e != NULL;
 466             e = uu_list_next(v->gv_dependents, e))
 467                 func(e->ge_vertex, arg);
 468 }
 469 
 470 static void
 471 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 472         void *), void *arg)
 473 {
 474         graph_edge_t *e;
 475 
 476         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 477 
 478         for (e = uu_list_first(v->gv_dependencies);
 479             e != NULL;
 480             e = uu_list_next(v->gv_dependencies, e)) {
 481 
 482                 func(e->ge_vertex, arg);
 483         }
 484 }
 485 
 486 /*
 487  * Generic graph walking function.
 488  *
 489  * Given a vertex, this function will walk either dependencies
 490  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 491  * for the entire graph.  It will avoid cycles and never visit the same vertex
 492  * twice.
 493  *
 494  * We avoid traversing exclusion dependencies, because they are allowed to
 495  * create cycles in the graph.  When propagating satisfiability, there is no
 496  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 497  * test for satisfiability.
 498  *
 499  * The walker takes two callbacks.  The first is called before examining the
 500  * dependents of each vertex.  The second is called on each vertex after
 501  * examining its dependents.  This allows is_path_to() to construct a path only
 502  * after the target vertex has been found.
 503  */
 504 typedef enum {
 505         WALK_DEPENDENTS,
 506         WALK_DEPENDENCIES
 507 } graph_walk_dir_t;
 508 
 509 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 510 
 511 typedef struct graph_walk_info {
 512         graph_walk_dir_t        gi_dir;
 513         uchar_t                 *gi_visited;    /* vertex bitmap */
 514         int                     (*gi_pre)(graph_vertex_t *, void *);
 515         void                    (*gi_post)(graph_vertex_t *, void *);
 516         void                    *gi_arg;        /* callback arg */
 517         int                     gi_ret;         /* return value */
 518 } graph_walk_info_t;
 519 
 520 static int
 521 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 522 {
 523         uu_list_t *list;
 524         int r;
 525         graph_vertex_t *v = e->ge_vertex;
 526         int i;
 527         uint_t b;
 528 
 529         i = v->gv_id / 8;
 530         b = 1 << (v->gv_id % 8);
 531 
 532         /*
 533          * Check to see if we've visited this vertex already.
 534          */
 535         if (gip->gi_visited[i] & b)
 536                 return (UU_WALK_NEXT);
 537 
 538         gip->gi_visited[i] |= b;
 539 
 540         /*
 541          * Don't follow exclusions.
 542          */
 543         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 544                 return (UU_WALK_NEXT);
 545 
 546         /*
 547          * Call pre-visit callback.  If this doesn't terminate the walk,
 548          * continue search.
 549          */
 550         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 551                 /*
 552                  * Recurse using appropriate list.
 553                  */
 554                 if (gip->gi_dir == WALK_DEPENDENTS)
 555                         list = v->gv_dependents;
 556                 else
 557                         list = v->gv_dependencies;
 558 
 559                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 560                     gip, 0);
 561                 assert(r == 0);
 562         }
 563 
 564         /*
 565          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 566          */
 567         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 568 
 569         /*
 570          * If given a post-callback, call the function for every vertex.
 571          */
 572         if (gip->gi_post != NULL)
 573                 (void) gip->gi_post(v, gip->gi_arg);
 574 
 575         /*
 576          * Preserve the callback's return value.  If the callback returns
 577          * UU_WALK_DONE, then we propagate that to the caller in order to
 578          * terminate the walk.
 579          */
 580         return (gip->gi_ret);
 581 }
 582 
 583 static void
 584 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 585     int (*pre)(graph_vertex_t *, void *),
 586     void (*post)(graph_vertex_t *, void *), void *arg)
 587 {
 588         graph_walk_info_t gi;
 589         graph_edge_t fake;
 590         size_t sz = dictionary->dict_new_id / 8 + 1;
 591 
 592         gi.gi_visited = startd_zalloc(sz);
 593         gi.gi_pre = pre;
 594         gi.gi_post = post;
 595         gi.gi_arg = arg;
 596         gi.gi_dir = dir;
 597         gi.gi_ret = 0;
 598 
 599         /*
 600          * Fake up an edge for the first iteration
 601          */
 602         fake.ge_vertex = v;
 603         (void) graph_walk_recurse(&fake, &gi);
 604 
 605         startd_free(gi.gi_visited, sz);
 606 }
 607 
 608 typedef struct child_search {
 609         int     id;             /* id of vertex to look for */
 610         uint_t  depth;          /* recursion depth */
 611         /*
 612          * While the vertex is not found, path is NULL.  After the search, if
 613          * the vertex was found then path should point to a -1-terminated
 614          * array of vertex id's which constitute the path to the vertex.
 615          */
 616         int     *path;
 617 } child_search_t;
 618 
 619 static int
 620 child_pre(graph_vertex_t *v, void *arg)
 621 {
 622         child_search_t *cs = arg;
 623 
 624         cs->depth++;
 625 
 626         if (v->gv_id == cs->id) {
 627                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 628                 cs->path[cs->depth] = -1;
 629                 return (UU_WALK_DONE);
 630         }
 631 
 632         return (UU_WALK_NEXT);
 633 }
 634 
 635 static void
 636 child_post(graph_vertex_t *v, void *arg)
 637 {
 638         child_search_t *cs = arg;
 639 
 640         cs->depth--;
 641 
 642         if (cs->path != NULL)
 643                 cs->path[cs->depth] = v->gv_id;
 644 }
 645 
 646 /*
 647  * Look for a path from from to to.  If one exists, returns a pointer to
 648  * a NULL-terminated array of pointers to the vertices along the path.  If
 649  * there is no path, returns NULL.
 650  */
 651 static int *
 652 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 653 {
 654         child_search_t cs;
 655 
 656         cs.id = to->gv_id;
 657         cs.depth = 0;
 658         cs.path = NULL;
 659 
 660         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 661 
 662         return (cs.path);
 663 }
 664 
 665 /*
 666  * Given an array of int's as returned by is_path_to, allocates a string of
 667  * their names joined by newlines.  Returns the size of the allocated buffer
 668  * in *sz and frees path.
 669  */
 670 static void
 671 path_to_str(int *path, char **cpp, size_t *sz)
 672 {
 673         int i;
 674         graph_vertex_t *v;
 675         size_t allocd, new_allocd;
 676         char *new, *name;
 677 
 678         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 679         assert(path[0] != -1);
 680 
 681         allocd = 1;
 682         *cpp = startd_alloc(1);
 683         (*cpp)[0] = '\0';
 684 
 685         for (i = 0; path[i] != -1; ++i) {
 686                 name = NULL;
 687 
 688                 v = vertex_get_by_id(path[i]);
 689 
 690                 if (v == NULL)
 691                         name = "<deleted>";
 692                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 693                         name = v->gv_name;
 694 
 695                 if (name != NULL) {
 696                         new_allocd = allocd + strlen(name) + 1;
 697                         new = startd_alloc(new_allocd);
 698                         (void) strcpy(new, *cpp);
 699                         (void) strcat(new, name);
 700                         (void) strcat(new, "\n");
 701 
 702                         startd_free(*cpp, allocd);
 703 
 704                         *cpp = new;
 705                         allocd = new_allocd;
 706                 }
 707         }
 708 
 709         startd_free(path, sizeof (int) * (i + 1));
 710 
 711         *sz = allocd;
 712 }
 713 
 714 
 715 /*
 716  * This function along with run_sulogin() implements an exclusion relationship
 717  * between system/console-login and sulogin.  run_sulogin() will fail if
 718  * system/console-login is online, and the graph engine should call
 719  * graph_clogin_start() to bring system/console-login online, which defers the
 720  * start if sulogin is running.
 721  */
 722 static void
 723 graph_clogin_start(graph_vertex_t *v)
 724 {
 725         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 726 
 727         if (sulogin_running)
 728                 console_login_ready = B_TRUE;
 729         else
 730                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 731 }
 732 
 733 static void
 734 graph_su_start(graph_vertex_t *v)
 735 {
 736         /*
 737          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 738          * entry with a runlevel of 'S', before jumping to the final
 739          * target runlevel (as set in initdefault).  We mimic that legacy
 740          * behavior here.
 741          */
 742         utmpx_set_runlevel('S', '0', B_FALSE);
 743         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 744 }
 745 
 746 static void
 747 graph_post_su_online(void)
 748 {
 749         graph_runlevel_changed('S', 1);
 750 }
 751 
 752 static void
 753 graph_post_su_disable(void)
 754 {
 755         graph_runlevel_changed('S', 0);
 756 }
 757 
 758 static void
 759 graph_post_mu_online(void)
 760 {
 761         graph_runlevel_changed('2', 1);
 762 }
 763 
 764 static void
 765 graph_post_mu_disable(void)
 766 {
 767         graph_runlevel_changed('2', 0);
 768 }
 769 
 770 static void
 771 graph_post_mus_online(void)
 772 {
 773         graph_runlevel_changed('3', 1);
 774 }
 775 
 776 static void
 777 graph_post_mus_disable(void)
 778 {
 779         graph_runlevel_changed('3', 0);
 780 }
 781 
 782 static struct special_vertex_info {
 783         const char      *name;
 784         void            (*start_f)(graph_vertex_t *);
 785         void            (*post_online_f)(void);
 786         void            (*post_disable_f)(void);
 787 } special_vertices[] = {
 788         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 789         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 790             graph_post_su_online, graph_post_su_disable },
 791         { SCF_MILESTONE_MULTI_USER, NULL,
 792             graph_post_mu_online, graph_post_mu_disable },
 793         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 794             graph_post_mus_online, graph_post_mus_disable },
 795         { NULL },
 796 };
 797 
 798 
 799 void
 800 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 801 {
 802         switch (e) {
 803         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 804                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 805 
 806                 MUTEX_LOCK(&st->st_load_lock);
 807                 st->st_load_instances++;
 808                 MUTEX_UNLOCK(&st->st_load_lock);
 809                 break;
 810 
 811         case RESTARTER_EVENT_TYPE_ENABLE:
 812                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 813                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 814                     v->gv_state == RESTARTER_STATE_DISABLED ||
 815                     v->gv_state == RESTARTER_STATE_MAINT);
 816                 break;
 817 
 818         case RESTARTER_EVENT_TYPE_DISABLE:
 819         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 820                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 821                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 822                 break;
 823 
 824         case RESTARTER_EVENT_TYPE_STOP:
 825                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 826                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 827                     v->gv_state == RESTARTER_STATE_ONLINE);
 828                 break;
 829 
 830         case RESTARTER_EVENT_TYPE_START:
 831                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 832                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 833                 break;
 834 
 835         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 836         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 837         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 838         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 839         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 840         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 841         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 842         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 843         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 844                 break;
 845 
 846         default:
 847 #ifndef NDEBUG
 848                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 849 #endif
 850                 abort();
 851         }
 852 
 853         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e);
 854 }
 855 
 856 static void
 857 graph_unset_restarter(graph_vertex_t *v)
 858 {
 859         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 860         assert(v->gv_flags & GV_CONFIGURED);
 861 
 862         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 863 
 864         if (v->gv_restarter_id != -1) {
 865                 graph_vertex_t *rv;
 866 
 867                 rv = vertex_get_by_id(v->gv_restarter_id);
 868                 graph_remove_edge(v, rv);
 869         }
 870 
 871         v->gv_restarter_id = -1;
 872         v->gv_restarter_channel = NULL;
 873 }
 874 
 875 /*
 876  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 877  * dgraph otherwise return VERTEX_INUSE.
 878  */
 879 static int
 880 free_if_unrefed(graph_vertex_t *v)
 881 {
 882         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 883 
 884         if (v->gv_refs > 0)
 885                 return (VERTEX_INUSE);
 886 
 887         if (v->gv_type == GVT_SVC &&
 888             uu_list_numnodes(v->gv_dependents) == 0 &&
 889             uu_list_numnodes(v->gv_dependencies) == 0) {
 890                 graph_remove_vertex(v);
 891                 return (VERTEX_REMOVED);
 892         } else if (v->gv_type == GVT_INST &&
 893             (v->gv_flags & GV_CONFIGURED) == 0 &&
 894             uu_list_numnodes(v->gv_dependents) == 1 &&
 895             uu_list_numnodes(v->gv_dependencies) == 0) {
 896                 remove_inst_vertex(v);
 897                 return (VERTEX_REMOVED);
 898         }
 899 
 900         return (VERTEX_INUSE);
 901 }
 902 
 903 static void
 904 delete_depgroup(graph_vertex_t *v)
 905 {
 906         graph_edge_t *e;
 907         graph_vertex_t *dv;
 908 
 909         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 910         assert(v->gv_type == GVT_GROUP);
 911         assert(uu_list_numnodes(v->gv_dependents) == 0);
 912 
 913         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 914                 dv = e->ge_vertex;
 915 
 916                 graph_remove_edge(v, dv);
 917 
 918                 switch (dv->gv_type) {
 919                 case GVT_INST:          /* instance dependency */
 920                 case GVT_SVC:           /* service dependency */
 921                         (void) free_if_unrefed(dv);
 922                         break;
 923 
 924                 case GVT_FILE:          /* file dependency */
 925                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 926                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 927                                 graph_remove_vertex(dv);
 928                         break;
 929 
 930                 default:
 931 #ifndef NDEBUG
 932                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 933                             __LINE__, dv->gv_type);
 934 #endif
 935                         abort();
 936                 }
 937         }
 938 
 939         graph_remove_vertex(v);
 940 }
 941 
 942 static int
 943 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
 944 {
 945         graph_vertex_t *v = ptrs[0];
 946         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
 947         graph_vertex_t *dv;
 948 
 949         dv = e->ge_vertex;
 950 
 951         /*
 952          * We have four possibilities here:
 953          *   - GVT_INST: restarter
 954          *   - GVT_GROUP - GVT_INST: instance dependency
 955          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
 956          *   - GVT_GROUP - GVT_FILE: file dependency
 957          */
 958         switch (dv->gv_type) {
 959         case GVT_INST:  /* restarter */
 960                 assert(dv->gv_id == v->gv_restarter_id);
 961                 if (delete_restarter_dep)
 962                         graph_remove_edge(v, dv);
 963                 break;
 964 
 965         case GVT_GROUP: /* pg dependency */
 966                 graph_remove_edge(v, dv);
 967                 delete_depgroup(dv);
 968                 break;
 969 
 970         case GVT_FILE:
 971                 /* These are currently not direct dependencies */
 972 
 973         default:
 974 #ifndef NDEBUG
 975                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
 976                     dv->gv_type);
 977 #endif
 978                 abort();
 979         }
 980 
 981         return (UU_WALK_NEXT);
 982 }
 983 
 984 static void
 985 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
 986 {
 987         void *ptrs[2];
 988         int r;
 989 
 990         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
 991         assert(v->gv_type == GVT_INST);
 992 
 993         ptrs[0] = v;
 994         ptrs[1] = (void *)delete_restarter_dep;
 995 
 996         r = uu_list_walk(v->gv_dependencies,
 997             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
 998         assert(r == 0);
 999 }
1000 
1001 /*
1002  * int graph_insert_vertex_unconfigured()
1003  *   Insert a vertex without sending any restarter events. If the vertex
1004  *   already exists or creation is successful, return a pointer to it in *vp.
1005  *
1006  *   If type is not GVT_GROUP, dt can remain unset.
1007  *
1008  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1009  *   doesn't agree with type, or type doesn't agree with dt).
1010  */
1011 static int
1012 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1013     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1014 {
1015         int r;
1016         int i;
1017 
1018         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1019 
1020         switch (type) {
1021         case GVT_SVC:
1022         case GVT_INST:
1023                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1024                         return (EINVAL);
1025                 break;
1026 
1027         case GVT_FILE:
1028                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1029                         return (EINVAL);
1030                 break;
1031 
1032         case GVT_GROUP:
1033                 if (dt <= 0 || rt < 0)
1034                         return (EINVAL);
1035                 break;
1036 
1037         default:
1038 #ifndef NDEBUG
1039                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1040 #endif
1041                 abort();
1042         }
1043 
1044         *vp = vertex_get_by_name(fmri);
1045         if (*vp != NULL)
1046                 return (EEXIST);
1047 
1048         *vp = graph_add_vertex(fmri);
1049 
1050         (*vp)->gv_type = type;
1051         (*vp)->gv_depgroup = dt;
1052         (*vp)->gv_restart = rt;
1053 
1054         (*vp)->gv_flags = 0;
1055         (*vp)->gv_state = RESTARTER_STATE_NONE;
1056 
1057         for (i = 0; special_vertices[i].name != NULL; ++i) {
1058                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1059                         (*vp)->gv_start_f = special_vertices[i].start_f;
1060                         (*vp)->gv_post_online_f =
1061                             special_vertices[i].post_online_f;
1062                         (*vp)->gv_post_disable_f =
1063                             special_vertices[i].post_disable_f;
1064                         break;
1065                 }
1066         }
1067 
1068         (*vp)->gv_restarter_id = -1;
1069         (*vp)->gv_restarter_channel = 0;
1070 
1071         if (type == GVT_INST) {
1072                 char *sfmri;
1073                 graph_vertex_t *sv;
1074 
1075                 sfmri = inst_fmri_to_svc_fmri(fmri);
1076                 sv = vertex_get_by_name(sfmri);
1077                 if (sv == NULL) {
1078                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1079                             0, &sv);
1080                         assert(r == 0);
1081                 }
1082                 startd_free(sfmri, max_scf_fmri_size);
1083 
1084                 graph_add_edge(sv, *vp);
1085         }
1086 
1087         /*
1088          * If this vertex is in the subgraph, mark it as so, for both
1089          * GVT_INST and GVT_SERVICE verteces.
1090          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1091          * depends on it, in which case it's already been added to the graph
1092          * and marked as in the subgraph (by refresh_vertex()).  If a
1093          * GVT_SERVICE vertex was freshly added (by the code above), it means
1094          * that it has no dependents, and cannot be in the subgraph.
1095          * Regardless of this, we still check that gv_flags includes
1096          * GV_INSUBGRAPH in the event that future behavior causes the above
1097          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1098          */
1099 
1100         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1101 
1102         return (0);
1103 }
1104 
1105 /*
1106  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1107  */
1108 static int
1109 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1110 {
1111         hrtime_t now;
1112 
1113         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1114 
1115         /* cycle detection */
1116         now = gethrtime();
1117 
1118         /* Don't follow exclusions. */
1119         if (!(fv->gv_type == GVT_GROUP &&
1120             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1121                 *pathp = is_path_to(tv, fv);
1122                 if (*pathp)
1123                         return (ELOOP);
1124         }
1125 
1126         dep_cycle_ns += gethrtime() - now;
1127         ++dep_inserts;
1128         now = gethrtime();
1129 
1130         graph_add_edge(fv, tv);
1131 
1132         dep_insert_ns += gethrtime() - now;
1133 
1134         /* Check if the dependency adds the "to" vertex to the subgraph */
1135         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1136 
1137         return (0);
1138 }
1139 
1140 static int
1141 inst_running(graph_vertex_t *v)
1142 {
1143         assert(v->gv_type == GVT_INST);
1144 
1145         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1146             v->gv_state == RESTARTER_STATE_DEGRADED)
1147                 return (1);
1148 
1149         return (0);
1150 }
1151 
1152 /*
1153  * The dependency evaluation functions return
1154  *   1 - dependency satisfied
1155  *   0 - dependency unsatisfied
1156  *   -1 - dependency unsatisfiable (without administrator intervention)
1157  *
1158  * The functions also take a boolean satbility argument.  When true, the
1159  * functions may recurse in order to determine satisfiability.
1160  */
1161 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1162 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1163 
1164 /*
1165  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1166  * is unsatisfiable if any elements are unsatisfiable.
1167  */
1168 static int
1169 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1170 {
1171         graph_edge_t *edge;
1172         int i;
1173         boolean_t any_unsatisfied;
1174 
1175         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1176                 return (1);
1177 
1178         any_unsatisfied = B_FALSE;
1179 
1180         for (edge = uu_list_first(groupv->gv_dependencies);
1181             edge != NULL;
1182             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1183                 i = dependency_satisfied(edge->ge_vertex, satbility);
1184                 if (i == 1)
1185                         continue;
1186 
1187                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1188                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1189                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1190 
1191                 if (!satbility)
1192                         return (0);
1193 
1194                 if (i == -1)
1195                         return (-1);
1196 
1197                 any_unsatisfied = B_TRUE;
1198         }
1199 
1200         return (any_unsatisfied ? 0 : 1);
1201 }
1202 
1203 /*
1204  * A require_any dependency is satisfied if any element is satisfied.  It is
1205  * satisfiable if any element is satisfiable.
1206  */
1207 static int
1208 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1209 {
1210         graph_edge_t *edge;
1211         int s;
1212         boolean_t satisfiable;
1213 
1214         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1215                 return (1);
1216 
1217         satisfiable = B_FALSE;
1218 
1219         for (edge = uu_list_first(groupv->gv_dependencies);
1220             edge != NULL;
1221             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1222                 s = dependency_satisfied(edge->ge_vertex, satbility);
1223 
1224                 if (s == 1)
1225                         return (1);
1226 
1227                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1228                     "require_any(%s): %s is unsatisfi%s.\n",
1229                     groupv->gv_name, edge->ge_vertex->gv_name,
1230                     s == 0 ? "ed" : "able");
1231 
1232                 if (satbility && s == 0)
1233                         satisfiable = B_TRUE;
1234         }
1235 
1236         return (!satbility || satisfiable ? 0 : -1);
1237 }
1238 
1239 /*
1240  * An optional_all dependency only considers elements which are configured,
1241  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1242  * is unsatisfied.
1243  *
1244  * Offline dependencies which are waiting for a dependency to come online are
1245  * unsatisfied.  Offline dependences which cannot possibly come online
1246  * (unsatisfiable) are always considered satisfied.
1247  */
1248 static int
1249 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1250 {
1251         graph_edge_t *edge;
1252         graph_vertex_t *v;
1253         boolean_t any_qualified;
1254         boolean_t any_unsatisfied;
1255         int i;
1256 
1257         any_qualified = B_FALSE;
1258         any_unsatisfied = B_FALSE;
1259 
1260         for (edge = uu_list_first(groupv->gv_dependencies);
1261             edge != NULL;
1262             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1263                 v = edge->ge_vertex;
1264 
1265                 switch (v->gv_type) {
1266                 case GVT_INST:
1267                         /* Skip missing or disabled instances */
1268                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1269                             (GV_CONFIGURED | GV_ENABLED))
1270                                 continue;
1271 
1272                         if (v->gv_state == RESTARTER_STATE_MAINT)
1273                                 continue;
1274 
1275                         if (v->gv_flags & GV_TOOFFLINE)
1276                                 continue;
1277 
1278                         any_qualified = B_TRUE;
1279                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1280                                 /*
1281                                  * For offline dependencies, treat unsatisfiable
1282                                  * as satisfied.
1283                                  */
1284                                 i = dependency_satisfied(v, B_TRUE);
1285                                 if (i == -1)
1286                                         i = 1;
1287                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1288                                 /*
1289                                  * The service is enabled, but hasn't
1290                                  * transitioned out of disabled yet.  Treat it
1291                                  * as unsatisfied (not unsatisfiable).
1292                                  */
1293                                 i = 0;
1294                         } else {
1295                                 i = dependency_satisfied(v, satbility);
1296                         }
1297                         break;
1298 
1299                 case GVT_FILE:
1300                         any_qualified = B_TRUE;
1301                         i = dependency_satisfied(v, satbility);
1302 
1303                         break;
1304 
1305                 case GVT_SVC: {
1306                         boolean_t svc_any_qualified;
1307                         boolean_t svc_satisfied;
1308                         boolean_t svc_satisfiable;
1309                         graph_vertex_t *v2;
1310                         graph_edge_t *e2;
1311 
1312                         svc_any_qualified = B_FALSE;
1313                         svc_satisfied = B_FALSE;
1314                         svc_satisfiable = B_FALSE;
1315 
1316                         for (e2 = uu_list_first(v->gv_dependencies);
1317                             e2 != NULL;
1318                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1319                                 v2 = e2->ge_vertex;
1320                                 assert(v2->gv_type == GVT_INST);
1321 
1322                                 if ((v2->gv_flags &
1323                                     (GV_CONFIGURED | GV_ENABLED)) !=
1324                                     (GV_CONFIGURED | GV_ENABLED))
1325                                         continue;
1326 
1327                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1328                                         continue;
1329 
1330                                 if (v2->gv_flags & GV_TOOFFLINE)
1331                                         continue;
1332 
1333                                 svc_any_qualified = B_TRUE;
1334 
1335                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1336                                         /*
1337                                          * For offline dependencies, treat
1338                                          * unsatisfiable as satisfied.
1339                                          */
1340                                         i = dependency_satisfied(v2, B_TRUE);
1341                                         if (i == -1)
1342                                                 i = 1;
1343                                 } else if (v2->gv_state ==
1344                                     RESTARTER_STATE_DISABLED) {
1345                                         i = 0;
1346                                 } else {
1347                                         i = dependency_satisfied(v2, satbility);
1348                                 }
1349 
1350                                 if (i == 1) {
1351                                         svc_satisfied = B_TRUE;
1352                                         break;
1353                                 }
1354                                 if (i == 0)
1355                                         svc_satisfiable = B_TRUE;
1356                         }
1357 
1358                         if (!svc_any_qualified)
1359                                 continue;
1360                         any_qualified = B_TRUE;
1361                         if (svc_satisfied) {
1362                                 i = 1;
1363                         } else if (svc_satisfiable) {
1364                                 i = 0;
1365                         } else {
1366                                 i = -1;
1367                         }
1368                         break;
1369                 }
1370 
1371                 case GVT_GROUP:
1372                 default:
1373 #ifndef NDEBUG
1374                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1375                             __LINE__, v->gv_type);
1376 #endif
1377                         abort();
1378                 }
1379 
1380                 if (i == 1)
1381                         continue;
1382 
1383                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1384                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1385                     v->gv_name, i == 0 ? "ed" : "able");
1386 
1387                 if (!satbility)
1388                         return (0);
1389                 if (i == -1)
1390                         return (-1);
1391                 any_unsatisfied = B_TRUE;
1392         }
1393 
1394         if (!any_qualified)
1395                 return (1);
1396 
1397         return (any_unsatisfied ? 0 : 1);
1398 }
1399 
1400 /*
1401  * An exclude_all dependency is unsatisfied if any non-service element is
1402  * satisfied or any service instance which is configured, enabled, and not in
1403  * maintenance is satisfied.  Usually when unsatisfied, it is also
1404  * unsatisfiable.
1405  */
1406 #define LOG_EXCLUDE(u, v)                                               \
1407         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1408             "exclude_all(%s): %s is satisfied.\n",                      \
1409             (u)->gv_name, (v)->gv_name)
1410 
1411 /* ARGSUSED */
1412 static int
1413 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1414 {
1415         graph_edge_t *edge, *e2;
1416         graph_vertex_t *v, *v2;
1417 
1418         for (edge = uu_list_first(groupv->gv_dependencies);
1419             edge != NULL;
1420             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1421                 v = edge->ge_vertex;
1422 
1423                 switch (v->gv_type) {
1424                 case GVT_INST:
1425                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1426                                 continue;
1427 
1428                         switch (v->gv_state) {
1429                         case RESTARTER_STATE_ONLINE:
1430                         case RESTARTER_STATE_DEGRADED:
1431                                 LOG_EXCLUDE(groupv, v);
1432                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1433 
1434                         case RESTARTER_STATE_OFFLINE:
1435                         case RESTARTER_STATE_UNINIT:
1436                                 LOG_EXCLUDE(groupv, v);
1437                                 return (0);
1438 
1439                         case RESTARTER_STATE_DISABLED:
1440                         case RESTARTER_STATE_MAINT:
1441                                 continue;
1442 
1443                         default:
1444 #ifndef NDEBUG
1445                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1446                                     __FILE__, __LINE__, v->gv_state);
1447 #endif
1448                                 abort();
1449                         }
1450                         /* NOTREACHED */
1451 
1452                 case GVT_SVC:
1453                         break;
1454 
1455                 case GVT_FILE:
1456                         if (!file_ready(v))
1457                                 continue;
1458                         LOG_EXCLUDE(groupv, v);
1459                         return (-1);
1460 
1461                 case GVT_GROUP:
1462                 default:
1463 #ifndef NDEBUG
1464                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1465                             __LINE__, v->gv_type);
1466 #endif
1467                         abort();
1468                 }
1469 
1470                 /* v represents a service */
1471                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1472                         continue;
1473 
1474                 for (e2 = uu_list_first(v->gv_dependencies);
1475                     e2 != NULL;
1476                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1477                         v2 = e2->ge_vertex;
1478                         assert(v2->gv_type == GVT_INST);
1479 
1480                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1481                                 continue;
1482 
1483                         switch (v2->gv_state) {
1484                         case RESTARTER_STATE_ONLINE:
1485                         case RESTARTER_STATE_DEGRADED:
1486                                 LOG_EXCLUDE(groupv, v2);
1487                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1488 
1489                         case RESTARTER_STATE_OFFLINE:
1490                         case RESTARTER_STATE_UNINIT:
1491                                 LOG_EXCLUDE(groupv, v2);
1492                                 return (0);
1493 
1494                         case RESTARTER_STATE_DISABLED:
1495                         case RESTARTER_STATE_MAINT:
1496                                 continue;
1497 
1498                         default:
1499 #ifndef NDEBUG
1500                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1501                                     __FILE__, __LINE__, v2->gv_type);
1502 #endif
1503                                 abort();
1504                         }
1505                 }
1506         }
1507 
1508         return (1);
1509 }
1510 
1511 /*
1512  * int instance_satisfied()
1513  *   Determine if all the dependencies are satisfied for the supplied instance
1514  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1515  *   without administrator intervention.
1516  */
1517 static int
1518 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1519 {
1520         assert(v->gv_type == GVT_INST);
1521         assert(!inst_running(v));
1522 
1523         return (require_all_satisfied(v, satbility));
1524 }
1525 
1526 /*
1527  * Decide whether v can satisfy a dependency.  v can either be a child of
1528  * a group vertex, or of an instance vertex.
1529  */
1530 static int
1531 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1532 {
1533         switch (v->gv_type) {
1534         case GVT_INST:
1535                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1536                         if (v->gv_flags & GV_DEATHROW) {
1537                                 /*
1538                                  * A dependency on an instance with GV_DEATHROW
1539                                  * flag is always considered as satisfied.
1540                                  */
1541                                 return (1);
1542                         }
1543                         return (-1);
1544                 }
1545 
1546                 /*
1547                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1548                  * to have its dependencies unsatisfiable.
1549                  */
1550                 if (v->gv_flags & GV_TOOFFLINE)
1551                         return (-1);
1552 
1553                 switch (v->gv_state) {
1554                 case RESTARTER_STATE_ONLINE:
1555                 case RESTARTER_STATE_DEGRADED:
1556                         return (1);
1557 
1558                 case RESTARTER_STATE_OFFLINE:
1559                         if (!satbility)
1560                                 return (0);
1561                         return (instance_satisfied(v, satbility) != -1 ?
1562                             0 : -1);
1563 
1564                 case RESTARTER_STATE_DISABLED:
1565                 case RESTARTER_STATE_MAINT:
1566                         return (-1);
1567 
1568                 case RESTARTER_STATE_UNINIT:
1569                         return (0);
1570 
1571                 default:
1572 #ifndef NDEBUG
1573                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1574                             __FILE__, __LINE__, v->gv_state);
1575 #endif
1576                         abort();
1577                         /* NOTREACHED */
1578                 }
1579 
1580         case GVT_SVC:
1581                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1582                         return (-1);
1583                 return (require_any_satisfied(v, satbility));
1584 
1585         case GVT_FILE:
1586                 /* i.e., we assume files will not be automatically generated */
1587                 return (file_ready(v) ? 1 : -1);
1588 
1589         case GVT_GROUP:
1590                 break;
1591 
1592         default:
1593 #ifndef NDEBUG
1594                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1595                     v->gv_type);
1596 #endif
1597                 abort();
1598                 /* NOTREACHED */
1599         }
1600 
1601         switch (v->gv_depgroup) {
1602         case DEPGRP_REQUIRE_ANY:
1603                 return (require_any_satisfied(v, satbility));
1604 
1605         case DEPGRP_REQUIRE_ALL:
1606                 return (require_all_satisfied(v, satbility));
1607 
1608         case DEPGRP_OPTIONAL_ALL:
1609                 return (optional_all_satisfied(v, satbility));
1610 
1611         case DEPGRP_EXCLUDE_ALL:
1612                 return (exclude_all_satisfied(v, satbility));
1613 
1614         default:
1615 #ifndef NDEBUG
1616                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1617                     __LINE__, v->gv_depgroup);
1618 #endif
1619                 abort();
1620         }
1621 }
1622 
1623 void
1624 graph_start_if_satisfied(graph_vertex_t *v)
1625 {
1626         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1627             instance_satisfied(v, B_FALSE) == 1) {
1628                 if (v->gv_start_f == NULL)
1629                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1630                 else
1631                         v->gv_start_f(v);
1632         }
1633 }
1634 
1635 /*
1636  * propagate_satbility()
1637  *
1638  * This function is used when the given vertex changes state in such a way that
1639  * one of its dependents may become unsatisfiable.  This happens when an
1640  * instance transitions between offline -> online, or from !running ->
1641  * maintenance, as well as when an instance is removed from the graph.
1642  *
1643  * We have to walk all the dependents, since optional_all dependencies several
1644  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1645  *
1646  *      +-----+  optional_all  +-----+  require_all  +-----+
1647  *      |  A  |--------------->|  B  |-------------->|  C  |
1648  *      +-----+                +-----+               +-----+
1649  *
1650  *                                              offline -> maintenance
1651  *
1652  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1653  * an optional dependency, what was previously an unsatisfiable situation is now
1654  * satisfied (B will never come online, even though its state hasn't changed).
1655  *
1656  * Note that it's not necessary to continue examining dependents after reaching
1657  * an optional_all dependency.  It's not possible for an optional_all dependency
1658  * to change satisfiability without also coming online, in which case we get a
1659  * start event and propagation continues naturally.  However, it does no harm to
1660  * continue propagating satisfiability (as it is a relatively rare event), and
1661  * keeps the walker code simple and generic.
1662  */
1663 /*ARGSUSED*/
1664 static int
1665 satbility_cb(graph_vertex_t *v, void *arg)
1666 {
1667         if (v->gv_type == GVT_INST)
1668                 graph_start_if_satisfied(v);
1669 
1670         return (UU_WALK_NEXT);
1671 }
1672 
1673 static void
1674 propagate_satbility(graph_vertex_t *v)
1675 {
1676         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1677 }
1678 
1679 static void propagate_stop(graph_vertex_t *, void *);
1680 
1681 /* ARGSUSED */
1682 static void
1683 propagate_start(graph_vertex_t *v, void *arg)
1684 {
1685         switch (v->gv_type) {
1686         case GVT_INST:
1687                 graph_start_if_satisfied(v);
1688                 break;
1689 
1690         case GVT_GROUP:
1691                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1692                         graph_walk_dependents(v, propagate_stop,
1693                             (void *)RERR_RESTART);
1694                         break;
1695                 }
1696                 /* FALLTHROUGH */
1697 
1698         case GVT_SVC:
1699                 graph_walk_dependents(v, propagate_start, NULL);
1700                 break;
1701 
1702         case GVT_FILE:
1703 #ifndef NDEBUG
1704                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1705                     __FILE__, __LINE__);
1706 #endif
1707                 abort();
1708                 /* NOTREACHED */
1709 
1710         default:
1711 #ifndef NDEBUG
1712                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1713                     v->gv_type);
1714 #endif
1715                 abort();
1716         }
1717 }
1718 
1719 static void
1720 propagate_stop(graph_vertex_t *v, void *arg)
1721 {
1722         graph_edge_t *e;
1723         graph_vertex_t *svc;
1724         restarter_error_t err = (restarter_error_t)arg;
1725 
1726         switch (v->gv_type) {
1727         case GVT_INST:
1728                 /* Restarter */
1729                 if (err > RERR_NONE && inst_running(v))
1730                         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1731                 break;
1732 
1733         case GVT_SVC:
1734                 graph_walk_dependents(v, propagate_stop, arg);
1735                 break;
1736 
1737         case GVT_FILE:
1738 #ifndef NDEBUG
1739                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1740                     __FILE__, __LINE__);
1741 #endif
1742                 abort();
1743                 /* NOTREACHED */
1744 
1745         case GVT_GROUP:
1746                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1747                         graph_walk_dependents(v, propagate_start, NULL);
1748                         break;
1749                 }
1750 
1751                 if (err == RERR_NONE || err > v->gv_restart)
1752                         break;
1753 
1754                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1755                 e = uu_list_first(v->gv_dependents);
1756                 svc = e->ge_vertex;
1757 
1758                 if (inst_running(svc))
1759                         vertex_send_event(svc, RESTARTER_EVENT_TYPE_STOP);
1760                 break;
1761 
1762         default:
1763 #ifndef NDEBUG
1764                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1765                     v->gv_type);
1766 #endif
1767                 abort();
1768         }
1769 }
1770 
1771 static void
1772 offline_vertex(graph_vertex_t *v)
1773 {
1774         scf_handle_t *h = libscf_handle_create_bound_loop();
1775         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1776         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1777         restarter_instance_state_t state, next_state;
1778         int r;
1779 
1780         assert(v->gv_type == GVT_INST);
1781 
1782         if (scf_inst == NULL)
1783                 bad_error("safe_scf_instance_create", scf_error());
1784         if (pg == NULL)
1785                 bad_error("safe_scf_pg_create", scf_error());
1786 
1787         /* if the vertex is already going offline, return */
1788 rep_retry:
1789         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1790             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1791                 switch (scf_error()) {
1792                 case SCF_ERROR_CONNECTION_BROKEN:
1793                         libscf_handle_rebind(h);
1794                         goto rep_retry;
1795 
1796                 case SCF_ERROR_NOT_FOUND:
1797                         scf_pg_destroy(pg);
1798                         scf_instance_destroy(scf_inst);
1799                         (void) scf_handle_unbind(h);
1800                         scf_handle_destroy(h);
1801                         return;
1802                 }
1803                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1804                     scf_strerror(scf_error()));
1805         }
1806 
1807         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1808         if (r != 0) {
1809                 switch (scf_error()) {
1810                 case SCF_ERROR_CONNECTION_BROKEN:
1811                         libscf_handle_rebind(h);
1812                         goto rep_retry;
1813 
1814                 case SCF_ERROR_NOT_SET:
1815                 case SCF_ERROR_NOT_FOUND:
1816                         scf_pg_destroy(pg);
1817                         scf_instance_destroy(scf_inst);
1818                         (void) scf_handle_unbind(h);
1819                         scf_handle_destroy(h);
1820                         return;
1821 
1822                 default:
1823                         bad_error("scf_instance_get_pg", scf_error());
1824                 }
1825         } else {
1826                 r = libscf_read_states(pg, &state, &next_state);
1827                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1828                     next_state == RESTARTER_STATE_DISABLED)) {
1829                         log_framework(LOG_DEBUG,
1830                             "%s: instance is already going down.\n",
1831                             v->gv_name);
1832                         scf_pg_destroy(pg);
1833                         scf_instance_destroy(scf_inst);
1834                         (void) scf_handle_unbind(h);
1835                         scf_handle_destroy(h);
1836                         return;
1837                 }
1838         }
1839 
1840         scf_pg_destroy(pg);
1841         scf_instance_destroy(scf_inst);
1842         (void) scf_handle_unbind(h);
1843         scf_handle_destroy(h);
1844 
1845         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1846 }
1847 
1848 /*
1849  * void graph_enable_by_vertex()
1850  *   If admin is non-zero, this is an administrative request for change
1851  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1852  *   a plain DISABLE restarter event.
1853  */
1854 void
1855 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1856 {
1857         graph_vertex_t *v;
1858         int r;
1859 
1860         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1861         assert((vertex->gv_flags & GV_CONFIGURED));
1862 
1863         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1864             (enable ? GV_ENABLED : 0);
1865 
1866         if (enable) {
1867                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1868                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1869                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1870                         /*
1871                          * In case the vertex was notified to go down,
1872                          * but now can return online, clear the _TOOFFLINE
1873                          * and _TODISABLE flags.
1874                          */
1875                         vertex->gv_flags &= ~GV_TOOFFLINE;
1876                         vertex->gv_flags &= ~GV_TODISABLE;
1877 
1878                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1879                 }
1880 
1881                 /*
1882                  * Wait for state update from restarter before sending _START or
1883                  * _STOP.
1884                  */
1885 
1886                 return;
1887         }
1888 
1889         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1890                 return;
1891 
1892         if (!admin) {
1893                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1894 
1895                 /*
1896                  * Wait for state update from restarter before sending _START or
1897                  * _STOP.
1898                  */
1899 
1900                 return;
1901         }
1902 
1903         /*
1904          * If it is a DISABLE event requested by the administrator then we are
1905          * offlining the dependents first.
1906          */
1907 
1908         /*
1909          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1910          * clear the GV_TOOFFLINE bits from all the services because
1911          * other DISABLE events might be handled at the same time.
1912          */
1913         vertex->gv_flags |= GV_TOOFFLINE;
1914 
1915         /* remember which vertex to disable... */
1916         vertex->gv_flags |= GV_TODISABLE;
1917 
1918         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1919             "disabling %s.\n", vertex->gv_name);
1920 
1921         /* set GV_TOOFFLINE for its dependents */
1922         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1923             NULL, 0);
1924         assert(r == 0);
1925 
1926         /* disable the instance now if there is nothing else to offline */
1927         if (insubtree_dependents_down(vertex) == B_TRUE) {
1928                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1929                 return;
1930         }
1931 
1932         /*
1933          * This loop is similar to the one used for the graph reversal shutdown
1934          * and could be improved in term of performance for the subtree reversal
1935          * disable case.
1936          */
1937         for (v = uu_list_first(dgraph); v != NULL;
1938             v = uu_list_next(dgraph, v)) {
1939                 /* skip the vertex we are disabling for now */
1940                 if (v == vertex)
1941                         continue;
1942 
1943                 if (v->gv_type != GVT_INST ||
1944                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1945                     (v->gv_flags & GV_ENABLED) == 0 ||
1946                     (v->gv_flags & GV_TOOFFLINE) == 0)
1947                         continue;
1948 
1949                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1950                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
1951                         /* continue if there is nothing to offline */
1952                         continue;
1953                 }
1954 
1955                 /*
1956                  * Instances which are up need to come down before we're
1957                  * done, but we can only offline the leaves here. An
1958                  * instance is a leaf when all its dependents are down.
1959                  */
1960                 if (insubtree_dependents_down(v) == B_TRUE) {
1961                         log_framework(LOG_DEBUG, "Offlining in-subtree "
1962                             "instance %s for %s.\n",
1963                             v->gv_name, vertex->gv_name);
1964                         offline_vertex(v);
1965                 }
1966         }
1967 }
1968 
1969 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
1970 
1971 /*
1972  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
1973  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
1974  * v is already configured and fmri_arg indicates the current restarter, do
1975  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
1976  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
1977  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
1978  * ECONNABORTED if the repository connection is broken, and ELOOP
1979  * if the dependency would create a cycle.  In the last case, *pathp will
1980  * point to a -1-terminated array of ids which compose the path from v to
1981  * restarter_fmri.
1982  */
1983 int
1984 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
1985     int **pathp)
1986 {
1987         char *restarter_fmri = NULL;
1988         graph_vertex_t *rv;
1989         int err;
1990         int id;
1991 
1992         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1993 
1994         if (fmri_arg[0] != '\0') {
1995                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
1996                 if (err != 0) {
1997                         assert(err == EINVAL);
1998                         return (err);
1999                 }
2000         }
2001 
2002         if (restarter_fmri == NULL ||
2003             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2004                 if (v->gv_flags & GV_CONFIGURED) {
2005                         if (v->gv_restarter_id == -1) {
2006                                 if (restarter_fmri != NULL)
2007                                         startd_free(restarter_fmri,
2008                                             max_scf_fmri_size);
2009                                 return (0);
2010                         }
2011 
2012                         graph_unset_restarter(v);
2013                 }
2014 
2015                 /* Master restarter, nothing to do. */
2016                 v->gv_restarter_id = -1;
2017                 v->gv_restarter_channel = NULL;
2018                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2019                 return (0);
2020         }
2021 
2022         if (v->gv_flags & GV_CONFIGURED) {
2023                 id = dict_lookup_byname(restarter_fmri);
2024                 if (id != -1 && v->gv_restarter_id == id) {
2025                         startd_free(restarter_fmri, max_scf_fmri_size);
2026                         return (0);
2027                 }
2028 
2029                 graph_unset_restarter(v);
2030         }
2031 
2032         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2033             RERR_NONE, &rv);
2034         startd_free(restarter_fmri, max_scf_fmri_size);
2035         assert(err == 0 || err == EEXIST);
2036 
2037         if (rv->gv_delegate_initialized == 0) {
2038                 rv->gv_delegate_channel = restarter_protocol_init_delegate(
2039                     rv->gv_name);
2040                 rv->gv_delegate_initialized = 1;
2041         }
2042         v->gv_restarter_id = rv->gv_id;
2043         v->gv_restarter_channel = rv->gv_delegate_channel;
2044 
2045         err = graph_insert_dependency(v, rv, pathp);
2046         if (err != 0) {
2047                 assert(err == ELOOP);
2048                 return (ELOOP);
2049         }
2050 
2051         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2052 
2053         if (!(rv->gv_flags & GV_CONFIGURED)) {
2054                 scf_instance_t *inst;
2055 
2056                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2057                 switch (err) {
2058                 case 0:
2059                         err = configure_vertex(rv, inst);
2060                         scf_instance_destroy(inst);
2061                         switch (err) {
2062                         case 0:
2063                         case ECANCELED:
2064                                 break;
2065 
2066                         case ECONNABORTED:
2067                                 return (ECONNABORTED);
2068 
2069                         default:
2070                                 bad_error("configure_vertex", err);
2071                         }
2072                         break;
2073 
2074                 case ECONNABORTED:
2075                         return (ECONNABORTED);
2076 
2077                 case ENOENT:
2078                         break;
2079 
2080                 case ENOTSUP:
2081                         /*
2082                          * The fmri doesn't specify an instance - translate
2083                          * to EINVAL.
2084                          */
2085                         return (EINVAL);
2086 
2087                 case EINVAL:
2088                 default:
2089                         bad_error("libscf_fmri_get_instance", err);
2090                 }
2091         }
2092 
2093         return (0);
2094 }
2095 
2096 
2097 /*
2098  * Add all of the instances of the service named by fmri to the graph.
2099  * Returns
2100  *   0 - success
2101  *   ENOENT - service indicated by fmri does not exist
2102  *
2103  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2104  * otherwise.
2105  */
2106 static int
2107 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2108 {
2109         scf_service_t *svc;
2110         scf_instance_t *inst;
2111         scf_iter_t *iter;
2112         char *inst_fmri;
2113         int ret, r;
2114 
2115         *reboundp = B_FALSE;
2116 
2117         svc = safe_scf_service_create(h);
2118         inst = safe_scf_instance_create(h);
2119         iter = safe_scf_iter_create(h);
2120         inst_fmri = startd_alloc(max_scf_fmri_size);
2121 
2122 rebound:
2123         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2124             SCF_DECODE_FMRI_EXACT) != 0) {
2125                 switch (scf_error()) {
2126                 case SCF_ERROR_CONNECTION_BROKEN:
2127                 default:
2128                         libscf_handle_rebind(h);
2129                         *reboundp = B_TRUE;
2130                         goto rebound;
2131 
2132                 case SCF_ERROR_NOT_FOUND:
2133                         ret = ENOENT;
2134                         goto out;
2135 
2136                 case SCF_ERROR_INVALID_ARGUMENT:
2137                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2138                 case SCF_ERROR_NOT_BOUND:
2139                 case SCF_ERROR_HANDLE_MISMATCH:
2140                         bad_error("scf_handle_decode_fmri", scf_error());
2141                 }
2142         }
2143 
2144         if (scf_iter_service_instances(iter, svc) != 0) {
2145                 switch (scf_error()) {
2146                 case SCF_ERROR_CONNECTION_BROKEN:
2147                 default:
2148                         libscf_handle_rebind(h);
2149                         *reboundp = B_TRUE;
2150                         goto rebound;
2151 
2152                 case SCF_ERROR_DELETED:
2153                         ret = ENOENT;
2154                         goto out;
2155 
2156                 case SCF_ERROR_HANDLE_MISMATCH:
2157                 case SCF_ERROR_NOT_BOUND:
2158                 case SCF_ERROR_NOT_SET:
2159                         bad_error("scf_iter_service_instances", scf_error());
2160                 }
2161         }
2162 
2163         for (;;) {
2164                 r = scf_iter_next_instance(iter, inst);
2165                 if (r == 0)
2166                         break;
2167                 if (r != 1) {
2168                         switch (scf_error()) {
2169                         case SCF_ERROR_CONNECTION_BROKEN:
2170                         default:
2171                                 libscf_handle_rebind(h);
2172                                 *reboundp = B_TRUE;
2173                                 goto rebound;
2174 
2175                         case SCF_ERROR_DELETED:
2176                                 ret = ENOENT;
2177                                 goto out;
2178 
2179                         case SCF_ERROR_HANDLE_MISMATCH:
2180                         case SCF_ERROR_NOT_BOUND:
2181                         case SCF_ERROR_NOT_SET:
2182                         case SCF_ERROR_INVALID_ARGUMENT:
2183                                 bad_error("scf_iter_next_instance",
2184                                     scf_error());
2185                         }
2186                 }
2187 
2188                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2189                     0) {
2190                         switch (scf_error()) {
2191                         case SCF_ERROR_CONNECTION_BROKEN:
2192                                 libscf_handle_rebind(h);
2193                                 *reboundp = B_TRUE;
2194                                 goto rebound;
2195 
2196                         case SCF_ERROR_DELETED:
2197                                 continue;
2198 
2199                         case SCF_ERROR_NOT_BOUND:
2200                         case SCF_ERROR_NOT_SET:
2201                                 bad_error("scf_instance_to_fmri", scf_error());
2202                         }
2203                 }
2204 
2205                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2206                 switch (r) {
2207                 case 0:
2208                 case ECANCELED:
2209                         break;
2210 
2211                 case EEXIST:
2212                         continue;
2213 
2214                 case ECONNABORTED:
2215                         libscf_handle_rebind(h);
2216                         *reboundp = B_TRUE;
2217                         goto rebound;
2218 
2219                 case EINVAL:
2220                 default:
2221                         bad_error("dgraph_add_instance", r);
2222                 }
2223         }
2224 
2225         ret = 0;
2226 
2227 out:
2228         startd_free(inst_fmri, max_scf_fmri_size);
2229         scf_iter_destroy(iter);
2230         scf_instance_destroy(inst);
2231         scf_service_destroy(svc);
2232         return (ret);
2233 }
2234 
2235 struct depfmri_info {
2236         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2237         gv_type_t       type;           /* type of dependency */
2238         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2239         const char      *pg_name;       /* Name of dependency pg */
2240         scf_handle_t    *h;
2241         int             err;            /* return error code */
2242         int             **pathp;        /* return circular dependency path */
2243 };
2244 
2245 /*
2246  * Find or create a vertex for fmri and make info->v depend on it.
2247  * Returns
2248  *   0 - success
2249  *   nonzero - failure
2250  *
2251  * On failure, sets info->err to
2252  *   EINVAL - fmri is invalid
2253  *            fmri does not match info->type
2254  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2255  *           will point to an array of the ids of the members of the cycle.
2256  *   ECONNABORTED - repository connection was broken
2257  *   ECONNRESET - succeeded, but repository connection was reset
2258  */
2259 static int
2260 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2261 {
2262         int err;
2263         graph_vertex_t *depgroup_v, *v;
2264         char *fmri_copy, *cfmri;
2265         size_t fmri_copy_sz;
2266         const char *scope, *service, *instance, *pg;
2267         scf_instance_t *inst;
2268         boolean_t rebound;
2269 
2270         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2271 
2272         /* Get or create vertex for FMRI */
2273         depgroup_v = info->v;
2274 
2275         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2276                 if (info->type != GVT_FILE) {
2277                         log_framework(LOG_NOTICE,
2278                             "FMRI \"%s\" is not allowed for the \"%s\" "
2279                             "dependency's type of instance %s.\n", fmri,
2280                             info->pg_name, info->inst_fmri);
2281                         return (info->err = EINVAL);
2282                 }
2283 
2284                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2285                     RERR_NONE, &v);
2286                 switch (err) {
2287                 case 0:
2288                         break;
2289 
2290                 case EEXIST:
2291                         assert(v->gv_type == GVT_FILE);
2292                         break;
2293 
2294                 case EINVAL:            /* prevented above */
2295                 default:
2296                         bad_error("graph_insert_vertex_unconfigured", err);
2297                 }
2298         } else {
2299                 if (info->type != GVT_INST) {
2300                         log_framework(LOG_NOTICE,
2301                             "FMRI \"%s\" is not allowed for the \"%s\" "
2302                             "dependency's type of instance %s.\n", fmri,
2303                             info->pg_name, info->inst_fmri);
2304                         return (info->err = EINVAL);
2305                 }
2306 
2307                 /*
2308                  * We must canonify fmri & add a vertex for it.
2309                  */
2310                 fmri_copy_sz = strlen(fmri) + 1;
2311                 fmri_copy = startd_alloc(fmri_copy_sz);
2312                 (void) strcpy(fmri_copy, fmri);
2313 
2314                 /* Determine if the FMRI is a property group or instance */
2315                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2316                     &instance, &pg, NULL) != 0) {
2317                         startd_free(fmri_copy, fmri_copy_sz);
2318                         log_framework(LOG_NOTICE,
2319                             "Dependency \"%s\" of %s has invalid FMRI "
2320                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2321                             fmri);
2322                         return (info->err = EINVAL);
2323                 }
2324 
2325                 if (service == NULL || pg != NULL) {
2326                         startd_free(fmri_copy, fmri_copy_sz);
2327                         log_framework(LOG_NOTICE,
2328                             "Dependency \"%s\" of %s does not designate a "
2329                             "service or instance.\n", info->pg_name,
2330                             info->inst_fmri);
2331                         return (info->err = EINVAL);
2332                 }
2333 
2334                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2335                         cfmri = uu_msprintf("svc:/%s%s%s",
2336                             service, instance ? ":" : "", instance ? instance :
2337                             "");
2338                 } else {
2339                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2340                             scope, service, instance ? ":" : "", instance ?
2341                             instance : "");
2342                 }
2343 
2344                 startd_free(fmri_copy, fmri_copy_sz);
2345 
2346                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2347                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2348                     RERR_NONE, &v);
2349                 uu_free(cfmri);
2350                 switch (err) {
2351                 case 0:
2352                         break;
2353 
2354                 case EEXIST:
2355                         /* Verify v. */
2356                         if (instance != NULL)
2357                                 assert(v->gv_type == GVT_INST);
2358                         else
2359                                 assert(v->gv_type == GVT_SVC);
2360                         break;
2361 
2362                 default:
2363                         bad_error("graph_insert_vertex_unconfigured", err);
2364                 }
2365         }
2366 
2367         /* Add dependency from depgroup_v to new vertex */
2368         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2369         switch (info->err) {
2370         case 0:
2371                 break;
2372 
2373         case ELOOP:
2374                 return (ELOOP);
2375 
2376         default:
2377                 bad_error("graph_insert_dependency", info->err);
2378         }
2379 
2380         /* This must be after we insert the dependency, to avoid looping. */
2381         switch (v->gv_type) {
2382         case GVT_INST:
2383                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2384                         break;
2385 
2386                 inst = safe_scf_instance_create(info->h);
2387 
2388                 rebound = B_FALSE;
2389 
2390 rebound:
2391                 err = libscf_lookup_instance(v->gv_name, inst);
2392                 switch (err) {
2393                 case 0:
2394                         err = configure_vertex(v, inst);
2395                         switch (err) {
2396                         case 0:
2397                         case ECANCELED:
2398                                 break;
2399 
2400                         case ECONNABORTED:
2401                                 libscf_handle_rebind(info->h);
2402                                 rebound = B_TRUE;
2403                                 goto rebound;
2404 
2405                         default:
2406                                 bad_error("configure_vertex", err);
2407                         }
2408                         break;
2409 
2410                 case ENOENT:
2411                         break;
2412 
2413                 case ECONNABORTED:
2414                         libscf_handle_rebind(info->h);
2415                         rebound = B_TRUE;
2416                         goto rebound;
2417 
2418                 case EINVAL:
2419                 case ENOTSUP:
2420                 default:
2421                         bad_error("libscf_fmri_get_instance", err);
2422                 }
2423 
2424                 scf_instance_destroy(inst);
2425 
2426                 if (rebound)
2427                         return (info->err = ECONNRESET);
2428                 break;
2429 
2430         case GVT_SVC:
2431                 (void) add_service(v->gv_name, info->h, &rebound);
2432                 if (rebound)
2433                         return (info->err = ECONNRESET);
2434         }
2435 
2436         return (0);
2437 }
2438 
2439 struct deppg_info {
2440         graph_vertex_t  *v;             /* GVT_INST vertex */
2441         int             err;            /* return error */
2442         int             **pathp;        /* return circular dependency path */
2443 };
2444 
2445 /*
2446  * Make info->v depend on a new GVT_GROUP node for this property group,
2447  * and then call process_dependency_fmri() for the values of the entity
2448  * property.  Return 0 on success, or if something goes wrong return nonzero
2449  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2450  * process_dependency_fmri().
2451  */
2452 static int
2453 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2454 {
2455         scf_handle_t *h;
2456         depgroup_type_t deptype;
2457         restarter_error_t rerr;
2458         struct depfmri_info linfo;
2459         char *fmri, *pg_name;
2460         size_t fmri_sz;
2461         graph_vertex_t *depgrp;
2462         scf_property_t *prop;
2463         int err;
2464         int empty;
2465         scf_error_t scferr;
2466         ssize_t len;
2467 
2468         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2469 
2470         h = scf_pg_handle(pg);
2471 
2472         pg_name = startd_alloc(max_scf_name_size);
2473 
2474         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2475         if (len < 0) {
2476                 startd_free(pg_name, max_scf_name_size);
2477                 switch (scf_error()) {
2478                 case SCF_ERROR_CONNECTION_BROKEN:
2479                 default:
2480                         return (info->err = ECONNABORTED);
2481 
2482                 case SCF_ERROR_DELETED:
2483                         return (info->err = 0);
2484 
2485                 case SCF_ERROR_NOT_SET:
2486                         bad_error("scf_pg_get_name", scf_error());
2487                 }
2488         }
2489 
2490         /*
2491          * Skip over empty dependency groups.  Since dependency property
2492          * groups are updated atomically, they are either empty or
2493          * fully populated.
2494          */
2495         empty = depgroup_empty(h, pg);
2496         if (empty < 0) {
2497                 log_error(LOG_INFO,
2498                     "Error reading dependency group \"%s\" of %s: %s\n",
2499                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2500                 startd_free(pg_name, max_scf_name_size);
2501                 return (info->err = EINVAL);
2502 
2503         } else if (empty == 1) {
2504                 log_framework(LOG_DEBUG,
2505                     "Ignoring empty dependency group \"%s\" of %s\n",
2506                     pg_name, info->v->gv_name);
2507                 startd_free(pg_name, max_scf_name_size);
2508                 return (info->err = 0);
2509         }
2510 
2511         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2512         fmri = startd_alloc(fmri_sz);
2513 
2514         (void) snprintf(fmri, max_scf_name_size, "%s>%s", info->v->gv_name,
2515             pg_name);
2516 
2517         /* Validate the pg before modifying the graph */
2518         deptype = depgroup_read_grouping(h, pg);
2519         if (deptype == DEPGRP_UNSUPPORTED) {
2520                 log_error(LOG_INFO,
2521                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2522                     pg_name, info->v->gv_name);
2523                 startd_free(fmri, fmri_sz);
2524                 startd_free(pg_name, max_scf_name_size);
2525                 return (info->err = EINVAL);
2526         }
2527 
2528         rerr = depgroup_read_restart(h, pg);
2529         if (rerr == RERR_UNSUPPORTED) {
2530                 log_error(LOG_INFO,
2531                     "Dependency \"%s\" of %s has an unknown restart_on value."
2532                     "\n", pg_name, info->v->gv_name);
2533                 startd_free(fmri, fmri_sz);
2534                 startd_free(pg_name, max_scf_name_size);
2535                 return (info->err = EINVAL);
2536         }
2537 
2538         prop = safe_scf_property_create(h);
2539 
2540         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2541                 scferr = scf_error();
2542                 scf_property_destroy(prop);
2543                 if (scferr == SCF_ERROR_DELETED) {
2544                         startd_free(fmri, fmri_sz);
2545                         startd_free(pg_name, max_scf_name_size);
2546                         return (info->err = 0);
2547                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2548                         startd_free(fmri, fmri_sz);
2549                         startd_free(pg_name, max_scf_name_size);
2550                         return (info->err = ECONNABORTED);
2551                 }
2552 
2553                 log_error(LOG_INFO,
2554                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2555                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2556 
2557                 startd_free(fmri, fmri_sz);
2558                 startd_free(pg_name, max_scf_name_size);
2559 
2560                 return (info->err = EINVAL);
2561         }
2562 
2563         /* Create depgroup vertex for pg */
2564         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2565             rerr, &depgrp);
2566         assert(err == 0);
2567         startd_free(fmri, fmri_sz);
2568 
2569         /* Add dependency from inst vertex to new vertex */
2570         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2571         /* ELOOP can't happen because this should be a new vertex */
2572         assert(err == 0);
2573 
2574         linfo.v = depgrp;
2575         linfo.type = depgroup_read_scheme(h, pg);
2576         linfo.inst_fmri = info->v->gv_name;
2577         linfo.pg_name = pg_name;
2578         linfo.h = h;
2579         linfo.err = 0;
2580         linfo.pathp = info->pathp;
2581         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2582             &linfo);
2583 
2584         scf_property_destroy(prop);
2585         startd_free(pg_name, max_scf_name_size);
2586 
2587         switch (err) {
2588         case 0:
2589         case EINTR:
2590                 return (info->err = linfo.err);
2591 
2592         case ECONNABORTED:
2593         case EINVAL:
2594                 return (info->err = err);
2595 
2596         case ECANCELED:
2597                 return (info->err = 0);
2598 
2599         case ECONNRESET:
2600                 return (info->err = ECONNABORTED);
2601 
2602         default:
2603                 bad_error("walk_property_astrings", err);
2604                 /* NOTREACHED */
2605         }
2606 }
2607 
2608 /*
2609  * Build the dependency info for v from the repository.  Returns 0 on success,
2610  * ECONNABORTED on repository disconnection, EINVAL if the repository
2611  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2612  * In the last case, *pathp will point to a -1-terminated array of ids which
2613  * constitute the rest of the dependency cycle.
2614  */
2615 static int
2616 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2617 {
2618         struct deppg_info info;
2619         int err;
2620         uint_t old_configured;
2621 
2622         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2623 
2624         /*
2625          * Mark the vertex as configured during dependency insertion to avoid
2626          * dependency cycles (which can appear in the graph if one of the
2627          * vertices is an exclusion-group).
2628          */
2629         old_configured = v->gv_flags & GV_CONFIGURED;
2630         v->gv_flags |= GV_CONFIGURED;
2631 
2632         info.err = 0;
2633         info.v = v;
2634         info.pathp = pathp;
2635 
2636         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2637             &info);
2638 
2639         if (!old_configured)
2640                 v->gv_flags &= ~GV_CONFIGURED;
2641 
2642         switch (err) {
2643         case 0:
2644         case EINTR:
2645                 return (info.err);
2646 
2647         case ECONNABORTED:
2648                 return (ECONNABORTED);
2649 
2650         case ECANCELED:
2651                 /* Should get delete event, so return 0. */
2652                 return (0);
2653 
2654         default:
2655                 bad_error("walk_dependency_pgs", err);
2656                 /* NOTREACHED */
2657         }
2658 }
2659 
2660 
2661 static void
2662 handle_cycle(const char *fmri, int *path)
2663 {
2664         const char *cp;
2665         size_t sz;
2666 
2667         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2668 
2669         path_to_str(path, (char **)&cp, &sz);
2670 
2671         log_error(LOG_ERR, "Transitioning %s to maintenance "
2672             "because it completes a dependency cycle (see svcs -xv for "
2673             "details):\n%s", fmri ? fmri : "?", cp);
2674 
2675         startd_free((void *)cp, sz);
2676 }
2677 
2678 /*
2679  * Increment the vertex's reference count to prevent the vertex removal
2680  * from the dgraph.
2681  */
2682 static void
2683 vertex_ref(graph_vertex_t *v)
2684 {
2685         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2686 
2687         v->gv_refs++;
2688 }
2689 
2690 /*
2691  * Decrement the vertex's reference count and remove the vertex from
2692  * the dgraph when possible.
2693  *
2694  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2695  * return VERTEX_INUSE.
2696  */
2697 static int
2698 vertex_unref(graph_vertex_t *v)
2699 {
2700         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2701         assert(v->gv_refs > 0);
2702 
2703         v->gv_refs--;
2704 
2705         return (free_if_unrefed(v));
2706 }
2707 
2708 /*
2709  * When run on the dependencies of a vertex, populates list with
2710  * graph_edge_t's which point to the service vertices or the instance
2711  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2712  *
2713  * Increment the vertex's reference count once the vertex is inserted
2714  * in the list. The vertex won't be able to be deleted from the dgraph
2715  * while it is referenced.
2716  */
2717 static int
2718 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2719 {
2720         graph_vertex_t *v = e->ge_vertex;
2721         graph_edge_t *new;
2722         int r;
2723 
2724         switch (v->gv_type) {
2725         case GVT_INST:
2726         case GVT_SVC:
2727                 break;
2728 
2729         case GVT_GROUP:
2730                 r = uu_list_walk(v->gv_dependencies,
2731                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2732                 assert(r == 0);
2733                 return (UU_WALK_NEXT);
2734 
2735         case GVT_FILE:
2736                 return (UU_WALK_NEXT);
2737 
2738         default:
2739 #ifndef NDEBUG
2740                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2741                     __LINE__, v->gv_type);
2742 #endif
2743                 abort();
2744         }
2745 
2746         new = startd_alloc(sizeof (*new));
2747         new->ge_vertex = v;
2748         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2749         r = uu_list_insert_before(list, NULL, new);
2750         assert(r == 0);
2751 
2752         /*
2753          * Because we are inserting the vertex in a list, we don't want
2754          * the vertex to be freed while the list is in use. In order to
2755          * achieve that, increment the vertex's reference count.
2756          */
2757         vertex_ref(v);
2758 
2759         return (UU_WALK_NEXT);
2760 }
2761 
2762 static boolean_t
2763 should_be_in_subgraph(graph_vertex_t *v)
2764 {
2765         graph_edge_t *e;
2766 
2767         if (v == milestone)
2768                 return (B_TRUE);
2769 
2770         /*
2771          * v is in the subgraph if any of its dependents are in the subgraph.
2772          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2773          * count if we're enabled.
2774          */
2775         for (e = uu_list_first(v->gv_dependents);
2776             e != NULL;
2777             e = uu_list_next(v->gv_dependents, e)) {
2778                 graph_vertex_t *dv = e->ge_vertex;
2779 
2780                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2781                         continue;
2782 
2783                 /*
2784                  * Don't include instances that are optional and disabled.
2785                  */
2786                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2787 
2788                         int in = 0;
2789                         graph_edge_t *ee;
2790 
2791                         for (ee = uu_list_first(dv->gv_dependents);
2792                             ee != NULL;
2793                             ee = uu_list_next(dv->gv_dependents, ee)) {
2794 
2795                                 graph_vertex_t *ddv = e->ge_vertex;
2796 
2797                                 if (ddv->gv_type == GVT_GROUP &&
2798                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2799                                         continue;
2800 
2801                                 if (ddv->gv_type == GVT_GROUP &&
2802                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2803                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2804                                         continue;
2805 
2806                                 in = 1;
2807                         }
2808                         if (!in)
2809                                 continue;
2810                 }
2811                 if (v->gv_type == GVT_INST &&
2812                     dv->gv_type == GVT_GROUP &&
2813                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2814                     !(v->gv_flags & GV_ENBLD_NOOVR))
2815                         continue;
2816 
2817                 /* Don't include excluded services and instances */
2818                 if (dv->gv_type == GVT_GROUP &&
2819                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2820                         continue;
2821 
2822                 return (B_TRUE);
2823         }
2824 
2825         return (B_FALSE);
2826 }
2827 
2828 /*
2829  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2830  * any bits change, manipulate the repository appropriately.  Returns 0 or
2831  * ECONNABORTED.
2832  */
2833 static int
2834 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2835 {
2836         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2837         boolean_t new;
2838         graph_edge_t *e;
2839         scf_instance_t *inst;
2840         int ret = 0, r;
2841 
2842         assert(milestone != NULL && milestone != MILESTONE_NONE);
2843 
2844         new = should_be_in_subgraph(v);
2845 
2846         if (new == old)
2847                 return (0);
2848 
2849         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2850             "Removing %s from the subgraph.\n", v->gv_name);
2851 
2852         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2853             (new ? GV_INSUBGRAPH : 0);
2854 
2855         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2856                 int err;
2857 
2858 get_inst:
2859                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2860                 if (err != 0) {
2861                         switch (err) {
2862                         case ECONNABORTED:
2863                                 libscf_handle_rebind(h);
2864                                 ret = ECONNABORTED;
2865                                 goto get_inst;
2866 
2867                         case ENOENT:
2868                                 break;
2869 
2870                         case EINVAL:
2871                         case ENOTSUP:
2872                         default:
2873                                 bad_error("libscf_fmri_get_instance", err);
2874                         }
2875                 } else {
2876                         const char *f;
2877 
2878                         if (new) {
2879                                 err = libscf_delete_enable_ovr(inst);
2880                                 f = "libscf_delete_enable_ovr";
2881                         } else {
2882                                 err = libscf_set_enable_ovr(inst, 0);
2883                                 f = "libscf_set_enable_ovr";
2884                         }
2885                         scf_instance_destroy(inst);
2886                         switch (err) {
2887                         case 0:
2888                         case ECANCELED:
2889                                 break;
2890 
2891                         case ECONNABORTED:
2892                                 libscf_handle_rebind(h);
2893                                 /*
2894                                  * We must continue so the graph is updated,
2895                                  * but we must return ECONNABORTED so any
2896                                  * libscf state held by any callers is reset.
2897                                  */
2898                                 ret = ECONNABORTED;
2899                                 goto get_inst;
2900 
2901                         case EROFS:
2902                         case EPERM:
2903                                 log_error(LOG_WARNING,
2904                                     "Could not set %s/%s for %s: %s.\n",
2905                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2906                                     v->gv_name, strerror(err));
2907                                 break;
2908 
2909                         default:
2910                                 bad_error(f, err);
2911                         }
2912                 }
2913         }
2914 
2915         for (e = uu_list_first(v->gv_dependencies);
2916             e != NULL;
2917             e = uu_list_next(v->gv_dependencies, e)) {
2918                 r = eval_subgraph(e->ge_vertex, h);
2919                 if (r != 0) {
2920                         assert(r == ECONNABORTED);
2921                         ret = ECONNABORTED;
2922                 }
2923         }
2924 
2925         return (ret);
2926 }
2927 
2928 /*
2929  * Delete the (property group) dependencies of v & create new ones based on
2930  * inst.  If doing so would create a cycle, log a message and put the instance
2931  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2932  * ECONNABORTED.
2933  */
2934 int
2935 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2936 {
2937         int err;
2938         int *path;
2939         char *fmri;
2940         int r;
2941         scf_handle_t *h = scf_instance_handle(inst);
2942         uu_list_t *old_deps;
2943         int ret = 0;
2944         graph_edge_t *e;
2945         graph_vertex_t *vv;
2946 
2947         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2948         assert(v->gv_type == GVT_INST);
2949 
2950         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
2951 
2952         if (milestone > MILESTONE_NONE) {
2953                 /*
2954                  * In case some of v's dependencies are being deleted we must
2955                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
2956                  * after the new dependencies are in place.
2957                  */
2958                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
2959 
2960                 err = uu_list_walk(v->gv_dependencies,
2961                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
2962                 assert(err == 0);
2963         }
2964 
2965         delete_instance_dependencies(v, B_FALSE);
2966 
2967         err = set_dependencies(v, inst, &path);
2968         switch (err) {
2969         case 0:
2970                 break;
2971 
2972         case ECONNABORTED:
2973                 ret = err;
2974                 goto out;
2975 
2976         case EINVAL:
2977         case ELOOP:
2978                 r = libscf_instance_get_fmri(inst, &fmri);
2979                 switch (r) {
2980                 case 0:
2981                         break;
2982 
2983                 case ECONNABORTED:
2984                         ret = ECONNABORTED;
2985                         goto out;
2986 
2987                 case ECANCELED:
2988                         ret = 0;
2989                         goto out;
2990 
2991                 default:
2992                         bad_error("libscf_instance_get_fmri", r);
2993                 }
2994 
2995                 if (err == EINVAL) {
2996                         log_error(LOG_ERR, "Transitioning %s "
2997                             "to maintenance due to misconfiguration.\n",
2998                             fmri ? fmri : "?");
2999                         vertex_send_event(v,
3000                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3001                 } else {
3002                         handle_cycle(fmri, path);
3003                         vertex_send_event(v,
3004                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3005                 }
3006                 startd_free(fmri, max_scf_fmri_size);
3007                 ret = 0;
3008                 goto out;
3009 
3010         default:
3011                 bad_error("set_dependencies", err);
3012         }
3013 
3014         if (milestone > MILESTONE_NONE) {
3015                 boolean_t aborted = B_FALSE;
3016 
3017                 for (e = uu_list_first(old_deps);
3018                     e != NULL;
3019                     e = uu_list_next(old_deps, e)) {
3020                         vv = e->ge_vertex;
3021 
3022                         if (vertex_unref(vv) == VERTEX_INUSE &&
3023                             eval_subgraph(vv, h) == ECONNABORTED)
3024                                 aborted = B_TRUE;
3025                 }
3026 
3027                 for (e = uu_list_first(v->gv_dependencies);
3028                     e != NULL;
3029                     e = uu_list_next(v->gv_dependencies, e)) {
3030                         if (eval_subgraph(e->ge_vertex, h) ==
3031                             ECONNABORTED)
3032                                 aborted = B_TRUE;
3033                 }
3034 
3035                 if (aborted) {
3036                         ret = ECONNABORTED;
3037                         goto out;
3038                 }
3039         }
3040 
3041         graph_start_if_satisfied(v);
3042 
3043         ret = 0;
3044 
3045 out:
3046         if (milestone > MILESTONE_NONE) {
3047                 void *cookie = NULL;
3048 
3049                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3050                         startd_free(e, sizeof (*e));
3051 
3052                 uu_list_destroy(old_deps);
3053         }
3054 
3055         return (ret);
3056 }
3057 
3058 /*
3059  * Set up v according to inst.  That is, make sure it depends on its
3060  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3061  * the restarter, and send ENABLE or DISABLE as appropriate.
3062  *
3063  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3064  * ECANCELED if inst is deleted.
3065  */
3066 static int
3067 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3068 {
3069         scf_handle_t *h;
3070         scf_propertygroup_t *pg;
3071         scf_snapshot_t *snap;
3072         char *restarter_fmri = startd_alloc(max_scf_value_size);
3073         int enabled, enabled_ovr;
3074         int err;
3075         int *path;
3076         int deathrow;
3077 
3078         restarter_fmri[0] = '\0';
3079 
3080         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3081         assert(v->gv_type == GVT_INST);
3082         assert((v->gv_flags & GV_CONFIGURED) == 0);
3083 
3084         /* GV_INSUBGRAPH should already be set properly. */
3085         assert(should_be_in_subgraph(v) ==
3086             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3087 
3088         /*
3089          * If the instance fmri is in the deathrow list then set the
3090          * GV_DEATHROW flag on the vertex and create and set to true the
3091          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3092          * repository for this instance fmri.
3093          */
3094         if ((v->gv_flags & GV_DEATHROW) ||
3095             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3096                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3097                         /*
3098                          * Set flag GV_DEATHROW, create and set to true
3099                          * the SCF_PROPERTY_DEATHROW property in the
3100                          * non-persistent repository for this instance fmri.
3101                          */
3102                         v->gv_flags |= GV_DEATHROW;
3103 
3104                         switch (err = libscf_set_deathrow(inst, 1)) {
3105                         case 0:
3106                                 break;
3107 
3108                         case ECONNABORTED:
3109                         case ECANCELED:
3110                                 startd_free(restarter_fmri, max_scf_value_size);
3111                                 return (err);
3112 
3113                         case EROFS:
3114                                 log_error(LOG_WARNING, "Could not set %s/%s "
3115                                     "for deathrow %s: %s.\n",
3116                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3117                                     v->gv_name, strerror(err));
3118                                 break;
3119 
3120                         case EPERM:
3121                                 uu_die("Permission denied.\n");
3122                                 /* NOTREACHED */
3123 
3124                         default:
3125                                 bad_error("libscf_set_deathrow", err);
3126                         }
3127                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3128                             v->gv_name);
3129                 }
3130                 startd_free(restarter_fmri, max_scf_value_size);
3131                 return (0);
3132         }
3133 
3134         h = scf_instance_handle(inst);
3135 
3136         /*
3137          * Using a temporary deathrow boolean property, set through
3138          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3139          * because deathrow_fini() may already have been called, and in case
3140          * of a refresh, GV_DEATHROW may need to be set again.
3141          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3142          * has a temporary boolean property named 'deathrow' valued true
3143          * in a property group 'deathrow', -1 or 0 in all other cases.
3144          */
3145         err = libscf_get_deathrow(h, inst, &deathrow);
3146         switch (err) {
3147         case 0:
3148                 break;
3149 
3150         case ECONNABORTED:
3151         case ECANCELED:
3152                 startd_free(restarter_fmri, max_scf_value_size);
3153                 return (err);
3154 
3155         default:
3156                 bad_error("libscf_get_deathrow", err);
3157         }
3158 
3159         if (deathrow == 1) {
3160                 v->gv_flags |= GV_DEATHROW;
3161                 startd_free(restarter_fmri, max_scf_value_size);
3162                 return (0);
3163         }
3164 
3165         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3166 
3167         /*
3168          * If the instance does not have a restarter property group,
3169          * initialize its state to uninitialized/none, in case the restarter
3170          * is not enabled.
3171          */
3172         pg = safe_scf_pg_create(h);
3173 
3174         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3175                 instance_data_t idata;
3176                 uint_t count = 0, msecs = ALLOC_DELAY;
3177 
3178                 switch (scf_error()) {
3179                 case SCF_ERROR_NOT_FOUND:
3180                         break;
3181 
3182                 case SCF_ERROR_CONNECTION_BROKEN:
3183                 default:
3184                         scf_pg_destroy(pg);
3185                         return (ECONNABORTED);
3186 
3187                 case SCF_ERROR_DELETED:
3188                         scf_pg_destroy(pg);
3189                         return (ECANCELED);
3190 
3191                 case SCF_ERROR_NOT_SET:
3192                         bad_error("scf_instance_get_pg", scf_error());
3193                 }
3194 
3195                 switch (err = libscf_instance_get_fmri(inst,
3196                     (char **)&idata.i_fmri)) {
3197                 case 0:
3198                         break;
3199 
3200                 case ECONNABORTED:
3201                 case ECANCELED:
3202                         scf_pg_destroy(pg);
3203                         return (err);
3204 
3205                 default:
3206                         bad_error("libscf_instance_get_fmri", err);
3207                 }
3208 
3209                 idata.i_state = RESTARTER_STATE_NONE;
3210                 idata.i_next_state = RESTARTER_STATE_NONE;
3211 
3212 init_state:
3213                 switch (err = _restarter_commit_states(h, &idata,
3214                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE, NULL)) {
3215                 case 0:
3216                         break;
3217 
3218                 case ENOMEM:
3219                         ++count;
3220                         if (count < ALLOC_RETRY) {
3221                                 (void) poll(NULL, 0, msecs);
3222                                 msecs *= ALLOC_DELAY_MULT;
3223                                 goto init_state;
3224                         }
3225 
3226                         uu_die("Insufficient memory.\n");
3227                         /* NOTREACHED */
3228 
3229                 case ECONNABORTED:
3230                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3231                         scf_pg_destroy(pg);
3232                         return (ECONNABORTED);
3233 
3234                 case ENOENT:
3235                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3236                         scf_pg_destroy(pg);
3237                         return (ECANCELED);
3238 
3239                 case EPERM:
3240                 case EACCES:
3241                 case EROFS:
3242                         log_error(LOG_NOTICE, "Could not initialize state for "
3243                             "%s: %s.\n", idata.i_fmri, strerror(err));
3244                         break;
3245 
3246                 case EINVAL:
3247                 default:
3248                         bad_error("_restarter_commit_states", err);
3249                 }
3250 
3251                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3252         }
3253 
3254         scf_pg_destroy(pg);
3255 
3256         if (milestone != NULL) {
3257                 /*
3258                  * Make sure the enable-override is set properly before we
3259                  * read whether we should be enabled.
3260                  */
3261                 if (milestone == MILESTONE_NONE ||
3262                     !(v->gv_flags & GV_INSUBGRAPH)) {
3263                         /*
3264                          * This might seem unjustified after the milestone
3265                          * transition has completed (non_subgraph_svcs == 0),
3266                          * but it's important because when we boot to
3267                          * a milestone, we set the milestone before populating
3268                          * the graph, and all of the new non-subgraph services
3269                          * need to be disabled here.
3270                          */
3271                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3272                         case 0:
3273                                 break;
3274 
3275                         case ECONNABORTED:
3276                         case ECANCELED:
3277                                 return (err);
3278 
3279                         case EROFS:
3280                                 log_error(LOG_WARNING,
3281                                     "Could not set %s/%s for %s: %s.\n",
3282                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3283                                     v->gv_name, strerror(err));
3284                                 break;
3285 
3286                         case EPERM:
3287                                 uu_die("Permission denied.\n");
3288                                 /* NOTREACHED */
3289 
3290                         default:
3291                                 bad_error("libscf_set_enable_ovr", err);
3292                         }
3293                 } else {
3294                         assert(v->gv_flags & GV_INSUBGRAPH);
3295                         switch (err = libscf_delete_enable_ovr(inst)) {
3296                         case 0:
3297                                 break;
3298 
3299                         case ECONNABORTED:
3300                         case ECANCELED:
3301                                 return (err);
3302 
3303                         case EPERM:
3304                                 uu_die("Permission denied.\n");
3305                                 /* NOTREACHED */
3306 
3307                         default:
3308                                 bad_error("libscf_delete_enable_ovr", err);
3309                         }
3310                 }
3311         }
3312 
3313         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3314             &enabled_ovr, &restarter_fmri);
3315         switch (err) {
3316         case 0:
3317                 break;
3318 
3319         case ECONNABORTED:
3320         case ECANCELED:
3321                 startd_free(restarter_fmri, max_scf_value_size);
3322                 return (err);
3323 
3324         case ENOENT:
3325                 log_framework(LOG_DEBUG,
3326                     "Ignoring %s because it has no general property group.\n",
3327                     v->gv_name);
3328                 startd_free(restarter_fmri, max_scf_value_size);
3329                 return (0);
3330 
3331         default:
3332                 bad_error("libscf_get_basic_instance_data", err);
3333         }
3334 
3335         if (enabled == -1) {
3336                 startd_free(restarter_fmri, max_scf_value_size);
3337                 return (0);
3338         }
3339 
3340         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3341             (enabled ? GV_ENBLD_NOOVR : 0);
3342 
3343         if (enabled_ovr != -1)
3344                 enabled = enabled_ovr;
3345 
3346         v->gv_state = RESTARTER_STATE_UNINIT;
3347 
3348         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3349         scf_snapshot_destroy(snap);
3350 
3351         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3352         err = graph_change_restarter(v, restarter_fmri, h, &path);
3353         if (err != 0) {
3354                 instance_data_t idata;
3355                 uint_t count = 0, msecs = ALLOC_DELAY;
3356                 const char *reason;
3357 
3358                 if (err == ECONNABORTED) {
3359                         startd_free(restarter_fmri, max_scf_value_size);
3360                         return (err);
3361                 }
3362 
3363                 assert(err == EINVAL || err == ELOOP);
3364 
3365                 if (err == EINVAL) {
3366                         log_framework(LOG_ERR, emsg_invalid_restarter,
3367                             v->gv_name);
3368                         reason = "invalid_restarter";
3369                 } else {
3370                         handle_cycle(v->gv_name, path);
3371                         reason = "dependency_cycle";
3372                 }
3373 
3374                 startd_free(restarter_fmri, max_scf_value_size);
3375 
3376                 /*
3377                  * We didn't register the instance with the restarter, so we
3378                  * must set maintenance mode ourselves.
3379                  */
3380                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3381                 if (err != 0) {
3382                         assert(err == ECONNABORTED || err == ECANCELED);
3383                         return (err);
3384                 }
3385 
3386                 idata.i_state = RESTARTER_STATE_NONE;
3387                 idata.i_next_state = RESTARTER_STATE_NONE;
3388 
3389 set_maint:
3390                 switch (err = _restarter_commit_states(h, &idata,
3391                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, reason)) {
3392                 case 0:
3393                         break;
3394 
3395                 case ENOMEM:
3396                         ++count;
3397                         if (count < ALLOC_RETRY) {
3398                                 (void) poll(NULL, 0, msecs);
3399                                 msecs *= ALLOC_DELAY_MULT;
3400                                 goto set_maint;
3401                         }
3402 
3403                         uu_die("Insufficient memory.\n");
3404                         /* NOTREACHED */
3405 
3406                 case ECONNABORTED:
3407                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3408                         return (ECONNABORTED);
3409 
3410                 case ENOENT:
3411                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3412                         return (ECANCELED);
3413 
3414                 case EPERM:
3415                 case EACCES:
3416                 case EROFS:
3417                         log_error(LOG_NOTICE, "Could not initialize state for "
3418                             "%s: %s.\n", idata.i_fmri, strerror(err));
3419                         break;
3420 
3421                 case EINVAL:
3422                 default:
3423                         bad_error("_restarter_commit_states", err);
3424                 }
3425 
3426                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3427 
3428                 v->gv_state = RESTARTER_STATE_MAINT;
3429 
3430                 goto out;
3431         }
3432         startd_free(restarter_fmri, max_scf_value_size);
3433 
3434         /* Add all the other dependencies. */
3435         err = refresh_vertex(v, inst);
3436         if (err != 0) {
3437                 assert(err == ECONNABORTED);
3438                 return (err);
3439         }
3440 
3441 out:
3442         v->gv_flags |= GV_CONFIGURED;
3443 
3444         graph_enable_by_vertex(v, enabled, 0);
3445 
3446         return (0);
3447 }
3448 
3449 
3450 static void
3451 kill_user_procs(void)
3452 {
3453         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3454 
3455         /*
3456          * Despite its name, killall's role is to get select user processes--
3457          * basically those representing terminal-based logins-- to die.  Victims
3458          * are located by killall in the utmp database.  Since these are most
3459          * often shell based logins, and many shells mask SIGTERM (but are
3460          * responsive to SIGHUP) we first HUP and then shortly thereafter
3461          * kill -9.
3462          */
3463         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3464         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3465 
3466         /*
3467          * Note the selection of user id's 0, 1 and 15, subsequently
3468          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3469          * kludge-- a better policy is needed.
3470          *
3471          * Note that fork_with_timeout will only wait out the 1 second
3472          * "grace time" if pkill actually returns 0.  So if there are
3473          * no matches, this will run to completion much more quickly.
3474          */
3475         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3476         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3477 }
3478 
3479 static void
3480 do_uadmin(void)
3481 {
3482         int fd;
3483         struct statvfs vfs;
3484         time_t now;
3485         struct tm nowtm;
3486         char down_buf[256], time_buf[256];
3487 
3488         const char * const resetting = "/etc/svc/volatile/resetting";
3489 
3490         fd = creat(resetting, 0777);
3491         if (fd >= 0)
3492                 startd_close(fd);
3493         else
3494                 uu_warn("Could not create \"%s\"", resetting);
3495 
3496         /* Kill dhcpagent if we're not using nfs for root */
3497         if ((statvfs("/", &vfs) == 0) &&
3498             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3499                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3500 
3501         /*
3502          * Call sync(2) now, before we kill off user processes.  This takes
3503          * advantage of the several seconds of pause we have before the
3504          * killalls are done.  Time we can make good use of to get pages
3505          * moving out to disk.
3506          *
3507          * Inside non-global zones, we don't bother, and it's better not to
3508          * anyway, since sync(2) can have system-wide impact.
3509          */
3510         if (getzoneid() == 0)
3511                 sync();
3512 
3513         kill_user_procs();
3514 
3515         /*
3516          * Note that this must come after the killing of user procs, since
3517          * killall relies on utmpx, and this command affects the contents of
3518          * said file.
3519          */
3520         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3521                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3522 
3523         /*
3524          * For patches which may be installed as the system is shutting
3525          * down, we need to ensure, one more time, that the boot archive
3526          * really is up to date.
3527          */
3528         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3529                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3530 
3531         fork_with_timeout("/sbin/umountall -l", 0, 5);
3532         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3533             ">/dev/null 2>&1", 0, 5);
3534 
3535         /*
3536          * Try to get to consistency for whatever UFS filesystems are left.
3537          * This is pretty expensive, so we save it for the end in the hopes of
3538          * minimizing what it must do.  The other option would be to start in
3539          * parallel with the killall's, but lockfs tends to throw out much more
3540          * than is needed, and so subsequent commands (like umountall) take a
3541          * long time to get going again.
3542          *
3543          * Inside of zones, we don't bother, since we're not about to terminate
3544          * the whole OS instance.
3545          *
3546          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3547          */
3548         if (getzoneid() == 0) {
3549                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3550                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3551 
3552                 sync(); /* once more, with feeling */
3553         }
3554 
3555         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3556 
3557         /*
3558          * Construct and emit the last words from userland:
3559          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3560          *
3561          * Normally we'd use syslog, but with /var and other things
3562          * potentially gone, try to minimize the external dependencies.
3563          */
3564         now = time(NULL);
3565         (void) localtime_r(&now, &nowtm);
3566 
3567         if (strftime(down_buf, sizeof (down_buf),
3568             "%b %e %T The system is down.", &nowtm) == 0) {
3569                 (void) strlcpy(down_buf, "The system is down.",
3570                     sizeof (down_buf));
3571         }
3572 
3573         if (halting_time != 0 && halting_time <= now) {
3574                 (void) snprintf(time_buf, sizeof (time_buf),
3575                     "  Shutdown took %lu seconds.", now - halting_time);
3576         } else {
3577                 time_buf[0] = '\0';
3578         }
3579         (void) printf("%s%s\n", down_buf, time_buf);
3580 
3581         (void) uadmin(A_SHUTDOWN, halting, NULL);
3582         uu_warn("uadmin() failed");
3583 
3584         if (remove(resetting) != 0 && errno != ENOENT)
3585                 uu_warn("Could not remove \"%s\"", resetting);
3586 }
3587 
3588 /*
3589  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3590  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3591  */
3592 boolean_t
3593 can_come_up(void)
3594 {
3595         int i;
3596 
3597         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3598 
3599         /*
3600          * If we are booting to single user (boot -s),
3601          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3602          * spawns sulogin after single-user is online (see specials.c).
3603          */
3604         i = (booting_to_single_user ? 0 : 1);
3605 
3606         for (; up_svcs[i] != NULL; ++i) {
3607                 if (up_svcs_p[i] == NULL) {
3608                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3609 
3610                         if (up_svcs_p[i] == NULL)
3611                                 continue;
3612                 }
3613 
3614                 /*
3615                  * Ignore unconfigured services (the ones that have been
3616                  * mentioned in a dependency from other services, but do
3617                  * not exist in the repository).  Services which exist
3618                  * in the repository but don't have general/enabled
3619                  * property will be also ignored.
3620                  */
3621                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3622                         continue;
3623 
3624                 switch (up_svcs_p[i]->gv_state) {
3625                 case RESTARTER_STATE_ONLINE:
3626                 case RESTARTER_STATE_DEGRADED:
3627                         /*
3628                          * Deactivate verbose boot once a login service has been
3629                          * reached.
3630                          */
3631                         st->st_log_login_reached = 1;
3632                         /*FALLTHROUGH*/
3633                 case RESTARTER_STATE_UNINIT:
3634                         return (B_TRUE);
3635 
3636                 case RESTARTER_STATE_OFFLINE:
3637                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3638                                 return (B_TRUE);
3639                         log_framework(LOG_DEBUG,
3640                             "can_come_up(): %s is unsatisfiable.\n",
3641                             up_svcs_p[i]->gv_name);
3642                         continue;
3643 
3644                 case RESTARTER_STATE_DISABLED:
3645                 case RESTARTER_STATE_MAINT:
3646                         log_framework(LOG_DEBUG,
3647                             "can_come_up(): %s is in state %s.\n",
3648                             up_svcs_p[i]->gv_name,
3649                             instance_state_str[up_svcs_p[i]->gv_state]);
3650                         continue;
3651 
3652                 default:
3653 #ifndef NDEBUG
3654                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3655                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3656 #endif
3657                         abort();
3658                 }
3659         }
3660 
3661         /*
3662          * In the seed repository, console-login is unsatisfiable because
3663          * services are missing.  To behave correctly in that case we don't want
3664          * to return false until manifest-import is online.
3665          */
3666 
3667         if (manifest_import_p == NULL) {
3668                 manifest_import_p = vertex_get_by_name(manifest_import);
3669 
3670                 if (manifest_import_p == NULL)
3671                         return (B_FALSE);
3672         }
3673 
3674         switch (manifest_import_p->gv_state) {
3675         case RESTARTER_STATE_ONLINE:
3676         case RESTARTER_STATE_DEGRADED:
3677         case RESTARTER_STATE_DISABLED:
3678         case RESTARTER_STATE_MAINT:
3679                 break;
3680 
3681         case RESTARTER_STATE_OFFLINE:
3682                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3683                         break;
3684                 /* FALLTHROUGH */
3685 
3686         case RESTARTER_STATE_UNINIT:
3687                 return (B_TRUE);
3688         }
3689 
3690         return (B_FALSE);
3691 }
3692 
3693 /*
3694  * Runs sulogin.  Returns
3695  *   0 - success
3696  *   EALREADY - sulogin is already running
3697  *   EBUSY - console-login is running
3698  */
3699 static int
3700 run_sulogin(const char *msg)
3701 {
3702         graph_vertex_t *v;
3703 
3704         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3705 
3706         if (sulogin_running)
3707                 return (EALREADY);
3708 
3709         v = vertex_get_by_name(console_login_fmri);
3710         if (v != NULL && inst_running(v))
3711                 return (EBUSY);
3712 
3713         sulogin_running = B_TRUE;
3714 
3715         MUTEX_UNLOCK(&dgraph_lock);
3716 
3717         fork_sulogin(B_FALSE, msg);
3718 
3719         MUTEX_LOCK(&dgraph_lock);
3720 
3721         sulogin_running = B_FALSE;
3722 
3723         if (console_login_ready) {
3724                 v = vertex_get_by_name(console_login_fmri);
3725 
3726                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE &&
3727                     !inst_running(v)) {
3728                         if (v->gv_start_f == NULL)
3729                                 vertex_send_event(v,
3730                                     RESTARTER_EVENT_TYPE_START);
3731                         else
3732                                 v->gv_start_f(v);
3733                 }
3734 
3735                 console_login_ready = B_FALSE;
3736         }
3737 
3738         return (0);
3739 }
3740 
3741 /*
3742  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3743  * keeps sulogin from stepping on console-login's toes.
3744  */
3745 /* ARGSUSED */
3746 static void *
3747 sulogin_thread(void *unused)
3748 {
3749         MUTEX_LOCK(&dgraph_lock);
3750 
3751         assert(sulogin_thread_running);
3752 
3753         do {
3754                 (void) run_sulogin("Console login service(s) cannot run\n");
3755         } while (!can_come_up());
3756 
3757         sulogin_thread_running = B_FALSE;
3758         MUTEX_UNLOCK(&dgraph_lock);
3759 
3760         return (NULL);
3761 }
3762 
3763 /* ARGSUSED */
3764 void *
3765 single_user_thread(void *unused)
3766 {
3767         uint_t left;
3768         scf_handle_t *h;
3769         scf_instance_t *inst;
3770         scf_property_t *prop;
3771         scf_value_t *val;
3772         const char *msg;
3773         char *buf;
3774         int r;
3775 
3776         MUTEX_LOCK(&single_user_thread_lock);
3777         single_user_thread_count++;
3778 
3779         if (!booting_to_single_user)
3780                 kill_user_procs();
3781 
3782         if (go_single_user_mode || booting_to_single_user) {
3783                 msg = "SINGLE USER MODE\n";
3784         } else {
3785                 assert(go_to_level1);
3786 
3787                 fork_rc_script('1', "start", B_TRUE);
3788 
3789                 uu_warn("The system is ready for administration.\n");
3790 
3791                 msg = "";
3792         }
3793 
3794         MUTEX_UNLOCK(&single_user_thread_lock);
3795 
3796         for (;;) {
3797                 MUTEX_LOCK(&dgraph_lock);
3798                 r = run_sulogin(msg);
3799                 MUTEX_UNLOCK(&dgraph_lock);
3800                 if (r == 0)
3801                         break;
3802 
3803                 assert(r == EALREADY || r == EBUSY);
3804 
3805                 left = 3;
3806                 while (left > 0)
3807                         left = sleep(left);
3808         }
3809 
3810         MUTEX_LOCK(&single_user_thread_lock);
3811 
3812         /*
3813          * If another single user thread has started, let it finish changing
3814          * the run level.
3815          */
3816         if (single_user_thread_count > 1) {
3817                 single_user_thread_count--;
3818                 MUTEX_UNLOCK(&single_user_thread_lock);
3819                 return (NULL);
3820         }
3821 
3822         h = libscf_handle_create_bound_loop();
3823         inst = scf_instance_create(h);
3824         prop = safe_scf_property_create(h);
3825         val = safe_scf_value_create(h);
3826         buf = startd_alloc(max_scf_fmri_size);
3827 
3828 lookup:
3829         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3830             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3831                 switch (scf_error()) {
3832                 case SCF_ERROR_NOT_FOUND:
3833                         r = libscf_create_self(h);
3834                         if (r == 0)
3835                                 goto lookup;
3836                         assert(r == ECONNABORTED);
3837                         /* FALLTHROUGH */
3838 
3839                 case SCF_ERROR_CONNECTION_BROKEN:
3840                         libscf_handle_rebind(h);
3841                         goto lookup;
3842 
3843                 case SCF_ERROR_INVALID_ARGUMENT:
3844                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3845                 case SCF_ERROR_NOT_BOUND:
3846                 case SCF_ERROR_HANDLE_MISMATCH:
3847                 default:
3848                         bad_error("scf_handle_decode_fmri", scf_error());
3849                 }
3850         }
3851 
3852         MUTEX_LOCK(&dgraph_lock);
3853 
3854         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3855             SCF_PROPERTY_MILESTONE);
3856         switch (r) {
3857         case 0:
3858         case ECANCELED:
3859                 break;
3860 
3861         case ECONNABORTED:
3862                 MUTEX_UNLOCK(&dgraph_lock);
3863                 libscf_handle_rebind(h);
3864                 goto lookup;
3865 
3866         case EPERM:
3867         case EACCES:
3868         case EROFS:
3869                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3870                     "%s.\n", strerror(r));
3871                 break;
3872 
3873         default:
3874                 bad_error("scf_instance_delete_prop", r);
3875         }
3876 
3877         MUTEX_UNLOCK(&dgraph_lock);
3878 
3879         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3880         switch (r) {
3881         case ECANCELED:
3882         case ENOENT:
3883         case EINVAL:
3884                 (void) strcpy(buf, "all");
3885                 /* FALLTHROUGH */
3886 
3887         case 0:
3888                 uu_warn("Returning to milestone %s.\n", buf);
3889                 break;
3890 
3891         case ECONNABORTED:
3892                 libscf_handle_rebind(h);
3893                 goto lookup;
3894 
3895         default:
3896                 bad_error("libscf_get_milestone", r);
3897         }
3898 
3899         r = dgraph_set_milestone(buf, h, B_FALSE);
3900         switch (r) {
3901         case 0:
3902         case ECONNRESET:
3903         case EALREADY:
3904         case EINVAL:
3905         case ENOENT:
3906                 break;
3907 
3908         default:
3909                 bad_error("dgraph_set_milestone", r);
3910         }
3911 
3912         /*
3913          * See graph_runlevel_changed().
3914          */
3915         MUTEX_LOCK(&dgraph_lock);
3916         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
3917         MUTEX_UNLOCK(&dgraph_lock);
3918 
3919         startd_free(buf, max_scf_fmri_size);
3920         scf_value_destroy(val);
3921         scf_property_destroy(prop);
3922         scf_instance_destroy(inst);
3923         scf_handle_destroy(h);
3924 
3925         /*
3926          * We'll give ourselves 3 seconds to respond to all of the enablings
3927          * that setting the milestone should have created before checking
3928          * whether to run sulogin.
3929          */
3930         left = 3;
3931         while (left > 0)
3932                 left = sleep(left);
3933 
3934         MUTEX_LOCK(&dgraph_lock);
3935         /*
3936          * Clearing these variables will allow the sulogin thread to run.  We
3937          * check here in case there aren't any more state updates anytime soon.
3938          */
3939         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
3940         if (!sulogin_thread_running && !can_come_up()) {
3941                 (void) startd_thread_create(sulogin_thread, NULL);
3942                 sulogin_thread_running = B_TRUE;
3943         }
3944         MUTEX_UNLOCK(&dgraph_lock);
3945         single_user_thread_count--;
3946         MUTEX_UNLOCK(&single_user_thread_lock);
3947         return (NULL);
3948 }
3949 
3950 
3951 /*
3952  * Dependency graph operations API.  These are handle-independent thread-safe
3953  * graph manipulation functions which are the entry points for the event
3954  * threads below.
3955  */
3956 
3957 /*
3958  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
3959  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
3960  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
3961  * Fetch whether the instance should be enabled from inst and send _ENABLE or
3962  * _DISABLE as appropriate.  Finally rummage through inst's dependency
3963  * property groups and add vertices and edges as appropriate.  If anything
3964  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
3965  * instance in maintenance.  Don't send _START or _STOP until we get a state
3966  * update in case we're being restarted and the service is already running.
3967  *
3968  * To support booting to a milestone, we must also make sure all dependencies
3969  * encountered are configured, if they exist in the repository.
3970  *
3971  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
3972  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
3973  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
3974  */
3975 int
3976 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
3977     boolean_t lock_graph)
3978 {
3979         graph_vertex_t *v;
3980         int err;
3981 
3982         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
3983                 return (0);
3984 
3985         /* Check for a vertex for inst_fmri. */
3986         if (lock_graph) {
3987                 MUTEX_LOCK(&dgraph_lock);
3988         } else {
3989                 assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3990         }
3991 
3992         v = vertex_get_by_name(inst_fmri);
3993 
3994         if (v != NULL) {
3995                 assert(v->gv_type == GVT_INST);
3996 
3997                 if (v->gv_flags & GV_CONFIGURED) {
3998                         if (lock_graph)
3999                                 MUTEX_UNLOCK(&dgraph_lock);
4000                         return (EEXIST);
4001                 }
4002         } else {
4003                 /* Add the vertex. */
4004                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4005                     RERR_NONE, &v);
4006                 if (err != 0) {
4007                         assert(err == EINVAL);
4008                         if (lock_graph)
4009                                 MUTEX_UNLOCK(&dgraph_lock);
4010                         return (EINVAL);
4011                 }
4012         }
4013 
4014         err = configure_vertex(v, inst);
4015 
4016         if (lock_graph)
4017                 MUTEX_UNLOCK(&dgraph_lock);
4018 
4019         return (err);
4020 }
4021 
4022 /*
4023  * Locate the vertex for this property group's instance.  If it doesn't exist
4024  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4025  * the restarter for the instance, and if it has changed, send
4026  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4027  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4028  * the new restarter.  Then fetch whether the instance should be enabled, and
4029  * if it is different from what we had, or if we changed the restarter, send
4030  * the appropriate _ENABLE or _DISABLE command.
4031  *
4032  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4033  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4034  * deleted, or -1 if the instance's general property group is deleted or if
4035  * its enabled property is misconfigured.
4036  */
4037 static int
4038 dgraph_update_general(scf_propertygroup_t *pg)
4039 {
4040         scf_handle_t *h;
4041         scf_instance_t *inst;
4042         char *fmri;
4043         char *restarter_fmri;
4044         graph_vertex_t *v;
4045         int err;
4046         int enabled, enabled_ovr;
4047         int oldflags;
4048 
4049         /* Find the vertex for this service */
4050         h = scf_pg_handle(pg);
4051 
4052         inst = safe_scf_instance_create(h);
4053 
4054         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4055                 switch (scf_error()) {
4056                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4057                         return (ENOTSUP);
4058 
4059                 case SCF_ERROR_CONNECTION_BROKEN:
4060                 default:
4061                         return (ECONNABORTED);
4062 
4063                 case SCF_ERROR_DELETED:
4064                         return (0);
4065 
4066                 case SCF_ERROR_NOT_SET:
4067                         bad_error("scf_pg_get_parent_instance", scf_error());
4068                 }
4069         }
4070 
4071         err = libscf_instance_get_fmri(inst, &fmri);
4072         switch (err) {
4073         case 0:
4074                 break;
4075 
4076         case ECONNABORTED:
4077                 scf_instance_destroy(inst);
4078                 return (ECONNABORTED);
4079 
4080         case ECANCELED:
4081                 scf_instance_destroy(inst);
4082                 return (0);
4083 
4084         default:
4085                 bad_error("libscf_instance_get_fmri", err);
4086         }
4087 
4088         log_framework(LOG_DEBUG,
4089             "Graph engine: Reloading general properties for %s.\n", fmri);
4090 
4091         MUTEX_LOCK(&dgraph_lock);
4092 
4093         v = vertex_get_by_name(fmri);
4094         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4095                 /* Will get the up-to-date properties. */
4096                 MUTEX_UNLOCK(&dgraph_lock);
4097                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4098                 startd_free(fmri, max_scf_fmri_size);
4099                 scf_instance_destroy(inst);
4100                 return (err == ECANCELED ? 0 : err);
4101         }
4102 
4103         /* Read enabled & restarter from repository. */
4104         restarter_fmri = startd_alloc(max_scf_value_size);
4105         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4106             &enabled_ovr, &restarter_fmri);
4107         if (err != 0 || enabled == -1) {
4108                 MUTEX_UNLOCK(&dgraph_lock);
4109                 scf_instance_destroy(inst);
4110                 startd_free(fmri, max_scf_fmri_size);
4111 
4112                 switch (err) {
4113                 case ENOENT:
4114                 case 0:
4115                         startd_free(restarter_fmri, max_scf_value_size);
4116                         return (-1);
4117 
4118                 case ECONNABORTED:
4119                 case ECANCELED:
4120                         startd_free(restarter_fmri, max_scf_value_size);
4121                         return (err);
4122 
4123                 default:
4124                         bad_error("libscf_get_basic_instance_data", err);
4125                 }
4126         }
4127 
4128         oldflags = v->gv_flags;
4129         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4130             (enabled ? GV_ENBLD_NOOVR : 0);
4131 
4132         if (enabled_ovr != -1)
4133                 enabled = enabled_ovr;
4134 
4135         /*
4136          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4137          * subgraph.
4138          */
4139         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4140                 (void) eval_subgraph(v, h);
4141 
4142         scf_instance_destroy(inst);
4143 
4144         /* Ignore restarter change for now. */
4145 
4146         startd_free(restarter_fmri, max_scf_value_size);
4147         startd_free(fmri, max_scf_fmri_size);
4148 
4149         /*
4150          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4151          * restarter didn't change and the enabled value didn't change, but
4152          * that's not easy to check and improbable anyway, so we'll just do
4153          * this.
4154          */
4155         graph_enable_by_vertex(v, enabled, 1);
4156 
4157         MUTEX_UNLOCK(&dgraph_lock);
4158 
4159         return (0);
4160 }
4161 
4162 /*
4163  * Delete all of the property group dependencies of v, update inst's running
4164  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4165  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4166  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4167  * the same for v's dependents.
4168  *
4169  * Returns
4170  *   0 - success
4171  *   ECONNABORTED - repository connection broken
4172  *   ECANCELED - inst was deleted
4173  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4174  *   -1 - libscf_snapshots_refresh() failed
4175  */
4176 static int
4177 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4178 {
4179         int r;
4180         int enabled;
4181 
4182         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4183         assert(v->gv_type == GVT_INST);
4184 
4185         /* Only refresh services with valid general/enabled properties. */
4186         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4187             v->gv_name, &enabled, NULL, NULL);
4188         switch (r) {
4189         case 0:
4190                 break;
4191 
4192         case ECONNABORTED:
4193         case ECANCELED:
4194                 return (r);
4195 
4196         case ENOENT:
4197                 log_framework(LOG_DEBUG,
4198                     "Ignoring %s because it has no general property group.\n",
4199                     v->gv_name);
4200                 return (EINVAL);
4201 
4202         default:
4203                 bad_error("libscf_get_basic_instance_data", r);
4204         }
4205 
4206         if (enabled == -1)
4207                 return (EINVAL);
4208 
4209         r = libscf_snapshots_refresh(inst, v->gv_name);
4210         if (r != 0) {
4211                 if (r != -1)
4212                         bad_error("libscf_snapshots_refresh", r);
4213 
4214                 /* error logged */
4215                 return (r);
4216         }
4217 
4218         r = refresh_vertex(v, inst);
4219         if (r != 0 && r != ECONNABORTED)
4220                 bad_error("refresh_vertex", r);
4221         return (r);
4222 }
4223 
4224 /*
4225  * Returns true only if none of this service's dependents are 'up' -- online
4226  * or degraded (offline is considered down in this situation). This function
4227  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4228  */
4229 static boolean_t
4230 insubtree_dependents_down(graph_vertex_t *v)
4231 {
4232         graph_vertex_t *vv;
4233         graph_edge_t *e;
4234 
4235         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4236 
4237         for (e = uu_list_first(v->gv_dependents); e != NULL;
4238             e = uu_list_next(v->gv_dependents, e)) {
4239                 vv = e->ge_vertex;
4240                 if (vv->gv_type == GVT_INST) {
4241                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4242                                 continue;
4243 
4244                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4245                                 continue;
4246 
4247                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4248                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4249                                 return (B_FALSE);
4250                 } else {
4251                         /*
4252                          * For dependency groups or service vertices, keep
4253                          * traversing to see if instances are running.
4254                          */
4255                         if (insubtree_dependents_down(vv) == B_FALSE)
4256                                 return (B_FALSE);
4257                 }
4258         }
4259 
4260         return (B_TRUE);
4261 }
4262 
4263 /*
4264  * Returns true only if none of this service's dependents are 'up' -- online,
4265  * degraded, or offline.
4266  */
4267 static int
4268 is_nonsubgraph_leaf(graph_vertex_t *v)
4269 {
4270         graph_vertex_t *vv;
4271         graph_edge_t *e;
4272 
4273         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4274 
4275         for (e = uu_list_first(v->gv_dependents);
4276             e != NULL;
4277             e = uu_list_next(v->gv_dependents, e)) {
4278 
4279                 vv = e->ge_vertex;
4280                 if (vv->gv_type == GVT_INST) {
4281                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4282                                 continue;
4283 
4284                         if (vv->gv_flags & GV_INSUBGRAPH)
4285                                 continue;
4286 
4287                         if (up_state(vv->gv_state))
4288                                 return (0);
4289                 } else {
4290                         /*
4291                          * For dependency group or service vertices, keep
4292                          * traversing to see if instances are running.
4293                          *
4294                          * We should skip exclude_all dependencies otherwise
4295                          * the vertex will never be considered as a leaf
4296                          * if the dependent is offline. The main reason for
4297                          * this is that disable_nonsubgraph_leaves() skips
4298                          * exclusion dependencies.
4299                          */
4300                         if (vv->gv_type == GVT_GROUP &&
4301                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4302                                 continue;
4303 
4304                         if (!is_nonsubgraph_leaf(vv))
4305                                 return (0);
4306                 }
4307         }
4308 
4309         return (1);
4310 }
4311 
4312 /*
4313  * Disable v temporarily.  Attempt to do this by setting its enabled override
4314  * property in the repository.  If that fails, send a _DISABLE command.
4315  * Returns 0 on success and ECONNABORTED if the repository connection is
4316  * broken.
4317  */
4318 static int
4319 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4320 {
4321         const char * const emsg = "Could not temporarily disable %s because "
4322             "%s.  Will stop service anyways.  Repository status for the "
4323             "service may be inaccurate.\n";
4324         const char * const emsg_cbroken =
4325             "the repository connection was broken";
4326 
4327         scf_instance_t *inst;
4328         int r;
4329 
4330         inst = scf_instance_create(h);
4331         if (inst == NULL) {
4332                 char buf[100];
4333 
4334                 (void) snprintf(buf, sizeof (buf),
4335                     "scf_instance_create() failed (%s)",
4336                     scf_strerror(scf_error()));
4337                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4338 
4339                 graph_enable_by_vertex(v, 0, 0);
4340                 return (0);
4341         }
4342 
4343         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4344             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4345         if (r != 0) {
4346                 switch (scf_error()) {
4347                 case SCF_ERROR_CONNECTION_BROKEN:
4348                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4349                         graph_enable_by_vertex(v, 0, 0);
4350                         return (ECONNABORTED);
4351 
4352                 case SCF_ERROR_NOT_FOUND:
4353                         return (0);
4354 
4355                 case SCF_ERROR_HANDLE_MISMATCH:
4356                 case SCF_ERROR_INVALID_ARGUMENT:
4357                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4358                 case SCF_ERROR_NOT_BOUND:
4359                 default:
4360                         bad_error("scf_handle_decode_fmri",
4361                             scf_error());
4362                 }
4363         }
4364 
4365         r = libscf_set_enable_ovr(inst, 0);
4366         switch (r) {
4367         case 0:
4368                 scf_instance_destroy(inst);
4369                 return (0);
4370 
4371         case ECANCELED:
4372                 scf_instance_destroy(inst);
4373                 return (0);
4374 
4375         case ECONNABORTED:
4376                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4377                 graph_enable_by_vertex(v, 0, 0);
4378                 return (ECONNABORTED);
4379 
4380         case EPERM:
4381                 log_error(LOG_WARNING, emsg, v->gv_name,
4382                     "the repository denied permission");
4383                 graph_enable_by_vertex(v, 0, 0);
4384                 return (0);
4385 
4386         case EROFS:
4387                 log_error(LOG_WARNING, emsg, v->gv_name,
4388                     "the repository is read-only");
4389                 graph_enable_by_vertex(v, 0, 0);
4390                 return (0);
4391 
4392         default:
4393                 bad_error("libscf_set_enable_ovr", r);
4394                 /* NOTREACHED */
4395         }
4396 }
4397 
4398 /*
4399  * Of the transitive instance dependencies of v, offline those which are
4400  * in the subtree and which are leaves (i.e., have no dependents which are
4401  * "up").
4402  */
4403 void
4404 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4405 {
4406         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4407 
4408         /* If v isn't an instance, recurse on its dependencies. */
4409         if (v->gv_type != GVT_INST) {
4410                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4411                 return;
4412         }
4413 
4414         /*
4415          * If v is not in the subtree, so should all of its dependencies,
4416          * so do nothing.
4417          */
4418         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4419                 return;
4420 
4421         /* If v isn't a leaf because it's already down, recurse. */
4422         if (!up_state(v->gv_state)) {
4423                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4424                 return;
4425         }
4426 
4427         /* if v is a leaf, offline it or disable it if it's the last one */
4428         if (insubtree_dependents_down(v) == B_TRUE) {
4429                 if (v->gv_flags & GV_TODISABLE)
4430                         vertex_send_event(v,
4431                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4432                 else
4433                         offline_vertex(v);
4434         }
4435 }
4436 
4437 void
4438 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4439 {
4440         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4441 }
4442 
4443 
4444 /*
4445  * Of the transitive instance dependencies of v, disable those which are not
4446  * in the subgraph and which are leaves (i.e., have no dependents which are
4447  * "up").
4448  */
4449 static void
4450 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4451 {
4452         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4453 
4454         /*
4455          * We must skip exclusion dependencies because they are allowed to
4456          * complete dependency cycles.  This is correct because A's exclusion
4457          * dependency on B doesn't bear on the order in which they should be
4458          * stopped.  Indeed, the exclusion dependency should guarantee that
4459          * they are never online at the same time.
4460          */
4461         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4462                 return;
4463 
4464         /* If v isn't an instance, recurse on its dependencies. */
4465         if (v->gv_type != GVT_INST)
4466                 goto recurse;
4467 
4468         if ((v->gv_flags & GV_CONFIGURED) == 0)
4469                 /*
4470                  * Unconfigured instances should have no dependencies, but in
4471                  * case they ever get them,
4472                  */
4473                 goto recurse;
4474 
4475         /*
4476          * If v is in the subgraph, so should all of its dependencies, so do
4477          * nothing.
4478          */
4479         if (v->gv_flags & GV_INSUBGRAPH)
4480                 return;
4481 
4482         /* If v isn't a leaf because it's already down, recurse. */
4483         if (!up_state(v->gv_state))
4484                 goto recurse;
4485 
4486         /* If v is disabled but not down yet, be patient. */
4487         if ((v->gv_flags & GV_ENABLED) == 0)
4488                 return;
4489 
4490         /* If v is a leaf, disable it. */
4491         if (is_nonsubgraph_leaf(v))
4492                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4493 
4494         return;
4495 
4496 recurse:
4497         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4498 }
4499 
4500 /*
4501  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4502  * Otherwise set its state to state.  If the instance has entered a state
4503  * which requires automatic action, take it (Uninitialized: do
4504  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4505  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4506  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4507  * _START.  Online, Degraded: if the instance wasn't running, update its start
4508  * snapshot.  Maintenance: no action.)
4509  *
4510  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4511  */
4512 static int
4513 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4514     restarter_instance_state_t state, restarter_error_t serr)
4515 {
4516         graph_vertex_t *v;
4517         int err = 0;
4518         restarter_instance_state_t old_state;
4519 
4520         MUTEX_LOCK(&dgraph_lock);
4521 
4522         v = vertex_get_by_name(inst_name);
4523         if (v == NULL) {
4524                 MUTEX_UNLOCK(&dgraph_lock);
4525                 return (ENOENT);
4526         }
4527 
4528         assert(v->gv_type == GVT_INST);
4529 
4530         switch (state) {
4531         case RESTARTER_STATE_UNINIT:
4532         case RESTARTER_STATE_DISABLED:
4533         case RESTARTER_STATE_OFFLINE:
4534         case RESTARTER_STATE_ONLINE:
4535         case RESTARTER_STATE_DEGRADED:
4536         case RESTARTER_STATE_MAINT:
4537                 break;
4538 
4539         default:
4540                 MUTEX_UNLOCK(&dgraph_lock);
4541                 return (EINVAL);
4542         }
4543 
4544         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4545             instance_state_str[v->gv_state], instance_state_str[state]);
4546 
4547         old_state = v->gv_state;
4548         v->gv_state = state;
4549 
4550         err = gt_transition(h, v, serr, old_state);
4551 
4552         MUTEX_UNLOCK(&dgraph_lock);
4553         return (err);
4554 }
4555 
4556 /*
4557  * Handle state changes during milestone shutdown.  See
4558  * dgraph_set_milestone().  If the repository connection is broken,
4559  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4560  * the vertex anyway.
4561  */
4562 int
4563 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4564     restarter_instance_state_t old_state)
4565 {
4566         int was_up, now_up;
4567         int ret = 0;
4568 
4569         assert(v->gv_type == GVT_INST);
4570 
4571         /* Don't care if we're not going to a milestone. */
4572         if (milestone == NULL)
4573                 return (0);
4574 
4575         /* Don't care if we already finished coming down. */
4576         if (non_subgraph_svcs == 0)
4577                 return (0);
4578 
4579         /* Don't care if the service is in the subgraph. */
4580         if (v->gv_flags & GV_INSUBGRAPH)
4581                 return (0);
4582 
4583         /*
4584          * Update non_subgraph_svcs.  It is the number of non-subgraph
4585          * services which are in online, degraded, or offline.
4586          */
4587 
4588         was_up = up_state(old_state);
4589         now_up = up_state(v->gv_state);
4590 
4591         if (!was_up && now_up) {
4592                 ++non_subgraph_svcs;
4593         } else if (was_up && !now_up) {
4594                 --non_subgraph_svcs;
4595 
4596                 if (non_subgraph_svcs == 0) {
4597                         if (halting != -1) {
4598                                 do_uadmin();
4599                         } else if (go_single_user_mode || go_to_level1) {
4600                                 (void) startd_thread_create(single_user_thread,
4601                                     NULL);
4602                         }
4603                         return (0);
4604                 }
4605         }
4606 
4607         /* If this service is a leaf, it should be disabled. */
4608         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4609                 int r;
4610 
4611                 r = disable_service_temporarily(v, h);
4612                 switch (r) {
4613                 case 0:
4614                         break;
4615 
4616                 case ECONNABORTED:
4617                         ret = ECONNABORTED;
4618                         break;
4619 
4620                 default:
4621                         bad_error("disable_service_temporarily", r);
4622                 }
4623         }
4624 
4625         /*
4626          * If the service just came down, propagate the disable to the newly
4627          * exposed leaves.
4628          */
4629         if (was_up && !now_up)
4630                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4631                     (void *)h);
4632 
4633         return (ret);
4634 }
4635 
4636 /*
4637  * Decide whether to start up an sulogin thread after a service is
4638  * finished changing state.  Only need to do the full can_come_up()
4639  * evaluation if an instance is changing state, we're not halfway through
4640  * loading the thread, and we aren't shutting down or going to the single
4641  * user milestone.
4642  */
4643 void
4644 graph_transition_sulogin(restarter_instance_state_t state,
4645     restarter_instance_state_t old_state)
4646 {
4647         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4648 
4649         if (state != old_state && st->st_load_complete &&
4650             !go_single_user_mode && !go_to_level1 &&
4651             halting == -1) {
4652                 if (!sulogin_thread_running && !can_come_up()) {
4653                         (void) startd_thread_create(sulogin_thread, NULL);
4654                         sulogin_thread_running = B_TRUE;
4655                 }
4656         }
4657 }
4658 
4659 /*
4660  * Propagate a start, stop event, or a satisfiability event.
4661  *
4662  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4663  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4664  * full dependent graph to check for newly satisfied nodes.  This is
4665  * necessary for cases when non-direct dependents may be effected but direct
4666  * dependents may not (e.g. for optional_all evaluations, see the
4667  * propagate_satbility() comments).
4668  *
4669  * PROPAGATE_SAT should be used whenever a non-running service moves into
4670  * a state which can satisfy optional dependencies, like disabled or
4671  * maintenance.
4672  */
4673 void
4674 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4675     restarter_error_t rerr)
4676 {
4677         if (type == PROPAGATE_STOP) {
4678                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4679         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4680                 graph_walk_dependents(v, propagate_start, NULL);
4681 
4682                 if (type == PROPAGATE_SAT)
4683                         propagate_satbility(v);
4684         } else {
4685 #ifndef NDEBUG
4686                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4687                     __LINE__, type);
4688 #endif
4689                 abort();
4690         }
4691 }
4692 
4693 /*
4694  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4695  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4696  * all property group dependencies, and the dependency on the restarter,
4697  * disposing of vertices as appropriate.  If other vertices depend on this
4698  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4699  * returns 0.
4700  */
4701 static int
4702 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4703 {
4704         graph_vertex_t *v;
4705         graph_edge_t *e;
4706         uu_list_t *old_deps;
4707         int err;
4708 
4709         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4710 
4711         MUTEX_LOCK(&dgraph_lock);
4712 
4713         v = vertex_get_by_name(fmri);
4714         if (v == NULL) {
4715                 MUTEX_UNLOCK(&dgraph_lock);
4716                 return (0);
4717         }
4718 
4719         /* Send restarter delete event. */
4720         if (v->gv_flags & GV_CONFIGURED)
4721                 graph_unset_restarter(v);
4722 
4723         if (milestone > MILESTONE_NONE) {
4724                 /*
4725                  * Make a list of v's current dependencies so we can
4726                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4727                  * are removed.
4728                  */
4729                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4730 
4731                 err = uu_list_walk(v->gv_dependencies,
4732                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4733                 assert(err == 0);
4734         }
4735 
4736         delete_instance_dependencies(v, B_TRUE);
4737 
4738         /*
4739          * Deleting an instance can both satisfy and unsatisfy dependencies,
4740          * depending on their type.  First propagate the stop as a RERR_RESTART
4741          * event -- deletion isn't a fault, just a normal stop.  This gives
4742          * dependent services the chance to do a clean shutdown.  Then, mark
4743          * the service as unconfigured and propagate the start event for the
4744          * optional_all dependencies that might have become satisfied.
4745          */
4746         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
4747 
4748         v->gv_flags &= ~GV_CONFIGURED;
4749         v->gv_flags &= ~GV_DEATHROW;
4750 
4751         graph_walk_dependents(v, propagate_start, NULL);
4752         propagate_satbility(v);
4753 
4754         /*
4755          * If there are no (non-service) dependents, the vertex can be
4756          * completely removed.
4757          */
4758         if (v != milestone && v->gv_refs == 0 &&
4759             uu_list_numnodes(v->gv_dependents) == 1)
4760                 remove_inst_vertex(v);
4761 
4762         if (milestone > MILESTONE_NONE) {
4763                 void *cookie = NULL;
4764 
4765                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
4766                         v = e->ge_vertex;
4767 
4768                         if (vertex_unref(v) == VERTEX_INUSE)
4769                                 while (eval_subgraph(v, h) == ECONNABORTED)
4770                                         libscf_handle_rebind(h);
4771 
4772                         startd_free(e, sizeof (*e));
4773                 }
4774 
4775                 uu_list_destroy(old_deps);
4776         }
4777 
4778         MUTEX_UNLOCK(&dgraph_lock);
4779 
4780         return (0);
4781 }
4782 
4783 /*
4784  * Return the eventual (maybe current) milestone in the form of a
4785  * legacy runlevel.
4786  */
4787 static char
4788 target_milestone_as_runlevel()
4789 {
4790         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4791 
4792         if (milestone == NULL)
4793                 return ('3');
4794         else if (milestone == MILESTONE_NONE)
4795                 return ('0');
4796 
4797         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
4798                 return ('2');
4799         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
4800                 return ('S');
4801         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
4802                 return ('3');
4803 
4804 #ifndef NDEBUG
4805         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
4806             __FILE__, __LINE__, milestone->gv_name);
4807 #endif
4808         abort();
4809         /* NOTREACHED */
4810 }
4811 
4812 static struct {
4813         char    rl;
4814         int     sig;
4815 } init_sigs[] = {
4816         { 'S', SIGBUS },
4817         { '0', SIGINT },
4818         { '1', SIGQUIT },
4819         { '2', SIGILL },
4820         { '3', SIGTRAP },
4821         { '4', SIGIOT },
4822         { '5', SIGEMT },
4823         { '6', SIGFPE },
4824         { 0, 0 }
4825 };
4826 
4827 static void
4828 signal_init(char rl)
4829 {
4830         pid_t init_pid;
4831         int i;
4832 
4833         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4834 
4835         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
4836             sizeof (init_pid)) != sizeof (init_pid)) {
4837                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
4838                 return;
4839         }
4840 
4841         for (i = 0; init_sigs[i].rl != 0; ++i)
4842                 if (init_sigs[i].rl == rl)
4843                         break;
4844 
4845         if (init_sigs[i].rl != 0) {
4846                 if (kill(init_pid, init_sigs[i].sig) != 0) {
4847                         switch (errno) {
4848                         case EPERM:
4849                         case ESRCH:
4850                                 log_error(LOG_NOTICE, "Could not signal init: "
4851                                     "%s.\n", strerror(errno));
4852                                 break;
4853 
4854                         case EINVAL:
4855                         default:
4856                                 bad_error("kill", errno);
4857                         }
4858                 }
4859         }
4860 }
4861 
4862 /*
4863  * This is called when one of the major milestones changes state, or when
4864  * init is signalled and tells us it was told to change runlevel.  We wait
4865  * to reach the milestone because this allows /etc/inittab entries to retain
4866  * some boot ordering: historically, entries could place themselves before/after
4867  * the running of /sbin/rcX scripts but we can no longer make the
4868  * distinction because the /sbin/rcX scripts no longer exist as punctuation
4869  * marks in /etc/inittab.
4870  *
4871  * Also, we only trigger an update when we reach the eventual target
4872  * milestone: without this, an /etc/inittab entry marked only for
4873  * runlevel 2 would be executed for runlevel 3, which is not how
4874  * /etc/inittab entries work.
4875  *
4876  * If we're single user coming online, then we set utmpx to the target
4877  * runlevel so that legacy scripts can work as expected.
4878  */
4879 static void
4880 graph_runlevel_changed(char rl, int online)
4881 {
4882         char trl;
4883 
4884         assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4885 
4886         trl = target_milestone_as_runlevel();
4887 
4888         if (online) {
4889                 if (rl == trl) {
4890                         current_runlevel = trl;
4891                         signal_init(trl);
4892                 } else if (rl == 'S') {
4893                         /*
4894                          * At boot, set the entry early for the benefit of the
4895                          * legacy init scripts.
4896                          */
4897                         utmpx_set_runlevel(trl, 'S', B_FALSE);
4898                 }
4899         } else {
4900                 if (rl == '3' && trl == '2') {
4901                         current_runlevel = trl;
4902                         signal_init(trl);
4903                 } else if (rl == '2' && trl == 'S') {
4904                         current_runlevel = trl;
4905                         signal_init(trl);
4906                 }
4907         }
4908 }
4909 
4910 /*
4911  * Move to a backwards-compatible runlevel by executing the appropriate
4912  * /etc/rc?.d/K* scripts and/or setting the milestone.
4913  *
4914  * Returns
4915  *   0 - success
4916  *   ECONNRESET - success, but handle was reset
4917  *   ECONNABORTED - repository connection broken
4918  *   ECANCELED - pg was deleted
4919  */
4920 static int
4921 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
4922 {
4923         char rl;
4924         scf_handle_t *h;
4925         int r;
4926         const char *ms = NULL;  /* what to commit as options/milestone */
4927         boolean_t rebound = B_FALSE;
4928         int mark_rl = 0;
4929 
4930         const char * const stop = "stop";
4931 
4932         r = libscf_extract_runlevel(prop, &rl);
4933         switch (r) {
4934         case 0:
4935                 break;
4936 
4937         case ECONNABORTED:
4938         case ECANCELED:
4939                 return (r);
4940 
4941         case EINVAL:
4942         case ENOENT:
4943                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
4944                     "ignoring.\n");
4945                 /* delete the bad property */
4946                 goto nolock_out;
4947 
4948         default:
4949                 bad_error("libscf_extract_runlevel", r);
4950         }
4951 
4952         switch (rl) {
4953         case 's':
4954                 rl = 'S';
4955                 /* FALLTHROUGH */
4956 
4957         case 'S':
4958         case '2':
4959         case '3':
4960                 /*
4961                  * These cases cause a milestone change, so
4962                  * graph_runlevel_changed() will eventually deal with
4963                  * signalling init.
4964                  */
4965                 break;
4966 
4967         case '0':
4968         case '1':
4969         case '4':
4970         case '5':
4971         case '6':
4972                 mark_rl = 1;
4973                 break;
4974 
4975         default:
4976                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
4977                 ms = NULL;
4978                 goto nolock_out;
4979         }
4980 
4981         h = scf_pg_handle(pg);
4982 
4983         MUTEX_LOCK(&dgraph_lock);
4984 
4985         /*
4986          * Since this triggers no milestone changes, force it by hand.
4987          */
4988         if (current_runlevel == '4' && rl == '3')
4989                 mark_rl = 1;
4990 
4991         /*
4992          * 1. If we are here after an "init X":
4993          *
4994          * init X
4995          *      init/lscf_set_runlevel()
4996          *              process_pg_event()
4997          *              dgraph_set_runlevel()
4998          *
4999          * then we haven't passed through graph_runlevel_changed() yet,
5000          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5001          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5002          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5003          * past this test.
5004          *
5005          * 2. On the other hand, if we are here after a "svcadm milestone":
5006          *
5007          * svcadm milestone X
5008          *      dgraph_set_milestone()
5009          *              handle_graph_update_event()
5010          *              dgraph_set_instance_state()
5011          *              graph_post_X_[online|offline]()
5012          *              graph_runlevel_changed()
5013          *              signal_init()
5014          *                      init/lscf_set_runlevel()
5015          *                              process_pg_event()
5016          *                              dgraph_set_runlevel()
5017          *
5018          * then we already passed through graph_runlevel_changed() (by the way
5019          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5020          * and already be equal to 'rl' so we are going to return immediately
5021          * from dgraph_set_runlevel() without changing the system runlevel and
5022          * without executing the /etc/rc?.d/K* scripts.
5023          */
5024         if (rl == current_runlevel) {
5025                 ms = NULL;
5026                 goto out;
5027         }
5028 
5029         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5030 
5031         /*
5032          * Make sure stop rc scripts see the new settings via who -r.
5033          */
5034         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5035 
5036         /*
5037          * Some run levels don't have a direct correspondence to any
5038          * milestones, so we have to signal init directly.
5039          */
5040         if (mark_rl) {
5041                 current_runlevel = rl;
5042                 signal_init(rl);
5043         }
5044 
5045         switch (rl) {
5046         case 'S':
5047                 uu_warn("The system is coming down for administration.  "
5048                     "Please wait.\n");
5049                 fork_rc_script(rl, stop, B_FALSE);
5050                 ms = single_user_fmri;
5051                 go_single_user_mode = B_TRUE;
5052                 break;
5053 
5054         case '0':
5055                 halting_time = time(NULL);
5056                 fork_rc_script(rl, stop, B_TRUE);
5057                 halting = AD_HALT;
5058                 goto uadmin;
5059 
5060         case '5':
5061                 halting_time = time(NULL);
5062                 fork_rc_script(rl, stop, B_TRUE);
5063                 halting = AD_POWEROFF;
5064                 goto uadmin;
5065 
5066         case '6':
5067                 halting_time = time(NULL);
5068                 fork_rc_script(rl, stop, B_TRUE);
5069                 halting = AD_BOOT;
5070                 goto uadmin;
5071 
5072 uadmin:
5073                 uu_warn("The system is coming down.  Please wait.\n");
5074                 ms = "none";
5075 
5076                 /*
5077                  * We can't wait until all services are offline since this
5078                  * thread is responsible for taking them offline.  Instead we
5079                  * set halting to the second argument for uadmin() and call
5080                  * do_uadmin() from dgraph_set_instance_state() when
5081                  * appropriate.
5082                  */
5083                 break;
5084 
5085         case '1':
5086                 if (current_runlevel != 'S') {
5087                         uu_warn("Changing to state 1.\n");
5088                         fork_rc_script(rl, stop, B_FALSE);
5089                 } else {
5090                         uu_warn("The system is coming up for administration.  "
5091                             "Please wait.\n");
5092                 }
5093                 ms = single_user_fmri;
5094                 go_to_level1 = B_TRUE;
5095                 break;
5096 
5097         case '2':
5098                 if (current_runlevel == '3' || current_runlevel == '4')
5099                         fork_rc_script(rl, stop, B_FALSE);
5100                 ms = multi_user_fmri;
5101                 break;
5102 
5103         case '3':
5104         case '4':
5105                 ms = "all";
5106                 break;
5107 
5108         default:
5109 #ifndef NDEBUG
5110                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5111                     __FILE__, __LINE__, rl, rl);
5112 #endif
5113                 abort();
5114         }
5115 
5116 out:
5117         MUTEX_UNLOCK(&dgraph_lock);
5118 
5119 nolock_out:
5120         switch (r = libscf_clear_runlevel(pg, ms)) {
5121         case 0:
5122                 break;
5123 
5124         case ECONNABORTED:
5125                 libscf_handle_rebind(h);
5126                 rebound = B_TRUE;
5127                 goto nolock_out;
5128 
5129         case ECANCELED:
5130                 break;
5131 
5132         case EPERM:
5133         case EACCES:
5134         case EROFS:
5135                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5136                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5137                 break;
5138 
5139         default:
5140                 bad_error("libscf_clear_runlevel", r);
5141         }
5142 
5143         return (rebound ? ECONNRESET : 0);
5144 }
5145 
5146 /*
5147  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5148  * to the instances that are supposed to go offline during an
5149  * administrative disable operation.
5150  */
5151 static int
5152 mark_subtree(graph_edge_t *e, void *arg)
5153 {
5154         graph_vertex_t *v;
5155         int r;
5156 
5157         v = e->ge_vertex;
5158 
5159         /* If it's already in the subgraph, skip. */
5160         if (v->gv_flags & GV_TOOFFLINE)
5161                 return (UU_WALK_NEXT);
5162 
5163         switch (v->gv_type) {
5164         case GVT_INST:
5165                 /* If the instance is already disabled, skip it. */
5166                 if (!(v->gv_flags & GV_ENABLED))
5167                         return (UU_WALK_NEXT);
5168 
5169                 v->gv_flags |= GV_TOOFFLINE;
5170                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5171                 break;
5172         case GVT_GROUP:
5173                 /*
5174                  * Skip all excluded and optional_all dependencies and decide
5175                  * whether to offline the service based on restart_on attribute.
5176                  */
5177                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL ||
5178                     v->gv_depgroup == DEPGRP_OPTIONAL_ALL ||
5179                     v->gv_restart < RERR_RESTART)
5180                         return (UU_WALK_NEXT);
5181                 break;
5182         }
5183 
5184         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5185             0);
5186         assert(r == 0);
5187         return (UU_WALK_NEXT);
5188 }
5189 
5190 static int
5191 mark_subgraph(graph_edge_t *e, void *arg)
5192 {
5193         graph_vertex_t *v;
5194         int r;
5195         int optional = (int)arg;
5196 
5197         v = e->ge_vertex;
5198 
5199         /* If it's already in the subgraph, skip. */
5200         if (v->gv_flags & GV_INSUBGRAPH)
5201                 return (UU_WALK_NEXT);
5202 
5203         /*
5204          * Keep track if walk has entered an optional dependency group
5205          */
5206         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5207                 optional = 1;
5208         }
5209         /*
5210          * Quit if we are in an optional dependency group and the instance
5211          * is disabled
5212          */
5213         if (optional && (v->gv_type == GVT_INST) &&
5214             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5215                 return (UU_WALK_NEXT);
5216 
5217         v->gv_flags |= GV_INSUBGRAPH;
5218 
5219         /* Skip all excluded dependencies. */
5220         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5221                 return (UU_WALK_NEXT);
5222 
5223         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5224             (void *)optional, 0);
5225         assert(r == 0);
5226         return (UU_WALK_NEXT);
5227 }
5228 
5229 /*
5230  * Bring down all services which are not dependencies of fmri.  The
5231  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5232  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5233  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5234  * could consider "offline" to be down, and refrain from sending start
5235  * commands for such services, but that's not strictly necessary, so we'll
5236  * decline to intrude on the state machine.  It would probably confuse users
5237  * anyway.
5238  *
5239  * The services should be brought down in reverse-dependency order, so we
5240  * can't do it all at once here.  We initiate by override-disabling the leaves
5241  * of the dependency tree -- those services which are up but have no
5242  * dependents which are up.  When they come down,
5243  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5244  * exposed leaves.  Perseverance will ensure completion.
5245  *
5246  * Sometimes we need to take action when the transition is complete, like
5247  * start sulogin or halt the system.  To tell when we're done, we initialize
5248  * non_subgraph_svcs here to be the number of services which need to come
5249  * down.  As each does, we decrement the counter.  When it hits zero, we take
5250  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5251  *
5252  * In case we're coming up, we also remove any enable-overrides for the
5253  * services which are dependencies of fmri.
5254  *
5255  * If norepository is true, the function will not change the repository.
5256  *
5257  * The decision to change the system run level in accordance with the milestone
5258  * is taken in dgraph_set_runlevel().
5259  *
5260  * Returns
5261  *   0 - success
5262  *   ECONNRESET - success, but handle was rebound
5263  *   EINVAL - fmri is invalid (error is logged)
5264  *   EALREADY - the milestone is already set to fmri
5265  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5266  */
5267 static int
5268 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5269 {
5270         const char *cfmri, *fs;
5271         graph_vertex_t *nm, *v;
5272         int ret = 0, r;
5273         scf_instance_t *inst;
5274         boolean_t isall, isnone, rebound = B_FALSE;
5275 
5276         /* Validate fmri */
5277         isall = (strcmp(fmri, "all") == 0);
5278         isnone = (strcmp(fmri, "none") == 0);
5279 
5280         if (!isall && !isnone) {
5281                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5282                         goto reject;
5283 
5284                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5285                     strcmp(cfmri, multi_user_fmri) != 0 &&
5286                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5287                         startd_free((void *)cfmri, max_scf_fmri_size);
5288 reject:
5289                         log_framework(LOG_WARNING,
5290                             "Rejecting request for invalid milestone \"%s\".\n",
5291                             fmri);
5292                         return (EINVAL);
5293                 }
5294         }
5295 
5296         inst = safe_scf_instance_create(h);
5297 
5298         MUTEX_LOCK(&dgraph_lock);
5299 
5300         if (milestone == NULL) {
5301                 if (isall) {
5302                         log_framework(LOG_DEBUG,
5303                             "Milestone already set to all.\n");
5304                         ret = EALREADY;
5305                         goto out;
5306                 }
5307         } else if (milestone == MILESTONE_NONE) {
5308                 if (isnone) {
5309                         log_framework(LOG_DEBUG,
5310                             "Milestone already set to none.\n");
5311                         ret = EALREADY;
5312                         goto out;
5313                 }
5314         } else {
5315                 if (!isall && !isnone &&
5316                     strcmp(cfmri, milestone->gv_name) == 0) {
5317                         log_framework(LOG_DEBUG,
5318                             "Milestone already set to %s.\n", cfmri);
5319                         ret = EALREADY;
5320                         goto out;
5321                 }
5322         }
5323 
5324         if (!isall && !isnone) {
5325                 nm = vertex_get_by_name(cfmri);
5326                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5327                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5328                             "because no such service exists.\n", cfmri);
5329                         ret = ENOENT;
5330                         goto out;
5331                 }
5332         }
5333 
5334         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5335 
5336         /*
5337          * Set milestone, removing the old one if this was the last reference.
5338          */
5339         if (milestone > MILESTONE_NONE)
5340                 (void) vertex_unref(milestone);
5341 
5342         if (isall)
5343                 milestone = NULL;
5344         else if (isnone)
5345                 milestone = MILESTONE_NONE;
5346         else {
5347                 milestone = nm;
5348                 /* milestone should count as a reference */
5349                 vertex_ref(milestone);
5350         }
5351 
5352         /* Clear all GV_INSUBGRAPH bits. */
5353         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5354                 v->gv_flags &= ~GV_INSUBGRAPH;
5355 
5356         if (!isall && !isnone) {
5357                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5358                 milestone->gv_flags |= GV_INSUBGRAPH;
5359 
5360                 r = uu_list_walk(milestone->gv_dependencies,
5361                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5362                 assert(r == 0);
5363         }
5364 
5365         /* Un-override services in the subgraph & override-disable the rest. */
5366         if (norepository)
5367                 goto out;
5368 
5369         non_subgraph_svcs = 0;
5370         for (v = uu_list_first(dgraph);
5371             v != NULL;
5372             v = uu_list_next(dgraph, v)) {
5373                 if (v->gv_type != GVT_INST ||
5374                     (v->gv_flags & GV_CONFIGURED) == 0)
5375                         continue;
5376 
5377 again:
5378                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5379                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5380                 if (r != 0) {
5381                         switch (scf_error()) {
5382                         case SCF_ERROR_CONNECTION_BROKEN:
5383                         default:
5384                                 libscf_handle_rebind(h);
5385                                 rebound = B_TRUE;
5386                                 goto again;
5387 
5388                         case SCF_ERROR_NOT_FOUND:
5389                                 continue;
5390 
5391                         case SCF_ERROR_HANDLE_MISMATCH:
5392                         case SCF_ERROR_INVALID_ARGUMENT:
5393                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5394                         case SCF_ERROR_NOT_BOUND:
5395                                 bad_error("scf_handle_decode_fmri",
5396                                     scf_error());
5397                         }
5398                 }
5399 
5400                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5401                         r = libscf_delete_enable_ovr(inst);
5402                         fs = "libscf_delete_enable_ovr";
5403                 } else {
5404                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5405 
5406                         /*
5407                          * Services which are up need to come down before
5408                          * we're done, but we can only disable the leaves
5409                          * here.
5410                          */
5411 
5412                         if (up_state(v->gv_state))
5413                                 ++non_subgraph_svcs;
5414 
5415                         /* If it's already disabled, don't bother. */
5416                         if ((v->gv_flags & GV_ENABLED) == 0)
5417                                 continue;
5418 
5419                         if (!is_nonsubgraph_leaf(v))
5420                                 continue;
5421 
5422                         r = libscf_set_enable_ovr(inst, 0);
5423                         fs = "libscf_set_enable_ovr";
5424                 }
5425                 switch (r) {
5426                 case 0:
5427                 case ECANCELED:
5428                         break;
5429 
5430                 case ECONNABORTED:
5431                         libscf_handle_rebind(h);
5432                         rebound = B_TRUE;
5433                         goto again;
5434 
5435                 case EPERM:
5436                 case EROFS:
5437                         log_error(LOG_WARNING,
5438                             "Could not set %s/%s for %s: %s.\n",
5439                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5440                             v->gv_name, strerror(r));
5441                         break;
5442 
5443                 default:
5444                         bad_error(fs, r);
5445                 }
5446         }
5447 
5448         if (halting != -1) {
5449                 if (non_subgraph_svcs > 1)
5450                         uu_warn("%d system services are now being stopped.\n",
5451                             non_subgraph_svcs);
5452                 else if (non_subgraph_svcs == 1)
5453                         uu_warn("One system service is now being stopped.\n");
5454                 else if (non_subgraph_svcs == 0)
5455                         do_uadmin();
5456         }
5457 
5458         ret = rebound ? ECONNRESET : 0;
5459 
5460 out:
5461         MUTEX_UNLOCK(&dgraph_lock);
5462         if (!isall && !isnone)
5463                 startd_free((void *)cfmri, max_scf_fmri_size);
5464         scf_instance_destroy(inst);
5465         return (ret);
5466 }
5467 
5468 
5469 /*
5470  * Returns 0, ECONNABORTED, or EINVAL.
5471  */
5472 static int
5473 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5474 {
5475         int r;
5476 
5477         switch (e->gpe_type) {
5478         case GRAPH_UPDATE_RELOAD_GRAPH:
5479                 log_error(LOG_WARNING,
5480                     "graph_event: reload graph unimplemented\n");
5481                 break;
5482 
5483         case GRAPH_UPDATE_STATE_CHANGE: {
5484                 protocol_states_t *states = e->gpe_data;
5485 
5486                 switch (r = dgraph_set_instance_state(h, e->gpe_inst,
5487                     states->ps_state, states->ps_err)) {
5488                 case 0:
5489                 case ENOENT:
5490                         break;
5491 
5492                 case ECONNABORTED:
5493                         return (ECONNABORTED);
5494 
5495                 case EINVAL:
5496                 default:
5497 #ifndef NDEBUG
5498                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5499                             "failed with unexpected error %d at %s:%d.\n", r,
5500                             __FILE__, __LINE__);
5501 #endif
5502                         abort();
5503                 }
5504 
5505                 startd_free(states, sizeof (protocol_states_t));
5506                 break;
5507         }
5508 
5509         default:
5510                 log_error(LOG_WARNING,
5511                     "graph_event_loop received an unknown event: %d\n",
5512                     e->gpe_type);
5513                 break;
5514         }
5515 
5516         return (0);
5517 }
5518 
5519 /*
5520  * graph_event_thread()
5521  *    Wait for state changes from the restarters.
5522  */
5523 /*ARGSUSED*/
5524 void *
5525 graph_event_thread(void *unused)
5526 {
5527         scf_handle_t *h;
5528         int err;
5529 
5530         h = libscf_handle_create_bound_loop();
5531 
5532         /*CONSTCOND*/
5533         while (1) {
5534                 graph_protocol_event_t *e;
5535 
5536                 MUTEX_LOCK(&gu->gu_lock);
5537 
5538                 while (gu->gu_wakeup == 0)
5539                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5540 
5541                 gu->gu_wakeup = 0;
5542 
5543                 while ((e = graph_event_dequeue()) != NULL) {
5544                         MUTEX_LOCK(&e->gpe_lock);
5545                         MUTEX_UNLOCK(&gu->gu_lock);
5546 
5547                         while ((err = handle_graph_update_event(h, e)) ==
5548                             ECONNABORTED)
5549                                 libscf_handle_rebind(h);
5550 
5551                         if (err == 0)
5552                                 graph_event_release(e);
5553                         else
5554                                 graph_event_requeue(e);
5555 
5556                         MUTEX_LOCK(&gu->gu_lock);
5557                 }
5558 
5559                 MUTEX_UNLOCK(&gu->gu_lock);
5560         }
5561 
5562         /*
5563          * Unreachable for now -- there's currently no graceful cleanup
5564          * called on exit().
5565          */
5566         MUTEX_UNLOCK(&gu->gu_lock);
5567         scf_handle_destroy(h);
5568         return (NULL);
5569 }
5570 
5571 static void
5572 set_initial_milestone(scf_handle_t *h)
5573 {
5574         scf_instance_t *inst;
5575         char *fmri, *cfmri;
5576         size_t sz;
5577         int r;
5578 
5579         inst = safe_scf_instance_create(h);
5580         fmri = startd_alloc(max_scf_fmri_size);
5581 
5582         /*
5583          * If -m milestone= was specified, we want to set options_ovr/milestone
5584          * to it.  Otherwise we want to read what the milestone should be set
5585          * to.  Either way we need our inst.
5586          */
5587 get_self:
5588         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5589             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5590                 switch (scf_error()) {
5591                 case SCF_ERROR_CONNECTION_BROKEN:
5592                         libscf_handle_rebind(h);
5593                         goto get_self;
5594 
5595                 case SCF_ERROR_NOT_FOUND:
5596                         if (st->st_subgraph != NULL &&
5597                             st->st_subgraph[0] != '\0') {
5598                                 sz = strlcpy(fmri, st->st_subgraph,
5599                                     max_scf_fmri_size);
5600                                 assert(sz < max_scf_fmri_size);
5601                         } else {
5602                                 fmri[0] = '\0';
5603                         }
5604                         break;
5605 
5606                 case SCF_ERROR_INVALID_ARGUMENT:
5607                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5608                 case SCF_ERROR_HANDLE_MISMATCH:
5609                 default:
5610                         bad_error("scf_handle_decode_fmri", scf_error());
5611                 }
5612         } else {
5613                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5614                         scf_propertygroup_t *pg;
5615 
5616                         pg = safe_scf_pg_create(h);
5617 
5618                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5619                         assert(sz < max_scf_fmri_size);
5620 
5621                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5622                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5623                             pg);
5624                         switch (r) {
5625                         case 0:
5626                                 break;
5627 
5628                         case ECONNABORTED:
5629                                 libscf_handle_rebind(h);
5630                                 goto get_self;
5631 
5632                         case EPERM:
5633                         case EACCES:
5634                         case EROFS:
5635                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5636                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5637                                     SCF_PROPERTY_MILESTONE, strerror(r));
5638                                 /* FALLTHROUGH */
5639 
5640                         case ECANCELED:
5641                                 sz = strlcpy(fmri, st->st_subgraph,
5642                                     max_scf_fmri_size);
5643                                 assert(sz < max_scf_fmri_size);
5644                                 break;
5645 
5646                         default:
5647                                 bad_error("libscf_inst_get_or_add_pg", r);
5648                         }
5649 
5650                         r = libscf_clear_runlevel(pg, fmri);
5651                         switch (r) {
5652                         case 0:
5653                                 break;
5654 
5655                         case ECONNABORTED:
5656                                 libscf_handle_rebind(h);
5657                                 goto get_self;
5658 
5659                         case EPERM:
5660                         case EACCES:
5661                         case EROFS:
5662                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5663                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5664                                     SCF_PROPERTY_MILESTONE, strerror(r));
5665                                 /* FALLTHROUGH */
5666 
5667                         case ECANCELED:
5668                                 sz = strlcpy(fmri, st->st_subgraph,
5669                                     max_scf_fmri_size);
5670                                 assert(sz < max_scf_fmri_size);
5671                                 break;
5672 
5673                         default:
5674                                 bad_error("libscf_clear_runlevel", r);
5675                         }
5676 
5677                         scf_pg_destroy(pg);
5678                 } else {
5679                         scf_property_t *prop;
5680                         scf_value_t *val;
5681 
5682                         prop = safe_scf_property_create(h);
5683                         val = safe_scf_value_create(h);
5684 
5685                         r = libscf_get_milestone(inst, prop, val, fmri,
5686                             max_scf_fmri_size);
5687                         switch (r) {
5688                         case 0:
5689                                 break;
5690 
5691                         case ECONNABORTED:
5692                                 libscf_handle_rebind(h);
5693                                 goto get_self;
5694 
5695                         case EINVAL:
5696                                 log_error(LOG_WARNING, "Milestone property is "
5697                                     "misconfigured.  Defaulting to \"all\".\n");
5698                                 /* FALLTHROUGH */
5699 
5700                         case ECANCELED:
5701                         case ENOENT:
5702                                 fmri[0] = '\0';
5703                                 break;
5704 
5705                         default:
5706                                 bad_error("libscf_get_milestone", r);
5707                         }
5708 
5709                         scf_value_destroy(val);
5710                         scf_property_destroy(prop);
5711                 }
5712         }
5713 
5714         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5715                 goto out;
5716 
5717         if (strcmp(fmri, "none") != 0) {
5718 retry:
5719                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5720                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5721                         switch (scf_error()) {
5722                         case SCF_ERROR_INVALID_ARGUMENT:
5723                                 log_error(LOG_WARNING,
5724                                     "Requested milestone \"%s\" is invalid.  "
5725                                     "Reverting to \"all\".\n", fmri);
5726                                 goto out;
5727 
5728                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5729                                 log_error(LOG_WARNING, "Requested milestone "
5730                                     "\"%s\" does not specify an instance.  "
5731                                     "Reverting to \"all\".\n", fmri);
5732                                 goto out;
5733 
5734                         case SCF_ERROR_CONNECTION_BROKEN:
5735                                 libscf_handle_rebind(h);
5736                                 goto retry;
5737 
5738                         case SCF_ERROR_NOT_FOUND:
5739                                 log_error(LOG_WARNING, "Requested milestone "
5740                                     "\"%s\" not in repository.  Reverting to "
5741                                     "\"all\".\n", fmri);
5742                                 goto out;
5743 
5744                         case SCF_ERROR_HANDLE_MISMATCH:
5745                         default:
5746                                 bad_error("scf_handle_decode_fmri",
5747                                     scf_error());
5748                         }
5749                 }
5750 
5751                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
5752                 assert(r == 0);
5753 
5754                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
5755                 startd_free(cfmri, max_scf_fmri_size);
5756                 switch (r) {
5757                 case 0:
5758                         break;
5759 
5760                 case ECONNABORTED:
5761                         goto retry;
5762 
5763                 case EINVAL:
5764                         log_error(LOG_WARNING,
5765                             "Requested milestone \"%s\" is invalid.  "
5766                             "Reverting to \"all\".\n", fmri);
5767                         goto out;
5768 
5769                 case ECANCELED:
5770                         log_error(LOG_WARNING,
5771                             "Requested milestone \"%s\" not "
5772                             "in repository.  Reverting to \"all\".\n",
5773                             fmri);
5774                         goto out;
5775 
5776                 case EEXIST:
5777                 default:
5778                         bad_error("dgraph_add_instance", r);
5779                 }
5780         }
5781 
5782         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
5783 
5784         r = dgraph_set_milestone(fmri, h, B_FALSE);
5785         switch (r) {
5786         case 0:
5787         case ECONNRESET:
5788         case EALREADY:
5789                 break;
5790 
5791         case EINVAL:
5792         case ENOENT:
5793         default:
5794                 bad_error("dgraph_set_milestone", r);
5795         }
5796 
5797 out:
5798         startd_free(fmri, max_scf_fmri_size);
5799         scf_instance_destroy(inst);
5800 }
5801 
5802 void
5803 set_restart_milestone(scf_handle_t *h)
5804 {
5805         scf_instance_t *inst;
5806         scf_property_t *prop;
5807         scf_value_t *val;
5808         char *fmri;
5809         int r;
5810 
5811         inst = safe_scf_instance_create(h);
5812 
5813 get_self:
5814         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
5815             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5816                 switch (scf_error()) {
5817                 case SCF_ERROR_CONNECTION_BROKEN:
5818                         libscf_handle_rebind(h);
5819                         goto get_self;
5820 
5821                 case SCF_ERROR_NOT_FOUND:
5822                         break;
5823 
5824                 case SCF_ERROR_INVALID_ARGUMENT:
5825                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5826                 case SCF_ERROR_HANDLE_MISMATCH:
5827                 default:
5828                         bad_error("scf_handle_decode_fmri", scf_error());
5829                 }
5830 
5831                 scf_instance_destroy(inst);
5832                 return;
5833         }
5834 
5835         prop = safe_scf_property_create(h);
5836         val = safe_scf_value_create(h);
5837         fmri = startd_alloc(max_scf_fmri_size);
5838 
5839         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
5840         switch (r) {
5841         case 0:
5842                 break;
5843 
5844         case ECONNABORTED:
5845                 libscf_handle_rebind(h);
5846                 goto get_self;
5847 
5848         case ECANCELED:
5849         case ENOENT:
5850         case EINVAL:
5851                 goto out;
5852 
5853         default:
5854                 bad_error("libscf_get_milestone", r);
5855         }
5856 
5857         r = dgraph_set_milestone(fmri, h, B_TRUE);
5858         switch (r) {
5859         case 0:
5860         case ECONNRESET:
5861         case EALREADY:
5862         case EINVAL:
5863         case ENOENT:
5864                 break;
5865 
5866         default:
5867                 bad_error("dgraph_set_milestone", r);
5868         }
5869 
5870 out:
5871         startd_free(fmri, max_scf_fmri_size);
5872         scf_value_destroy(val);
5873         scf_property_destroy(prop);
5874         scf_instance_destroy(inst);
5875 }
5876 
5877 /*
5878  * void *graph_thread(void *)
5879  *
5880  * Graph management thread.
5881  */
5882 /*ARGSUSED*/
5883 void *
5884 graph_thread(void *arg)
5885 {
5886         scf_handle_t *h;
5887         int err;
5888 
5889         h = libscf_handle_create_bound_loop();
5890 
5891         if (st->st_initial)
5892                 set_initial_milestone(h);
5893 
5894         MUTEX_LOCK(&dgraph_lock);
5895         initial_milestone_set = B_TRUE;
5896         err = pthread_cond_broadcast(&initial_milestone_cv);
5897         assert(err == 0);
5898         MUTEX_UNLOCK(&dgraph_lock);
5899 
5900         libscf_populate_graph(h);
5901 
5902         if (!st->st_initial)
5903                 set_restart_milestone(h);
5904 
5905         MUTEX_LOCK(&st->st_load_lock);
5906         st->st_load_complete = 1;
5907         (void) pthread_cond_broadcast(&st->st_load_cv);
5908         MUTEX_UNLOCK(&st->st_load_lock);
5909 
5910         MUTEX_LOCK(&dgraph_lock);
5911         /*
5912          * Now that we've set st_load_complete we need to check can_come_up()
5913          * since if we booted to a milestone, then there won't be any more
5914          * state updates.
5915          */
5916         if (!go_single_user_mode && !go_to_level1 &&
5917             halting == -1) {
5918                 if (!sulogin_thread_running && !can_come_up()) {
5919                         (void) startd_thread_create(sulogin_thread, NULL);
5920                         sulogin_thread_running = B_TRUE;
5921                 }
5922         }
5923         MUTEX_UNLOCK(&dgraph_lock);
5924 
5925         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
5926 
5927         /*CONSTCOND*/
5928         while (1) {
5929                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
5930                     &gu->gu_freeze_lock);
5931         }
5932 
5933         /*
5934          * Unreachable for now -- there's currently no graceful cleanup
5935          * called on exit().
5936          */
5937         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
5938         scf_handle_destroy(h);
5939 
5940         return (NULL);
5941 }
5942 
5943 
5944 /*
5945  * int next_action()
5946  *   Given an array of timestamps 'a' with 'num' elements, find the
5947  *   lowest non-zero timestamp and return its index. If there are no
5948  *   non-zero elements, return -1.
5949  */
5950 static int
5951 next_action(hrtime_t *a, int num)
5952 {
5953         hrtime_t t = 0;
5954         int i = 0, smallest = -1;
5955 
5956         for (i = 0; i < num; i++) {
5957                 if (t == 0) {
5958                         t = a[i];
5959                         smallest = i;
5960                 } else if (a[i] != 0 && a[i] < t) {
5961                         t = a[i];
5962                         smallest = i;
5963                 }
5964         }
5965 
5966         if (t == 0)
5967                 return (-1);
5968         else
5969                 return (smallest);
5970 }
5971 
5972 /*
5973  * void process_actions()
5974  *   Process actions requested by the administrator. Possibilities include:
5975  *   refresh, restart, maintenance mode off, maintenance mode on,
5976  *   maintenance mode immediate, and degraded.
5977  *
5978  *   The set of pending actions is represented in the repository as a
5979  *   per-instance property group, with each action being a single property
5980  *   in that group.  This property group is converted to an array, with each
5981  *   action type having an array slot.  The actions in the array at the
5982  *   time process_actions() is called are acted on in the order of the
5983  *   timestamp (which is the value stored in the slot).  A value of zero
5984  *   indicates that there is no pending action of the type associated with
5985  *   a particular slot.
5986  *
5987  *   Sending an action event multiple times before the restarter has a
5988  *   chance to process that action will force it to be run at the last
5989  *   timestamp where it appears in the ordering.
5990  *
5991  *   Turning maintenance mode on trumps all other actions.
5992  *
5993  *   Returns 0 or ECONNABORTED.
5994  */
5995 static int
5996 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
5997 {
5998         scf_property_t *prop = NULL;
5999         scf_value_t *val = NULL;
6000         scf_type_t type;
6001         graph_vertex_t *vertex;
6002         admin_action_t a;
6003         int i, ret = 0, r;
6004         hrtime_t action_ts[NACTIONS];
6005         char *inst_name;
6006 
6007         r = libscf_instance_get_fmri(inst, &inst_name);
6008         switch (r) {
6009         case 0:
6010                 break;
6011 
6012         case ECONNABORTED:
6013                 return (ECONNABORTED);
6014 
6015         case ECANCELED:
6016                 return (0);
6017 
6018         default:
6019                 bad_error("libscf_instance_get_fmri", r);
6020         }
6021 
6022         MUTEX_LOCK(&dgraph_lock);
6023 
6024         vertex = vertex_get_by_name(inst_name);
6025         if (vertex == NULL) {
6026                 MUTEX_UNLOCK(&dgraph_lock);
6027                 startd_free(inst_name, max_scf_fmri_size);
6028                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6029                     "The instance must have been removed.\n", inst_name);
6030                 return (0);
6031         }
6032 
6033         prop = safe_scf_property_create(h);
6034         val = safe_scf_value_create(h);
6035 
6036         for (i = 0; i < NACTIONS; i++) {
6037                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6038                         switch (scf_error()) {
6039                         case SCF_ERROR_CONNECTION_BROKEN:
6040                         default:
6041                                 ret = ECONNABORTED;
6042                                 goto out;
6043 
6044                         case SCF_ERROR_DELETED:
6045                                 goto out;
6046 
6047                         case SCF_ERROR_NOT_FOUND:
6048                                 action_ts[i] = 0;
6049                                 continue;
6050 
6051                         case SCF_ERROR_HANDLE_MISMATCH:
6052                         case SCF_ERROR_INVALID_ARGUMENT:
6053                         case SCF_ERROR_NOT_SET:
6054                                 bad_error("scf_pg_get_property", scf_error());
6055                         }
6056                 }
6057 
6058                 if (scf_property_type(prop, &type) != 0) {
6059                         switch (scf_error()) {
6060                         case SCF_ERROR_CONNECTION_BROKEN:
6061                         default:
6062                                 ret = ECONNABORTED;
6063                                 goto out;
6064 
6065                         case SCF_ERROR_DELETED:
6066                                 action_ts[i] = 0;
6067                                 continue;
6068 
6069                         case SCF_ERROR_NOT_SET:
6070                                 bad_error("scf_property_type", scf_error());
6071                         }
6072                 }
6073 
6074                 if (type != SCF_TYPE_INTEGER) {
6075                         action_ts[i] = 0;
6076                         continue;
6077                 }
6078 
6079                 if (scf_property_get_value(prop, val) != 0) {
6080                         switch (scf_error()) {
6081                         case SCF_ERROR_CONNECTION_BROKEN:
6082                         default:
6083                                 ret = ECONNABORTED;
6084                                 goto out;
6085 
6086                         case SCF_ERROR_DELETED:
6087                                 goto out;
6088 
6089                         case SCF_ERROR_NOT_FOUND:
6090                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6091                                 action_ts[i] = 0;
6092                                 continue;
6093 
6094                         case SCF_ERROR_NOT_SET:
6095                         case SCF_ERROR_PERMISSION_DENIED:
6096                                 bad_error("scf_property_get_value",
6097                                     scf_error());
6098                         }
6099                 }
6100 
6101                 r = scf_value_get_integer(val, &action_ts[i]);
6102                 assert(r == 0);
6103         }
6104 
6105         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6106         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6107             action_ts[ADMIN_EVENT_MAINT_ON]) {
6108                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6109                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6110 
6111                 vertex_send_event(vertex, admin_events[a]);
6112                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6113                 switch (r) {
6114                 case 0:
6115                 case EACCES:
6116                         break;
6117 
6118                 case ECONNABORTED:
6119                         ret = ECONNABORTED;
6120                         goto out;
6121 
6122                 case EPERM:
6123                         uu_die("Insufficient privilege.\n");
6124                         /* NOTREACHED */
6125 
6126                 default:
6127                         bad_error("libscf_unset_action", r);
6128                 }
6129         }
6130 
6131         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6132                 log_framework(LOG_DEBUG,
6133                     "Graph: processing %s action for %s.\n", admin_actions[a],
6134                     inst_name);
6135 
6136                 if (a == ADMIN_EVENT_REFRESH) {
6137                         r = dgraph_refresh_instance(vertex, inst);
6138                         switch (r) {
6139                         case 0:
6140                         case ECANCELED:
6141                         case EINVAL:
6142                         case -1:
6143                                 break;
6144 
6145                         case ECONNABORTED:
6146                                 /* pg & inst are reset now, so just return. */
6147                                 ret = ECONNABORTED;
6148                                 goto out;
6149 
6150                         default:
6151                                 bad_error("dgraph_refresh_instance", r);
6152                         }
6153                 }
6154 
6155                 vertex_send_event(vertex, admin_events[a]);
6156 
6157                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6158                 switch (r) {
6159                 case 0:
6160                 case EACCES:
6161                         break;
6162 
6163                 case ECONNABORTED:
6164                         ret = ECONNABORTED;
6165                         goto out;
6166 
6167                 case EPERM:
6168                         uu_die("Insufficient privilege.\n");
6169                         /* NOTREACHED */
6170 
6171                 default:
6172                         bad_error("libscf_unset_action", r);
6173                 }
6174 
6175                 action_ts[a] = 0;
6176         }
6177 
6178 out:
6179         MUTEX_UNLOCK(&dgraph_lock);
6180 
6181         scf_property_destroy(prop);
6182         scf_value_destroy(val);
6183         startd_free(inst_name, max_scf_fmri_size);
6184         return (ret);
6185 }
6186 
6187 /*
6188  * inst and pg_name are scratch space, and are unset on entry.
6189  * Returns
6190  *   0 - success
6191  *   ECONNRESET - success, but repository handle rebound
6192  *   ECONNABORTED - repository connection broken
6193  */
6194 static int
6195 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6196     char *pg_name)
6197 {
6198         int r;
6199         scf_property_t *prop;
6200         scf_value_t *val;
6201         char *fmri;
6202         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6203 
6204         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6205                 switch (scf_error()) {
6206                 case SCF_ERROR_CONNECTION_BROKEN:
6207                 default:
6208                         return (ECONNABORTED);
6209 
6210                 case SCF_ERROR_DELETED:
6211                         return (0);
6212 
6213                 case SCF_ERROR_NOT_SET:
6214                         bad_error("scf_pg_get_name", scf_error());
6215                 }
6216         }
6217 
6218         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6219             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6220                 r = dgraph_update_general(pg);
6221                 switch (r) {
6222                 case 0:
6223                 case ENOTSUP:
6224                 case ECANCELED:
6225                         return (0);
6226 
6227                 case ECONNABORTED:
6228                         return (ECONNABORTED);
6229 
6230                 case -1:
6231                         /* Error should have been logged. */
6232                         return (0);
6233 
6234                 default:
6235                         bad_error("dgraph_update_general", r);
6236                 }
6237         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6238                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6239                         switch (scf_error()) {
6240                         case SCF_ERROR_CONNECTION_BROKEN:
6241                                 return (ECONNABORTED);
6242 
6243                         case SCF_ERROR_DELETED:
6244                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6245                                 /* Ignore commands on services. */
6246                                 return (0);
6247 
6248                         case SCF_ERROR_NOT_BOUND:
6249                         case SCF_ERROR_HANDLE_MISMATCH:
6250                         case SCF_ERROR_NOT_SET:
6251                         default:
6252                                 bad_error("scf_pg_get_parent_instance",
6253                                     scf_error());
6254                         }
6255                 }
6256 
6257                 return (process_actions(h, pg, inst));
6258         }
6259 
6260         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6261             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6262                 return (0);
6263 
6264         /*
6265          * We only care about the options[_ovr] property groups of our own
6266          * instance, so get the fmri and compare.  Plus, once we know it's
6267          * correct, if the repository connection is broken we know exactly what
6268          * property group we were operating on, and can look it up again.
6269          */
6270         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6271                 switch (scf_error()) {
6272                 case SCF_ERROR_CONNECTION_BROKEN:
6273                         return (ECONNABORTED);
6274 
6275                 case SCF_ERROR_DELETED:
6276                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6277                         return (0);
6278 
6279                 case SCF_ERROR_HANDLE_MISMATCH:
6280                 case SCF_ERROR_NOT_BOUND:
6281                 case SCF_ERROR_NOT_SET:
6282                 default:
6283                         bad_error("scf_pg_get_parent_instance",
6284                             scf_error());
6285                 }
6286         }
6287 
6288         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6289         case 0:
6290                 break;
6291 
6292         case ECONNABORTED:
6293                 return (ECONNABORTED);
6294 
6295         case ECANCELED:
6296                 return (0);
6297 
6298         default:
6299                 bad_error("libscf_instance_get_fmri", r);
6300         }
6301 
6302         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6303                 startd_free(fmri, max_scf_fmri_size);
6304                 return (0);
6305         }
6306 
6307         prop = safe_scf_property_create(h);
6308         val = safe_scf_value_create(h);
6309 
6310         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6311                 /* See if we need to set the runlevel. */
6312                 /* CONSTCOND */
6313                 if (0) {
6314 rebind_pg:
6315                         libscf_handle_rebind(h);
6316                         rebound = B_TRUE;
6317 
6318                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6319                         switch (r) {
6320                         case 0:
6321                                 break;
6322 
6323                         case ECONNABORTED:
6324                                 goto rebind_pg;
6325 
6326                         case ENOENT:
6327                                 goto out;
6328 
6329                         case EINVAL:
6330                         case ENOTSUP:
6331                                 bad_error("libscf_lookup_instance", r);
6332                         }
6333 
6334                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6335                                 switch (scf_error()) {
6336                                 case SCF_ERROR_DELETED:
6337                                 case SCF_ERROR_NOT_FOUND:
6338                                         goto out;
6339 
6340                                 case SCF_ERROR_CONNECTION_BROKEN:
6341                                         goto rebind_pg;
6342 
6343                                 case SCF_ERROR_HANDLE_MISMATCH:
6344                                 case SCF_ERROR_NOT_BOUND:
6345                                 case SCF_ERROR_NOT_SET:
6346                                 case SCF_ERROR_INVALID_ARGUMENT:
6347                                 default:
6348                                         bad_error("scf_instance_get_pg",
6349                                             scf_error());
6350                                 }
6351                         }
6352                 }
6353 
6354                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6355                         r = dgraph_set_runlevel(pg, prop);
6356                         switch (r) {
6357                         case ECONNRESET:
6358                                 rebound = B_TRUE;
6359                                 rebind_inst = B_TRUE;
6360                                 /* FALLTHROUGH */
6361 
6362                         case 0:
6363                                 break;
6364 
6365                         case ECONNABORTED:
6366                                 goto rebind_pg;
6367 
6368                         case ECANCELED:
6369                                 goto out;
6370 
6371                         default:
6372                                 bad_error("dgraph_set_runlevel", r);
6373                         }
6374                 } else {
6375                         switch (scf_error()) {
6376                         case SCF_ERROR_CONNECTION_BROKEN:
6377                         default:
6378                                 goto rebind_pg;
6379 
6380                         case SCF_ERROR_DELETED:
6381                                 goto out;
6382 
6383                         case SCF_ERROR_NOT_FOUND:
6384                                 break;
6385 
6386                         case SCF_ERROR_INVALID_ARGUMENT:
6387                         case SCF_ERROR_HANDLE_MISMATCH:
6388                         case SCF_ERROR_NOT_BOUND:
6389                         case SCF_ERROR_NOT_SET:
6390                                 bad_error("scf_pg_get_property", scf_error());
6391                         }
6392                 }
6393         }
6394 
6395         if (rebind_inst) {
6396 lookup_inst:
6397                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6398                 switch (r) {
6399                 case 0:
6400                         break;
6401 
6402                 case ECONNABORTED:
6403                         libscf_handle_rebind(h);
6404                         rebound = B_TRUE;
6405                         goto lookup_inst;
6406 
6407                 case ENOENT:
6408                         goto out;
6409 
6410                 case EINVAL:
6411                 case ENOTSUP:
6412                         bad_error("libscf_lookup_instance", r);
6413                 }
6414         }
6415 
6416         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6417         switch (r) {
6418         case 0:
6419                 break;
6420 
6421         case ECONNABORTED:
6422                 libscf_handle_rebind(h);
6423                 rebound = B_TRUE;
6424                 goto lookup_inst;
6425 
6426         case EINVAL:
6427                 log_error(LOG_NOTICE,
6428                     "%s/%s property of %s is misconfigured.\n", pg_name,
6429                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6430                 /* FALLTHROUGH */
6431 
6432         case ECANCELED:
6433         case ENOENT:
6434                 (void) strcpy(fmri, "all");
6435                 break;
6436 
6437         default:
6438                 bad_error("libscf_get_milestone", r);
6439         }
6440 
6441         r = dgraph_set_milestone(fmri, h, B_FALSE);
6442         switch (r) {
6443         case 0:
6444         case ECONNRESET:
6445         case EALREADY:
6446                 break;
6447 
6448         case EINVAL:
6449                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6450                 break;
6451 
6452         case ENOENT:
6453                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6454                 break;
6455 
6456         default:
6457                 bad_error("dgraph_set_milestone", r);
6458         }
6459 
6460 out:
6461         startd_free(fmri, max_scf_fmri_size);
6462         scf_value_destroy(val);
6463         scf_property_destroy(prop);
6464 
6465         return (rebound ? ECONNRESET : 0);
6466 }
6467 
6468 /*
6469  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6470  * instance fmri or if 'fmri' matches the 'general' property group of an
6471  * instance (or the 'general/enabled' property).
6472  *
6473  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6474  */
6475 static void
6476 process_delete(char *fmri, scf_handle_t *h)
6477 {
6478         char *lfmri, *end_inst_fmri;
6479         const char *inst_name = NULL;
6480         const char *pg_name = NULL;
6481         const char *prop_name = NULL;
6482 
6483         lfmri = safe_strdup(fmri);
6484 
6485         /* Determine if the FMRI is a property group or instance */
6486         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6487             &prop_name) != SCF_SUCCESS) {
6488                 log_error(LOG_WARNING,
6489                     "Received invalid FMRI \"%s\" from repository server.\n",
6490                     fmri);
6491         } else if (inst_name != NULL && pg_name == NULL) {
6492                 (void) dgraph_remove_instance(fmri, h);
6493         } else if (inst_name != NULL && pg_name != NULL) {
6494                 /*
6495                  * If we're deleting the 'general' property group or
6496                  * 'general/enabled' property then the whole instance
6497                  * must be removed from the dgraph.
6498                  */
6499                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6500                         free(lfmri);
6501                         return;
6502                 }
6503 
6504                 if (prop_name != NULL &&
6505                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6506                         free(lfmri);
6507                         return;
6508                 }
6509 
6510                 /*
6511                  * Because the instance has already been deleted from the
6512                  * repository, we cannot use any scf_ functions to retrieve
6513                  * the instance FMRI however we can easily reconstruct it
6514                  * manually.
6515                  */
6516                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6517                 if (end_inst_fmri == NULL)
6518                         bad_error("process_delete", 0);
6519 
6520                 end_inst_fmri[0] = '\0';
6521 
6522                 (void) dgraph_remove_instance(fmri, h);
6523         }
6524 
6525         free(lfmri);
6526 }
6527 
6528 /*ARGSUSED*/
6529 void *
6530 repository_event_thread(void *unused)
6531 {
6532         scf_handle_t *h;
6533         scf_propertygroup_t *pg;
6534         scf_instance_t *inst;
6535         char *fmri = startd_alloc(max_scf_fmri_size);
6536         char *pg_name = startd_alloc(max_scf_value_size);
6537         int r;
6538 
6539         h = libscf_handle_create_bound_loop();
6540 
6541         pg = safe_scf_pg_create(h);
6542         inst = safe_scf_instance_create(h);
6543 
6544 retry:
6545         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6546                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6547                         libscf_handle_rebind(h);
6548                 } else {
6549                         log_error(LOG_WARNING,
6550                             "Couldn't set up repository notification "
6551                             "for property group type %s: %s\n",
6552                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6553 
6554                         (void) sleep(1);
6555                 }
6556 
6557                 goto retry;
6558         }
6559 
6560         /*CONSTCOND*/
6561         while (1) {
6562                 ssize_t res;
6563 
6564                 /* Note: fmri is only set on delete events. */
6565                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6566                 if (res < 0) {
6567                         libscf_handle_rebind(h);
6568                         goto retry;
6569                 } else if (res == 0) {
6570                         /*
6571                          * property group modified.  inst and pg_name are
6572                          * pre-allocated scratch space.
6573                          */
6574                         if (scf_pg_update(pg) < 0) {
6575                                 switch (scf_error()) {
6576                                 case SCF_ERROR_DELETED:
6577                                         continue;
6578 
6579                                 case SCF_ERROR_CONNECTION_BROKEN:
6580                                         log_error(LOG_WARNING,
6581                                             "Lost repository event due to "
6582                                             "disconnection.\n");
6583                                         libscf_handle_rebind(h);
6584                                         goto retry;
6585 
6586                                 case SCF_ERROR_NOT_BOUND:
6587                                 case SCF_ERROR_NOT_SET:
6588                                 default:
6589                                         bad_error("scf_pg_update", scf_error());
6590                                 }
6591                         }
6592 
6593                         r = process_pg_event(h, pg, inst, pg_name);
6594                         switch (r) {
6595                         case 0:
6596                                 break;
6597 
6598                         case ECONNABORTED:
6599                                 log_error(LOG_WARNING, "Lost repository event "
6600                                     "due to disconnection.\n");
6601                                 libscf_handle_rebind(h);
6602                                 /* FALLTHROUGH */
6603 
6604                         case ECONNRESET:
6605                                 goto retry;
6606 
6607                         default:
6608                                 bad_error("process_pg_event", r);
6609                         }
6610                 } else {
6611                         /*
6612                          * Service, instance, or pg deleted.
6613                          * Don't trust fmri on return.
6614                          */
6615                         process_delete(fmri, h);
6616                 }
6617         }
6618 
6619         /*NOTREACHED*/
6620         return (NULL);
6621 }
6622 
6623 void
6624 graph_engine_start()
6625 {
6626         int err;
6627 
6628         (void) startd_thread_create(graph_thread, NULL);
6629 
6630         MUTEX_LOCK(&dgraph_lock);
6631         while (!initial_milestone_set) {
6632                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6633                 assert(err == 0);
6634         }
6635         MUTEX_UNLOCK(&dgraph_lock);
6636 
6637         (void) startd_thread_create(repository_event_thread, NULL);
6638         (void) startd_thread_create(graph_event_thread, NULL);
6639 }