Print this page
    
    
      
        | Split | 
	Close | 
      
      | Expand all | 
      | Collapse all | 
    
    
          --- old/usr/src/cmd/dlmgmtd/dlmgmt_db.c
          +++ new/usr/src/cmd/dlmgmtd/dlmgmt_db.c
   1    1  /*
   2    2   * CDDL HEADER START
   3    3   *
   4    4   * The contents of this file are subject to the terms of the
   5    5   * Common Development and Distribution License (the "License").
   6    6   * You may not use this file except in compliance with the License.
   7    7   *
   8    8   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9    9   * or http://www.opensolaris.org/os/licensing.
  10   10   * See the License for the specific language governing permissions
  11   11   * and limitations under the License.
  12   12   *
  13   13   * When distributing Covered Code, include this CDDL HEADER in each
  14   14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15   15   * If applicable, add the following below this CDDL HEADER, with the
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  
  22   22  /*
  23   23   * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24   24   * Copyright 2015, Joyent Inc.
  25   25   */
  26   26  
  27   27  #include <assert.h>
  28   28  #include <ctype.h>
  29   29  #include <errno.h>
  30   30  #include <fcntl.h>
  31   31  #include <stdio.h>
  32   32  #include <stdlib.h>
  33   33  #include <string.h>
  34   34  #include <strings.h>
  35   35  #include <syslog.h>
  36   36  #include <zone.h>
  37   37  #include <sys/types.h>
  38   38  #include <sys/stat.h>
  39   39  #include <stropts.h>
  40   40  #include <sys/conf.h>
  41   41  #include <pthread.h>
  42   42  #include <unistd.h>
  43   43  #include <wait.h>
  44   44  #include <libcontract.h>
  45   45  #include <libcontract_priv.h>
  46   46  #include <sys/contract/process.h>
  47   47  #include <sys/vnic.h>
  48   48  #include <zone.h>
  49   49  #include "dlmgmt_impl.h"
  50   50  
  51   51  typedef enum dlmgmt_db_op {
  52   52          DLMGMT_DB_OP_WRITE,
  53   53          DLMGMT_DB_OP_DELETE,
  54   54          DLMGMT_DB_OP_READ
  55   55  } dlmgmt_db_op_t;
  56   56  
  57   57  typedef struct dlmgmt_db_req_s {
  58   58          struct dlmgmt_db_req_s  *ls_next;
  59   59          dlmgmt_db_op_t          ls_op;
  60   60          char                    ls_link[MAXLINKNAMELEN];
  61   61          datalink_id_t           ls_linkid;
  62   62          zoneid_t                ls_zoneid;
  63   63          uint32_t                ls_flags;       /* Either DLMGMT_ACTIVE or   */
  64   64                                                  /* DLMGMT_PERSIST, not both. */
  65   65  } dlmgmt_db_req_t;
  66   66  
  67   67  /*
  68   68   * List of pending db updates (e.g., because of a read-only filesystem).
  69   69   */
  70   70  static dlmgmt_db_req_t  *dlmgmt_db_req_head = NULL;
  71   71  static dlmgmt_db_req_t  *dlmgmt_db_req_tail = NULL;
  72   72  
  73   73  /*
  74   74   * rewrite_needed is set to B_TRUE by process_link_line() if it encounters a
  75   75   * line with an old format.  This will cause the file being read to be
  76   76   * re-written with the current format.
  77   77   */
  78   78  static boolean_t        rewrite_needed;
  79   79  
  80   80  static int              dlmgmt_db_update(dlmgmt_db_op_t, const char *,
  81   81                              dlmgmt_link_t *, uint32_t);
  82   82  static int              dlmgmt_process_db_req(dlmgmt_db_req_t *);
  83   83  static int              dlmgmt_process_db_onereq(dlmgmt_db_req_t *, boolean_t);
  84   84  static void             *dlmgmt_db_update_thread(void *);
  85   85  static boolean_t        process_link_line(char *, dlmgmt_link_t *);
  86   86  static int              process_db_write(dlmgmt_db_req_t *, FILE *, FILE *);
  87   87  static int              process_db_read(dlmgmt_db_req_t *, FILE *);
  88   88  static void             generate_link_line(dlmgmt_link_t *, boolean_t, char *);
  89   89  
  90   90  #define BUFLEN(lim, ptr)        (((lim) > (ptr)) ? ((lim) - (ptr)) : 0)
  91   91  #define MAXLINELEN              1024
  92   92  
  93   93  typedef void db_walk_func_t(dlmgmt_link_t *);
  94   94  
  95   95  /*
  96   96   * Translator functions to go from dladm_datatype_t to character strings.
  97   97   * Each function takes a pointer to a buffer, the size of the buffer,
  98   98   * the name of the attribute, and the value to be written.  The functions
  99   99   * return the number of bytes written to the buffer.  If the buffer is not big
 100  100   * enough to hold the string representing the value, then nothing is written
 101  101   * and 0 is returned.
 102  102   */
 103  103  typedef size_t write_func_t(char *, size_t, char *, void *);
 104  104  
 105  105  /*
 106  106   * Translator functions to read from a NULL terminated string buffer into
 107  107   * something of the given DLADM_TYPE_*.  The functions each return the number
 108  108   * of bytes read from the string buffer.  If there is an error reading data
 109  109   * from the buffer, then 0 is returned.  It is the caller's responsibility
 110  110   * to free the data allocated by these functions.
 111  111   */
 112  112  typedef size_t read_func_t(char *, void **);
 113  113  
 114  114  typedef struct translator_s {
 115  115          const char      *type_name;
 116  116          write_func_t    *write_func;
 117  117          read_func_t     *read_func;
 118  118  } translator_t;
 119  119  
 120  120  /*
 121  121   * Translator functions, defined later but declared here so that
 122  122   * the translator table can be defined.
 123  123   */
 124  124  static write_func_t     write_str, write_boolean, write_uint64;
 125  125  static read_func_t      read_str, read_boolean, read_int64;
 126  126  
 127  127  /*
 128  128   * Translator table, indexed by dladm_datatype_t.
 129  129   */
 130  130  static translator_t translators[] = {
 131  131          { "string",     write_str,      read_str        },
 132  132          { "boolean",    write_boolean,  read_boolean    },
 133  133          { "int",        write_uint64,   read_int64      }
 134  134  };
 135  135  
 136  136  static size_t ntranslators = sizeof (translators) / sizeof (translator_t);
 137  137  
 138  138  #define LINK_PROPERTY_DELIMINATOR       ";"
 139  139  #define LINK_PROPERTY_TYPE_VALUE_SEP    ","
 140  140  #define BASE_PROPERTY_LENGTH(t, n) (strlen(translators[(t)].type_name) +\
 141  141                                      strlen(LINK_PROPERTY_TYPE_VALUE_SEP) +\
 142  142                                      strlen(LINK_PROPERTY_DELIMINATOR) +\
 143  143                                      strlen((n)))
 144  144  #define GENERATE_PROPERTY_STRING(buf, length, conv, name, type, val) \
 145  145              (snprintf((buf), (length), "%s=%s%s" conv "%s", (name), \
 146  146              translators[(type)].type_name, \
 147  147              LINK_PROPERTY_TYPE_VALUE_SEP, (val), LINK_PROPERTY_DELIMINATOR))
 148  148  
 149  149  /*
 150  150   * Name of the cache file to keep the active <link name, linkid> mapping
 151  151   */
 152  152  char    cachefile[MAXPATHLEN];
 153  153  
 154  154  #define DLMGMT_PERSISTENT_DB_PATH       "/etc/dladm/datalink.conf"
 155  155  #define DLMGMT_MAKE_FILE_DB_PATH(buffer, persistent)    \
 156  156          (void) snprintf((buffer), MAXPATHLEN, "%s", \
 157  157          (persistent) ? DLMGMT_PERSISTENT_DB_PATH : cachefile);
 158  158  
 159  159  typedef struct zopen_arg {
 160  160          const char      *zopen_modestr;
 161  161          int             *zopen_pipe;
 162  162          int             zopen_fd;
 163  163  } zopen_arg_t;
 164  164  
 165  165  typedef struct zrename_arg {
 166  166          const char      *zrename_newname;
 167  167  } zrename_arg_t;
 168  168  
 169  169  typedef union zfoparg {
 170  170          zopen_arg_t     zfop_openarg;
 171  171          zrename_arg_t   zfop_renamearg;
 172  172  } zfoparg_t;
 173  173  
 174  174  typedef struct zfcbarg {
 175  175          boolean_t       zfarg_inglobalzone; /* is callback in global zone? */
 176  176          zoneid_t        zfarg_finglobalzone; /* is file in global zone? */
 177  177          const char      *zfarg_filename;
 178  178          zfoparg_t       *zfarg_oparg;
 179  179  } zfarg_t;
 180  180  #define zfarg_openarg   zfarg_oparg->zfop_openarg
 181  181  #define zfarg_renamearg zfarg_oparg->zfop_renamearg
 182  182  
 183  183  /* zone file callback */
 184  184  typedef int zfcb_t(zfarg_t *);
 185  185  
 186  186  /*
 187  187   * Execute an operation on filename relative to zoneid's zone root.  If the
 188  188   * file is in the global zone, then the zfcb() callback will simply be called
 189  189   * directly.  If the file is in a non-global zone, then zfcb() will be called
 190  190   * both from the global zone's context, and from the non-global zone's context
 191  191   * (from a fork()'ed child that has entered the non-global zone).  This is
 192  192   * done to allow the callback to communicate with itself if needed (e.g. to
 193  193   * pass back the file descriptor of an opened file).
 194  194   */
 195  195  static int
 196  196  dlmgmt_zfop(const char *filename, zoneid_t zoneid, zfcb_t *zfcb,
 197  197      zfoparg_t *zfoparg)
 198  198  {
 199  199          int             ctfd;
 200  200          int             err;
 201  201          pid_t           childpid;
 202  202          siginfo_t       info;
 203  203          zfarg_t         zfarg;
 204  204          ctid_t          ct;
 205  205  
 206  206          if (zoneid != GLOBAL_ZONEID) {
 207  207                  /*
 208  208                   * We need to access a file that isn't in the global zone.
 209  209                   * Accessing non-global zone files from the global zone is
 210  210                   * unsafe (due to symlink attacks), we'll need to fork a child
 211  211                   * that enters the zone in question and executes the callback
 212  212                   * that will operate on the file.
 213  213                   *
 214  214                   * Before we proceed with this zone tango, we need to create a
 215  215                   * new process contract for the child, as required by
 216  216                   * zone_enter().
 217  217                   */
 218  218                  errno = 0;
 219  219                  ctfd = open64("/system/contract/process/template", O_RDWR);
 220  220                  if (ctfd == -1)
 221  221                          return (errno);
 222  222                  if ((err = ct_tmpl_set_critical(ctfd, 0)) != 0 ||
 223  223                      (err = ct_tmpl_set_informative(ctfd, 0)) != 0 ||
 224  224                      (err = ct_pr_tmpl_set_fatal(ctfd, CT_PR_EV_HWERR)) != 0 ||
 225  225                      (err = ct_pr_tmpl_set_param(ctfd, CT_PR_PGRPONLY)) != 0 ||
 226  226                      (err = ct_tmpl_activate(ctfd)) != 0) {
 227  227                          (void) close(ctfd);
 228  228                          return (err);
 229  229                  }
 230  230                  childpid = fork();
 231  231                  switch (childpid) {
 232  232                  case -1:
 233  233                          (void) ct_tmpl_clear(ctfd);
 234  234                          (void) close(ctfd);
 235  235                          return (err);
 236  236                  case 0:
 237  237                          (void) ct_tmpl_clear(ctfd);
 238  238                          (void) close(ctfd);
 239  239                          /*
 240  240                           * Elevate our privileges as zone_enter() requires all
 241  241                           * privileges.
 242  242                           */
 243  243                          if ((err = dlmgmt_elevate_privileges()) != 0)
 244  244                                  _exit(err);
 245  245                          if (zone_enter(zoneid) == -1)
 246  246                                  _exit(errno);
 247  247                          if ((err = dlmgmt_drop_privileges()) != 0)
 248  248                                  _exit(err);
 249  249                          break;
 250  250                  default:
 251  251                          if (contract_latest(&ct) == -1)
 252  252                                  ct = -1;
 253  253                          (void) ct_tmpl_clear(ctfd);
 254  254                          (void) close(ctfd);
 255  255                          if (waitid(P_PID, childpid, &info, WEXITED) == -1) {
 256  256                                  (void) contract_abandon_id(ct);
 257  257                                  return (errno);
 258  258                          }
 259  259                          (void) contract_abandon_id(ct);
 260  260                          if (info.si_status != 0)
 261  261                                  return (info.si_status);
 262  262                  }
 263  263          }
 264  264  
 265  265          zfarg.zfarg_inglobalzone = (zoneid == GLOBAL_ZONEID || childpid != 0);
 266  266          zfarg.zfarg_finglobalzone = (zoneid == GLOBAL_ZONEID);
 267  267          zfarg.zfarg_filename = filename;
 268  268          zfarg.zfarg_oparg = zfoparg;
 269  269          err = zfcb(&zfarg);
 270  270          if (!zfarg.zfarg_inglobalzone)
 271  271                  _exit(err);
 272  272          return (err);
 273  273  }
 274  274  
 275  275  static int
 276  276  dlmgmt_zopen_cb(zfarg_t *zfarg)
 277  277  {
 278  278          struct strrecvfd recvfd;
 279  279          boolean_t       newfile = B_FALSE;
 280  280          boolean_t       inglobalzone = zfarg->zfarg_inglobalzone;
 281  281          zoneid_t        finglobalzone = zfarg->zfarg_finglobalzone;
 282  282          const char      *filename = zfarg->zfarg_filename;
 283  283          const char      *modestr = zfarg->zfarg_openarg.zopen_modestr;
 284  284          int             *p = zfarg->zfarg_openarg.zopen_pipe;
 285  285          struct stat     statbuf;
 286  286          int             oflags;
 287  287          mode_t          mode;
 288  288          int             fd = -1;
 289  289          int             err;
 290  290  
 291  291          /* We only ever open a file for reading or writing, not both. */
 292  292          oflags = (modestr[0] == 'r') ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC;
 293  293          mode = (modestr[0] == 'r') ? 0 : S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
 294  294  
 295  295          /* Open the file if we're in the same zone as the file. */
 296  296          if (inglobalzone == finglobalzone) {
 297  297                  /*
 298  298                   * First determine if we will be creating the file as part of
 299  299                   * opening it.  If so, then we'll need to ensure that it has
 300  300                   * the proper ownership after having opened it.
 301  301                   */
 302  302                  if (oflags & O_CREAT) {
 303  303                          if (stat(filename, &statbuf) == -1) {
 304  304                                  if (errno == ENOENT)
 305  305                                          newfile = B_TRUE;
 306  306                                  else
 307  307                                          return (errno);
 308  308                          }
 309  309                  }
 310  310                  if ((fd = open(filename, oflags, mode)) == -1)
 311  311                          return (errno);
 312  312                  if (newfile) {
 313  313                          if (chown(filename, UID_DLADM, GID_NETADM) == -1) {
 314  314                                  err = errno;
 315  315                                  (void) close(fd);
 316  316                                  return (err);
 317  317                          }
 318  318                  }
 319  319          }
 320  320  
 321  321          /*
 322  322           * If we're not in the global zone, send the file-descriptor back to
 323  323           * our parent in the global zone.
 324  324           */
 325  325          if (!inglobalzone) {
 326  326                  assert(!finglobalzone);
 327  327                  assert(fd != -1);
 328  328                  return (ioctl(p[1], I_SENDFD, fd) == -1 ? errno : 0);
 329  329          }
 330  330  
 331  331          /*
 332  332           * At this point, we know we're in the global zone.  If the file was
 333  333           * in a non-global zone, receive the file-descriptor from our child in
 334  334           * the non-global zone.
 335  335           */
 336  336          if (!finglobalzone) {
 337  337                  if (ioctl(p[0], I_RECVFD, &recvfd) == -1)
 338  338                          return (errno);
 339  339                  fd = recvfd.fd;
 340  340          }
 341  341  
 342  342          zfarg->zfarg_openarg.zopen_fd = fd;
 343  343          return (0);
 344  344  }
 345  345  
 346  346  static int
 347  347  dlmgmt_zunlink_cb(zfarg_t *zfarg)
 348  348  {
 349  349          if (zfarg->zfarg_inglobalzone != zfarg->zfarg_finglobalzone)
 350  350                  return (0);
 351  351          return (unlink(zfarg->zfarg_filename) == 0 ? 0 : errno);
 352  352  }
 353  353  
 354  354  static int
 355  355  dlmgmt_zrename_cb(zfarg_t *zfarg)
 356  356  {
 357  357          if (zfarg->zfarg_inglobalzone != zfarg->zfarg_finglobalzone)
 358  358                  return (0);
 359  359          return (rename(zfarg->zfarg_filename,
 360  360              zfarg->zfarg_renamearg.zrename_newname) == 0 ? 0 : errno);
 361  361  }
 362  362  
 363  363  /*
 364  364   * Same as fopen(3C), except that it opens the file relative to zoneid's zone
 365  365   * root.
 366  366   */
 367  367  static FILE *
 368  368  dlmgmt_zfopen(const char *filename, const char *modestr, zoneid_t zoneid,
 369  369      int *err)
 370  370  {
 371  371          int             p[2];
 372  372          zfoparg_t       zfoparg;
 373  373          FILE            *fp = NULL;
 374  374  
 375  375          if (zoneid != GLOBAL_ZONEID && pipe(p) == -1) {
 376  376                  *err = errno;
 377  377                  return (NULL);
 378  378          }
 379  379  
 380  380          zfoparg.zfop_openarg.zopen_modestr = modestr;
 381  381          zfoparg.zfop_openarg.zopen_pipe = p;
 382  382          *err = dlmgmt_zfop(filename, zoneid, dlmgmt_zopen_cb, &zfoparg);
 383  383          if (zoneid != GLOBAL_ZONEID) {
 384  384                  (void) close(p[0]);
 385  385                  (void) close(p[1]);
 386  386          }
 387  387          if (*err == 0) {
 388  388                  fp = fdopen(zfoparg.zfop_openarg.zopen_fd, modestr);
 389  389                  if (fp == NULL) {
 390  390                          *err = errno;
 391  391                          (void) close(zfoparg.zfop_openarg.zopen_fd);
 392  392                  }
 393  393          }
 394  394          return (fp);
 395  395  }
 396  396  
 397  397  /*
 398  398   * Same as rename(2), except that old and new are relative to zoneid's zone
 399  399   * root.
 400  400   */
 401  401  static int
 402  402  dlmgmt_zrename(const char *old, const char *new, zoneid_t zoneid)
 403  403  {
 404  404          zfoparg_t zfoparg;
 405  405  
 406  406          zfoparg.zfop_renamearg.zrename_newname = new;
 407  407          return (dlmgmt_zfop(old, zoneid, dlmgmt_zrename_cb, &zfoparg));
 408  408  }
 409  409  
 410  410  /*
 411  411   * Same as unlink(2), except that filename is relative to zoneid's zone root.
 412  412   */
 413  413  static int
 414  414  dlmgmt_zunlink(const char *filename, zoneid_t zoneid)
 415  415  {
 416  416          return (dlmgmt_zfop(filename, zoneid, dlmgmt_zunlink_cb, NULL));
 417  417  }
 418  418  
 419  419  static size_t
 420  420  write_str(char *buffer, size_t buffer_length, char *name, void *value)
 421  421  {
 422  422          char    *ptr = value;
 423  423          size_t  data_length = strnlen(ptr, buffer_length);
 424  424  
 425  425          /*
 426  426           * Strings are assumed to be NULL terminated.  In order to fit in
 427  427           * the buffer, the string's length must be less then buffer_length.
 428  428           * If the value is empty, there's no point in writing it, in fact,
 429  429           * we shouldn't even see that case.
 430  430           */
 431  431          if (data_length + BASE_PROPERTY_LENGTH(DLADM_TYPE_STR, name) ==
 432  432              buffer_length || data_length == 0)
 433  433                  return (0);
 434  434  
 435  435          /*
 436  436           * Since we know the string will fit in the buffer, snprintf will
 437  437           * always return less than buffer_length, so we can just return
 438  438           * whatever snprintf returns.
 439  439           */
 440  440          return (GENERATE_PROPERTY_STRING(buffer, buffer_length, "%s",
 441  441              name, DLADM_TYPE_STR, ptr));
 442  442  }
 443  443  
 444  444  static size_t
 445  445  write_boolean(char *buffer, size_t buffer_length, char *name, void *value)
 446  446  {
 447  447          boolean_t       *ptr = value;
 448  448  
 449  449          /*
 450  450           * Booleans are either zero or one, so we only need room for two
 451  451           * characters in the buffer.
 452  452           */
 453  453          if (buffer_length <= 1 + BASE_PROPERTY_LENGTH(DLADM_TYPE_BOOLEAN, name))
 454  454                  return (0);
 455  455  
 456  456          return (GENERATE_PROPERTY_STRING(buffer, buffer_length, "%d",
 457  457              name, DLADM_TYPE_BOOLEAN, *ptr));
 458  458  }
 459  459  
 460  460  static size_t
 461  461  write_uint64(char *buffer, size_t buffer_length, char *name, void *value)
 462  462  {
 463  463          uint64_t        *ptr = value;
 464  464  
 465  465          /*
 466  466           * Limit checking for uint64_t is a little trickier.
 467  467           */
 468  468          if (snprintf(NULL, 0, "%lld", *ptr)  +
 469  469              BASE_PROPERTY_LENGTH(DLADM_TYPE_UINT64, name) >= buffer_length)
 470  470                  return (0);
 471  471  
 472  472          return (GENERATE_PROPERTY_STRING(buffer, buffer_length, "%lld",
 473  473              name, DLADM_TYPE_UINT64, *ptr));
 474  474  }
 475  475  
 476  476  static size_t
 477  477  read_str(char *buffer, void **value)
 478  478  {
 479  479          char            *ptr = calloc(MAXLINKATTRVALLEN, sizeof (char));
 480  480          ssize_t         len;
 481  481  
 482  482          if (ptr == NULL || (len = strlcpy(ptr, buffer, MAXLINKATTRVALLEN))
 483  483              >= MAXLINKATTRVALLEN) {
 484  484                  free(ptr);
 485  485                  return (0);
 486  486          }
 487  487  
 488  488          *(char **)value = ptr;
 489  489  
 490  490          /* Account for NULL terminator */
 491  491          return (len + 1);
 492  492  }
 493  493  
 494  494  static size_t
 495  495  read_boolean(char *buffer, void **value)
 496  496  {
 497  497          boolean_t       *ptr = calloc(1, sizeof (boolean_t));
 498  498  
 499  499          if (ptr == NULL)
 500  500                  return (0);
 501  501  
 502  502          *ptr = atoi(buffer);
 503  503          *(boolean_t **)value = ptr;
 504  504  
 505  505          return (sizeof (boolean_t));
 506  506  }
 507  507  
 508  508  static size_t
 509  509  read_int64(char *buffer, void **value)
 510  510  {
 511  511          int64_t *ptr = calloc(1, sizeof (int64_t));
 512  512  
 513  513          if (ptr == NULL)
 514  514                  return (0);
 515  515  
 516  516          *ptr = (int64_t)atoll(buffer);
 517  517          *(int64_t **)value = ptr;
 518  518  
 519  519          return (sizeof (int64_t));
 520  520  }
 521  521  
 522  522  static dlmgmt_db_req_t *
 523  523  dlmgmt_db_req_alloc(dlmgmt_db_op_t op, const char *linkname,
 524  524      datalink_id_t linkid, zoneid_t zoneid, uint32_t flags, int *err)
 525  525  {
 526  526          dlmgmt_db_req_t *req;
 527  527  
 528  528          if ((req = calloc(1, sizeof (dlmgmt_db_req_t))) == NULL) {
 529  529                  *err = errno;
 530  530          } else {
 531  531                  req->ls_op = op;
 532  532                  if (linkname != NULL)
 533  533                          (void) strlcpy(req->ls_link, linkname, MAXLINKNAMELEN);
 534  534                  req->ls_linkid = linkid;
 535  535                  req->ls_zoneid = zoneid;
 536  536                  req->ls_flags = flags;
 537  537          }
 538  538          return (req);
 539  539  }
 540  540  
 541  541  /*
 542  542   * Update the db entry with name "entryname" using information from "linkp".
 543  543   */
 544  544  static int
 545  545  dlmgmt_db_update(dlmgmt_db_op_t op, const char *entryname, dlmgmt_link_t *linkp,
 546  546      uint32_t flags)
 547  547  {
  
    | 
      ↓ open down ↓ | 
    547 lines elided | 
    
      ↑ open up ↑ | 
  
 548  548          dlmgmt_db_req_t *req;
 549  549          int             err;
 550  550  
 551  551          /* It is either a persistent request or an active request, not both. */
 552  552          assert((flags == DLMGMT_PERSIST) || (flags == DLMGMT_ACTIVE));
 553  553  
 554  554          if ((req = dlmgmt_db_req_alloc(op, entryname, linkp->ll_linkid,
 555  555              linkp->ll_zoneid, flags, &err)) == NULL)
 556  556                  return (err);
 557  557  
 558      -        /* If transient op and onloan, use the global zone cache file. */
 559      -        if (flags == DLMGMT_ACTIVE && linkp->ll_onloan)
 560      -                req->ls_zoneid = GLOBAL_ZONEID;
 561      -
 562  558          /*
 563  559           * If the return error is EINPROGRESS, this request is handled
 564  560           * asynchronously; return success.
 565  561           */
 566  562          err = dlmgmt_process_db_req(req);
 567  563          if (err != EINPROGRESS)
 568  564                  free(req);
 569  565          else
 570  566                  err = 0;
 571  567          return (err);
 572  568  }
 573  569  
 574  570  #define DLMGMT_DB_OP_STR(op)                                    \
 575  571          (((op) == DLMGMT_DB_OP_READ) ? "read" :                 \
 576  572          (((op) == DLMGMT_DB_OP_WRITE) ? "write" : "delete"))
 577  573  
 578  574  #define DLMGMT_DB_CONF_STR(flag)                                \
 579  575          (((flag) == DLMGMT_ACTIVE) ? "active" :                 \
 580  576          (((flag) == DLMGMT_PERSIST) ? "persistent" : ""))
 581  577  
 582  578  static int
 583  579  dlmgmt_process_db_req(dlmgmt_db_req_t *req)
 584  580  {
 585  581          pthread_t       tid;
 586  582          boolean_t       writeop;
 587  583          int             err;
 588  584  
 589  585          /*
 590  586           * If there are already pending "write" requests, queue this request in
 591  587           * the pending list.  Note that this function is called while the
 592  588           * dlmgmt_rw_lock is held, so it is safe to access the global variables.
 593  589           */
 594  590          writeop = (req->ls_op != DLMGMT_DB_OP_READ);
 595  591          if (writeop && (req->ls_flags == DLMGMT_PERSIST) &&
 596  592              (dlmgmt_db_req_head != NULL)) {
 597  593                  dlmgmt_db_req_tail->ls_next = req;
 598  594                  dlmgmt_db_req_tail = req;
 599  595                  return (EINPROGRESS);
 600  596          }
 601  597  
 602  598          err = dlmgmt_process_db_onereq(req, writeop);
 603  599          if (err != EINPROGRESS && err != 0 && err != ENOENT) {
 604  600                  /*
 605  601                   * Log the error unless the request processing is still in
 606  602                   * progress or if the configuration file hasn't been created
 607  603                   * yet (ENOENT).
 608  604                   */
 609  605                  dlmgmt_log(LOG_WARNING, "dlmgmt_process_db_onereq() %s "
 610  606                      "operation on %s configuration failed: %s",
 611  607                      DLMGMT_DB_OP_STR(req->ls_op),
 612  608                      DLMGMT_DB_CONF_STR(req->ls_flags), strerror(err));
 613  609          }
 614  610  
 615  611          if (err == EINPROGRESS) {
 616  612                  assert(req->ls_flags == DLMGMT_PERSIST);
 617  613                  assert(writeop && dlmgmt_db_req_head == NULL);
 618  614                  dlmgmt_db_req_tail = dlmgmt_db_req_head = req;
 619  615                  err = pthread_create(&tid, NULL, dlmgmt_db_update_thread, NULL);
 620  616                  if (err == 0)
 621  617                          return (EINPROGRESS);
 622  618          }
 623  619          return (err);
 624  620  }
 625  621  
 626  622  static int
 627  623  dlmgmt_process_db_onereq(dlmgmt_db_req_t *req, boolean_t writeop)
 628  624  {
 629  625          int     err = 0;
 630  626          FILE    *fp, *nfp = NULL;
 631  627          char    file[MAXPATHLEN];
 632  628          char    newfile[MAXPATHLEN];
 633  629  
 634  630          DLMGMT_MAKE_FILE_DB_PATH(file, (req->ls_flags == DLMGMT_PERSIST));
 635  631          fp = dlmgmt_zfopen(file, "r", req->ls_zoneid, &err);
 636  632          /*
 637  633           * Note that it is not an error if the file doesn't exist.  If we're
 638  634           * reading, we treat this case the same way as an empty file.  If
 639  635           * we're writing, the file will be created when we open the file for
 640  636           * writing below.
 641  637           */
 642  638          if (fp == NULL && !writeop)
 643  639                  return (err);
 644  640  
 645  641          if (writeop) {
 646  642                  (void) snprintf(newfile, MAXPATHLEN, "%s.new", file);
 647  643                  nfp = dlmgmt_zfopen(newfile, "w", req->ls_zoneid, &err);
 648  644                  if (nfp == NULL) {
 649  645                          /*
 650  646                           * EROFS can happen at boot when the file system is
 651  647                           * read-only.  Return EINPROGRESS so that the caller
 652  648                           * can add this request to the pending request list
 653  649                           * and start a retry thread.
 654  650                           */
 655  651                          err = (errno == EROFS ? EINPROGRESS : errno);
 656  652                          goto done;
 657  653                  }
 658  654          }
 659  655          if (writeop) {
 660  656                  if ((err = process_db_write(req, fp, nfp)) == 0)
 661  657                          err = dlmgmt_zrename(newfile, file, req->ls_zoneid);
 662  658          } else {
 663  659                  err = process_db_read(req, fp);
 664  660          }
 665  661  
 666  662  done:
 667  663          if (nfp != NULL) {
 668  664                  (void) fclose(nfp);
 669  665                  if (err != 0)
 670  666                          (void) dlmgmt_zunlink(newfile, req->ls_zoneid);
 671  667          }
 672  668          (void) fclose(fp);
 673  669          return (err);
 674  670  }
 675  671  
 676  672  /*ARGSUSED*/
 677  673  static void *
 678  674  dlmgmt_db_update_thread(void *arg)
 679  675  {
 680  676          dlmgmt_db_req_t *req;
 681  677  
 682  678          dlmgmt_table_lock(B_TRUE);
 683  679  
 684  680          assert(dlmgmt_db_req_head != NULL);
 685  681          while ((req = dlmgmt_db_req_head) != NULL) {
 686  682                  assert(req->ls_flags == DLMGMT_PERSIST);
 687  683                  if (dlmgmt_process_db_onereq(req, B_TRUE) == EINPROGRESS) {
 688  684                          /*
 689  685                           * The filesystem is still read only. Go to sleep and
 690  686                           * try again.
 691  687                           */
 692  688                          dlmgmt_table_unlock();
 693  689                          (void) sleep(5);
 694  690                          dlmgmt_table_lock(B_TRUE);
 695  691                          continue;
 696  692                  }
 697  693  
 698  694                  /*
 699  695                   * The filesystem is no longer read only. Continue processing
 700  696                   * and remove the request from the pending list.
 701  697                   */
 702  698                  dlmgmt_db_req_head = req->ls_next;
 703  699                  if (dlmgmt_db_req_tail == req) {
 704  700                          assert(dlmgmt_db_req_head == NULL);
 705  701                          dlmgmt_db_req_tail = NULL;
 706  702                  }
 707  703                  free(req);
 708  704          }
 709  705  
 710  706          dlmgmt_table_unlock();
 711  707          return (NULL);
 712  708  }
 713  709  
 714  710  static int
 715  711  parse_linkprops(char *buf, dlmgmt_link_t *linkp)
 716  712  {
 717  713          boolean_t               found_type = B_FALSE;
 718  714          dladm_datatype_t        type = DLADM_TYPE_STR;
 719  715          int                     i, len;
  
    | 
      ↓ open down ↓ | 
    148 lines elided | 
    
      ↑ open up ↑ | 
  
 720  716          char                    *curr;
 721  717          char                    attr_name[MAXLINKATTRLEN];
 722  718          size_t                  attr_buf_len = 0;
 723  719          void                    *attr_buf = NULL;
 724  720          boolean_t               rename;
 725  721  
 726  722          curr = buf;
 727  723          len = strlen(buf);
 728  724          attr_name[0] = '\0';
 729  725          for (i = 0; i < len; i++) {
 730      -                rename = B_FALSE;
 731  726                  char            c = buf[i];
 732  727                  boolean_t       match = (c == '=' ||
 733  728                      (c == ',' && !found_type) || c == ';');
 734  729  
      730 +                rename = B_FALSE;
 735  731                  /*
 736  732                   * Move to the next character if there is no match and
 737  733                   * if we have not reached the last character.
 738  734                   */
 739  735                  if (!match && i != len - 1)
 740  736                          continue;
 741  737  
 742  738                  if (match) {
 743  739                          /*
 744  740                           * NUL-terminate the string pointed to by 'curr'.
 745  741                           */
 746  742                          buf[i] = '\0';
 747  743                          if (*curr == '\0')
 748  744                                  goto parse_fail;
 749  745                  }
 750  746  
 751  747                  if (attr_name[0] != '\0' && found_type) {
 752  748                          /*
 753  749                           * We get here after we have processed the "<prop>="
 754  750                           * pattern. The pattern we are now interested in is
 755  751                           * "<val>;".
 756  752                           */
 757  753                          if (c == '=')
 758  754                                  goto parse_fail;
 759  755  
 760  756                          if (strcmp(attr_name, "linkid") == 0) {
 761  757                                  if (read_int64(curr, &attr_buf) == 0)
 762  758                                          goto parse_fail;
 763  759                                  linkp->ll_linkid =
 764  760                                      (datalink_class_t)*(int64_t *)attr_buf;
 765  761                          } else if (strcmp(attr_name, "name") == 0) {
 766  762                                  if (read_str(curr, &attr_buf) == 0)
 767  763                                          goto parse_fail;
 768  764                                  (void) snprintf(linkp->ll_link,
 769  765                                      MAXLINKNAMELEN, "%s", attr_buf);
 770  766                          } else if (strcmp(attr_name, "class") == 0) {
 771  767                                  if (read_int64(curr, &attr_buf) == 0)
 772  768                                          goto parse_fail;
 773  769                                  linkp->ll_class =
 774  770                                      (datalink_class_t)*(int64_t *)attr_buf;
 775  771                          } else if (strcmp(attr_name, "media") == 0) {
 776  772                                  if (read_int64(curr, &attr_buf) == 0)
 777  773                                          goto parse_fail;
 778  774                                  linkp->ll_media =
 779  775                                      (uint32_t)*(int64_t *)attr_buf;
 780  776                          } else if (strcmp(attr_name, "zone") == 0) {
 781  777                                  if (read_str(curr, &attr_buf) == 0)
 782  778                                          goto parse_fail;
 783  779                                  linkp->ll_zoneid = getzoneidbyname(attr_buf);
 784  780                                  if (linkp->ll_zoneid == -1) {
 785  781                                          if (errno == EFAULT)
 786  782                                                  abort();
 787  783                                          /*
 788  784                                           * If we can't find the zone, assign the
 789  785                                           * link to the GZ and mark it for being
 790  786                                           * renamed.
 791  787                                           */
 792  788                                          linkp->ll_zoneid = 0;
 793  789                                          rename = B_TRUE;
 794  790                                  }
 795  791                          } else {
 796  792                                  attr_buf_len = translators[type].read_func(curr,
 797  793                                      &attr_buf);
 798  794                                  if (attr_buf_len == 0)
 799  795                                          goto parse_fail;
 800  796  
 801  797                                  if (linkattr_set(&(linkp->ll_head), attr_name,
 802  798                                      attr_buf, attr_buf_len, type) != 0) {
 803  799                                          free(attr_buf);
 804  800                                          goto parse_fail;
 805  801                                  }
 806  802                          }
 807  803  
 808  804                          free(attr_buf);
 809  805                          attr_name[0] = '\0';
 810  806                          found_type = B_FALSE;
 811  807                  } else if (attr_name[0] != '\0') {
 812  808                          /*
 813  809                           * Non-zero length attr_name and found_type of false
 814  810                           * indicates that we have not found the type for this
 815  811                           * attribute.  The pattern now is "<type>,<val>;", we
 816  812                           * want the <type> part of the pattern.
 817  813                           */
 818  814                          for (type = 0; type < ntranslators; type++) {
 819  815                                  if (strcmp(curr,
 820  816                                      translators[type].type_name) == 0) {
 821  817                                          found_type = B_TRUE;
 822  818                                          break;
 823  819                                  }
 824  820                          }
 825  821  
 826  822                          if (!found_type)
 827  823                                  goto parse_fail;
 828  824                  } else {
 829  825                          /*
 830  826                           * A zero length attr_name indicates we are looking
 831  827                           * at the beginning of a link attribute.
 832  828                           */
 833  829                          if (c != '=')
 834  830                                  goto parse_fail;
 835  831  
 836  832                          (void) snprintf(attr_name, MAXLINKATTRLEN, "%s", curr);
 837  833                  }
 838  834  
 839  835                  /*
 840  836                   * The zone that this link belongs to has died, we are
 841  837                   * reparenting it to the GZ and renaming it to avoid name
 842  838                   * collisions.
 843  839                   */
 844  840                  if (rename == B_TRUE) {
 845  841                          (void) snprintf(linkp->ll_link, MAXLINKNAMELEN,
 846  842                              "SUNWorphan%u", (uint16_t)(gethrtime() / 1000));
 847  843                  }
 848  844                  curr = buf + i + 1;
 849  845          }
 850  846  
 851  847          /* Correct any erroneous IPTUN datalink class constant in the file */
 852  848          if (linkp->ll_class == 0x60) {
 853  849                  linkp->ll_class = DATALINK_CLASS_IPTUN;
 854  850                  rewrite_needed = B_TRUE;
 855  851          }
 856  852  
 857  853          return (0);
 858  854  
 859  855  parse_fail:
 860  856          /*
 861  857           * Free linkp->ll_head (link attribute list)
 862  858           */
 863  859          linkattr_destroy(linkp);
 864  860          return (-1);
 865  861  }
 866  862  
 867  863  static boolean_t
 868  864  process_link_line(char *buf, dlmgmt_link_t *linkp)
 869  865  {
 870  866          int     i, len, llen;
 871  867          char    *str, *lasts;
 872  868          char    tmpbuf[MAXLINELEN];
 873  869  
 874  870          bzero(linkp, sizeof (*linkp));
 875  871          linkp->ll_linkid = DATALINK_INVALID_LINKID;
 876  872  
 877  873          /*
 878  874           * Use a copy of buf for parsing so that we can do whatever we want.
 879  875           */
 880  876          (void) strlcpy(tmpbuf, buf, MAXLINELEN);
 881  877  
 882  878          /*
 883  879           * Skip leading spaces, blank lines, and comments.
 884  880           */
 885  881          len = strlen(tmpbuf);
 886  882          for (i = 0; i < len; i++) {
 887  883                  if (!isspace(tmpbuf[i]))
 888  884                          break;
 889  885          }
 890  886          if (i == len || tmpbuf[i] == '#')
 891  887                  return (B_TRUE);
 892  888  
 893  889          str = tmpbuf + i;
 894  890          /*
 895  891           * Find the link name and assign it to the link structure.
 896  892           */
 897  893          if (strtok_r(str, " \n\t", &lasts) == NULL)
 898  894                  goto fail;
 899  895  
 900  896          llen = strlen(str);
 901  897          /*
 902  898           * Note that a previous version of the persistent datalink.conf file
 903  899           * stored the linkid as the first field.  In that case, the name will
 904  900           * be obtained through parse_linkprops from a property with the format
 905  901           * "name=<linkname>".  If we encounter such a format, we set
 906  902           * rewrite_needed so that dlmgmt_db_init() can rewrite the file with
 907  903           * the new format after it's done reading in the data.
 908  904           */
 909  905          if (isdigit(str[0])) {
 910  906                  linkp->ll_linkid = atoi(str);
 911  907                  rewrite_needed = B_TRUE;
 912  908          } else {
 913  909                  if (strlcpy(linkp->ll_link, str, sizeof (linkp->ll_link)) >=
 914  910                      sizeof (linkp->ll_link))
 915  911                          goto fail;
 916  912          }
 917  913  
 918  914          str += llen + 1;
 919  915          if (str >= tmpbuf + len)
 920  916                  goto fail;
 921  917  
 922  918          /*
 923  919           * Now find the list of link properties.
 924  920           */
 925  921          if ((str = strtok_r(str, " \n\t", &lasts)) == NULL)
 926  922                  goto fail;
 927  923  
 928  924          if (parse_linkprops(str, linkp) < 0)
 929  925                  goto fail;
 930  926  
 931  927          return (B_TRUE);
 932  928  
 933  929  fail:
 934  930          /*
 935  931           * Delete corrupted line.
 936  932           */
 937  933          buf[0] = '\0';
 938  934          return (B_FALSE);
 939  935  }
 940  936  
 941  937  /*
 942  938   * Find any properties in linkp that refer to "old", and rename to "new".
 943  939   * Return B_TRUE if any renaming occurred.
 944  940   */
 945  941  static int
 946  942  dlmgmt_attr_rename(dlmgmt_link_t *linkp, const char *old, const char *new,
 947  943      boolean_t *renamed)
 948  944  {
 949  945          dlmgmt_linkattr_t       *attrp;
 950  946          char                    *newval = NULL, *pname;
 951  947          char                    valcp[MAXLINKATTRVALLEN];
 952  948          size_t                  newsize;
 953  949  
 954  950          *renamed = B_FALSE;
 955  951  
 956  952          if ((attrp = linkattr_find(linkp->ll_head, "linkover")) != NULL ||
 957  953              (attrp = linkattr_find(linkp->ll_head, "simnetpeer")) != NULL) {
 958  954                  if (strcmp(old, (char *)attrp->lp_val) == 0) {
 959  955                          newsize = strlen(new) + 1;
 960  956                          if ((newval = malloc(newsize)) == NULL)
 961  957                                  return (errno);
 962  958                          (void) strcpy(newval, new);
 963  959                          free(attrp->lp_val);
 964  960                          attrp->lp_val = newval;
 965  961                          attrp->lp_sz = newsize;
 966  962                          *renamed = B_TRUE;
 967  963                  }
 968  964                  return (0);
 969  965          }
 970  966  
 971  967          if ((attrp = linkattr_find(linkp->ll_head, "portnames")) == NULL)
 972  968                  return (0);
 973  969  
 974  970          /* <linkname>:[<linkname>:]... */
 975  971          if ((newval = calloc(MAXLINKATTRVALLEN, sizeof (char))) == NULL)
 976  972                  return (errno);
 977  973  
 978  974          bcopy(attrp->lp_val, valcp, sizeof (valcp));
 979  975          pname = strtok(valcp, ":");
 980  976          while (pname != NULL) {
 981  977                  if (strcmp(pname, old) == 0) {
 982  978                          (void) strcat(newval, new);
 983  979                          *renamed = B_TRUE;
 984  980                  } else {
 985  981                          (void) strcat(newval, pname);
 986  982                  }
 987  983                  (void) strcat(newval, ":");
 988  984                  pname = strtok(NULL, ":");
 989  985          }
 990  986          if (*renamed) {
 991  987                  free(attrp->lp_val);
 992  988                  attrp->lp_val = newval;
 993  989                  attrp->lp_sz = strlen(newval) + 1;
 994  990          } else {
 995  991                  free(newval);
 996  992          }
 997  993          return (0);
 998  994  }
 999  995  
1000  996  static int
1001  997  process_db_write(dlmgmt_db_req_t *req, FILE *fp, FILE *nfp)
1002  998  {
1003  999          boolean_t               done = B_FALSE;
1004 1000          int                     err = 0;
1005 1001          dlmgmt_link_t           link_in_file, *linkp = NULL, *dblinkp;
1006 1002          boolean_t               persist = (req->ls_flags == DLMGMT_PERSIST);
1007 1003          boolean_t               writeall, rename, attr_renamed;
1008 1004          char                    buf[MAXLINELEN];
1009 1005  
1010 1006          writeall = (req->ls_linkid == DATALINK_ALL_LINKID);
1011 1007  
1012 1008          if (req->ls_op == DLMGMT_DB_OP_WRITE && !writeall) {
1013 1009                  /*
1014 1010                   * find the link in the avl tree with the given linkid.
1015 1011                   */
1016 1012                  linkp = link_by_id(req->ls_linkid, req->ls_zoneid);
1017 1013                  if (linkp == NULL || (linkp->ll_flags & req->ls_flags) == 0) {
1018 1014                          /*
1019 1015                           * This link has already been changed. This could
1020 1016                           * happen if the request is pending because of
1021 1017                           * read-only file-system. If so, we are done.
1022 1018                           */
1023 1019                          return (0);
1024 1020                  }
1025 1021                  /*
1026 1022                   * In the case of a rename, linkp's name has been updated to
1027 1023                   * the new name, and req->ls_link is the old link name.
1028 1024                   */
1029 1025                  rename = (strcmp(req->ls_link, linkp->ll_link) != 0);
1030 1026          }
1031 1027  
1032 1028          /*
1033 1029           * fp can be NULL if the file didn't initially exist and we're
1034 1030           * creating it as part of this write operation.
1035 1031           */
1036 1032          if (fp == NULL)
1037 1033                  goto write;
1038 1034  
1039 1035          while (err == 0 && fgets(buf, sizeof (buf), fp) != NULL &&
1040 1036              process_link_line(buf, &link_in_file)) {
1041 1037                  /*
1042 1038                   * Only the link name is needed. Free the memory allocated for
1043 1039                   * the link attributes list of link_in_file.
1044 1040                   */
1045 1041                  linkattr_destroy(&link_in_file);
1046 1042  
1047 1043                  if (link_in_file.ll_link[0] == '\0' || done) {
1048 1044                          /*
1049 1045                           * this is a comment line or we are done updating the
1050 1046                           * line for the specified link, write the rest of
1051 1047                           * lines out.
1052 1048                           */
1053 1049                          if (fputs(buf, nfp) == EOF)
1054 1050                                  err = errno;
1055 1051                          continue;
1056 1052                  }
1057 1053  
1058 1054                  switch (req->ls_op) {
1059 1055                  case DLMGMT_DB_OP_WRITE:
1060 1056                          /*
1061 1057                           * For write operations, we generate a new output line
1062 1058                           * if we're either writing all links (writeall) or if
1063 1059                           * the name of the link in the file matches the one
1064 1060                           * we're looking for.  Otherwise, we write out the
1065 1061                           * buffer as-is.
1066 1062                           *
1067 1063                           * If we're doing a rename operation, ensure that any
1068 1064                           * references to the link being renamed in link
1069 1065                           * properties are also updated before we write
1070 1066                           * anything.
1071 1067                           */
1072 1068                          if (writeall) {
1073 1069                                  linkp = link_by_name(link_in_file.ll_link,
1074 1070                                      req->ls_zoneid);
1075 1071                          }
1076 1072                          if (writeall || strcmp(req->ls_link,
1077 1073                              link_in_file.ll_link) == 0) {
1078 1074                                  generate_link_line(linkp, persist, buf);
1079 1075                                  if (!writeall && !rename)
1080 1076                                          done = B_TRUE;
1081 1077                          } else if (rename && persist) {
1082 1078                                  dblinkp = link_by_name(link_in_file.ll_link,
1083 1079                                      req->ls_zoneid);
1084 1080                                  err = dlmgmt_attr_rename(dblinkp, req->ls_link,
1085 1081                                      linkp->ll_link, &attr_renamed);
1086 1082                                  if (err != 0)
1087 1083                                          break;
1088 1084                                  if (attr_renamed) {
1089 1085                                          generate_link_line(dblinkp, persist,
1090 1086                                              buf);
1091 1087                                  }
1092 1088                          }
1093 1089                          if (fputs(buf, nfp) == EOF)
1094 1090                                  err = errno;
1095 1091                          break;
1096 1092                  case DLMGMT_DB_OP_DELETE:
1097 1093                          /*
1098 1094                           * Delete is simple.  If buf does not represent the
1099 1095                           * link we're deleting, write it out.
1100 1096                           */
1101 1097                          if (strcmp(req->ls_link, link_in_file.ll_link) != 0) {
1102 1098                                  if (fputs(buf, nfp) == EOF)
1103 1099                                          err = errno;
1104 1100                          } else {
1105 1101                                  done = B_TRUE;
1106 1102                          }
1107 1103                          break;
1108 1104                  case DLMGMT_DB_OP_READ:
1109 1105                  default:
1110 1106                          err = EINVAL;
1111 1107                          break;
1112 1108                  }
1113 1109          }
1114 1110  
1115 1111  write:
1116 1112          /*
1117 1113           * If we get to the end of the file and have not seen what linkid
1118 1114           * points to, write it out then.
1119 1115           */
1120 1116          if (req->ls_op == DLMGMT_DB_OP_WRITE && !writeall && !rename && !done) {
1121 1117                  generate_link_line(linkp, persist, buf);
1122 1118                  done = B_TRUE;
1123 1119                  if (fputs(buf, nfp) == EOF)
1124 1120                          err = errno;
1125 1121          }
1126 1122  
1127 1123          return (err);
1128 1124  }
1129 1125  
1130 1126  static int
1131 1127  process_db_read(dlmgmt_db_req_t *req, FILE *fp)
1132 1128  {
1133 1129          avl_index_t     name_where, id_where;
1134 1130          dlmgmt_link_t   link_in_file, *newlink, *link_in_db;
1135 1131          char            buf[MAXLINELEN];
1136 1132          int             err = 0;
1137 1133  
1138 1134          /*
1139 1135           * This loop processes each line of the configuration file.
1140 1136           */
1141 1137          while (fgets(buf, MAXLINELEN, fp) != NULL) {
1142 1138                  if (!process_link_line(buf, &link_in_file)) {
1143 1139                          err = EINVAL;
1144 1140                          break;
1145 1141                  }
1146 1142  
1147 1143                  /*
1148 1144                   * Skip the comment line.
1149 1145                   */
1150 1146                  if (link_in_file.ll_link[0] == '\0') {
1151 1147                          linkattr_destroy(&link_in_file);
1152 1148                          continue;
1153 1149                  }
1154 1150  
1155 1151                  if ((req->ls_flags & DLMGMT_ACTIVE) &&
1156 1152                      link_in_file.ll_linkid == DATALINK_INVALID_LINKID) {
1157 1153                          linkattr_destroy(&link_in_file);
1158 1154                          continue;
1159 1155                  }
1160 1156  
1161 1157                  link_in_file.ll_zoneid = req->ls_zoneid;
1162 1158                  link_in_db = link_by_name(link_in_file.ll_link,
1163 1159                      link_in_file.ll_zoneid);
1164 1160                  if (link_in_db != NULL) {
1165 1161                          /*
1166 1162                           * If the link in the database already has the flag
1167 1163                           * for this request set, then the entry is a
1168 1164                           * duplicate.  If it's not a duplicate, then simply
1169 1165                           * turn on the appropriate flag on the existing link.
1170 1166                           */
1171 1167                          if (link_in_db->ll_flags & req->ls_flags) {
1172 1168                                  dlmgmt_log(LOG_WARNING, "Duplicate links "
1173 1169                                      "in the repository: %s",
1174 1170                                      link_in_file.ll_link);
1175 1171                                  linkattr_destroy(&link_in_file);
1176 1172                          } else {
1177 1173                                  if (req->ls_flags & DLMGMT_PERSIST) {
1178 1174                                          /*
1179 1175                                           * Save the newly read properties into
1180 1176                                           * the existing link.
1181 1177                                           */
1182 1178                                          assert(link_in_db->ll_head == NULL);
1183 1179                                          link_in_db->ll_head =
1184 1180                                              link_in_file.ll_head;
1185 1181                                  } else {
1186 1182                                          linkattr_destroy(&link_in_file);
1187 1183                                  }
1188 1184                                  link_in_db->ll_flags |= req->ls_flags;
1189 1185                          }
1190 1186                  } else {
1191 1187                          /*
1192 1188                           * This is a new link.  Allocate a new dlmgmt_link_t
1193 1189                           * and add it to the trees.
1194 1190                           */
1195 1191                          newlink = calloc(1, sizeof (*newlink));
1196 1192                          if (newlink == NULL) {
1197 1193                                  dlmgmt_log(LOG_WARNING, "Unable to allocate "
1198 1194                                      "memory to create new link %s",
1199 1195                                      link_in_file.ll_link);
1200 1196                                  linkattr_destroy(&link_in_file);
1201 1197                                  continue;
1202 1198                          }
1203 1199                          bcopy(&link_in_file, newlink, sizeof (*newlink));
1204 1200  
1205 1201                          if (newlink->ll_linkid == DATALINK_INVALID_LINKID)
1206 1202                                  newlink->ll_linkid = dlmgmt_nextlinkid;
1207 1203                          if (avl_find(&dlmgmt_id_avl, newlink, &id_where) !=
1208 1204                              NULL) {
1209 1205                                  dlmgmt_log(LOG_WARNING, "Link ID %d is already"
1210 1206                                      " in use, destroying link %s",
1211 1207                                      newlink->ll_linkid, newlink->ll_link);
1212 1208                                  link_destroy(newlink);
1213 1209                                  continue;
1214 1210                          }
1215 1211  
1216 1212                          if ((req->ls_flags & DLMGMT_ACTIVE) &&
1217 1213                              link_activate(newlink) != 0) {
1218 1214                                  dlmgmt_log(LOG_WARNING, "Unable to activate %s",
1219 1215                                      newlink->ll_link);
1220 1216                                  link_destroy(newlink);
1221 1217                                  continue;
1222 1218                          }
1223 1219  
1224 1220                          avl_insert(&dlmgmt_id_avl, newlink, id_where);
1225 1221                          /*
1226 1222                           * link_activate call above can insert newlink in
1227 1223                           * dlmgmt_name_avl tree when activating a link that is
1228 1224                           * assigned to a NGZ.
1229 1225                           */
1230 1226                          if (avl_find(&dlmgmt_name_avl, newlink,
1231 1227                              &name_where) == NULL)
1232 1228                                  avl_insert(&dlmgmt_name_avl, newlink,
1233 1229                                      name_where);
1234 1230  
1235 1231                          dlmgmt_advance(newlink);
1236 1232                          newlink->ll_flags |= req->ls_flags;
1237 1233                  }
1238 1234          }
1239 1235  
1240 1236          return (err);
1241 1237  }
1242 1238  
1243 1239  /*
1244 1240   * Generate an entry in the link database.
1245 1241   * Each entry has this format:
1246 1242   * <link name>  <prop0>=<type>,<val>;...;<propn>=<type>,<val>;
1247 1243   */
1248 1244  static void
1249 1245  generate_link_line(dlmgmt_link_t *linkp, boolean_t persist, char *buf)
1250 1246  {
1251 1247          char                    tmpbuf[MAXLINELEN];
1252 1248          char                    *ptr = tmpbuf;
1253 1249          char                    *lim = tmpbuf + MAXLINELEN;
1254 1250          dlmgmt_linkattr_t       *cur_p = NULL;
1255 1251          uint64_t                u64;
1256 1252  
1257 1253          ptr += snprintf(ptr, BUFLEN(lim, ptr), "%s\t", linkp->ll_link);
1258 1254          if (!persist) {
1259 1255                  char zname[ZONENAME_MAX];
1260 1256                  /*
1261 1257                   * We store the linkid and the zone name in the active database
1262 1258                   * so that dlmgmtd can recover in the event that it is
1263 1259                   * restarted.
1264 1260                   */
1265 1261                  u64 = linkp->ll_linkid;
1266 1262                  ptr += write_uint64(ptr, BUFLEN(lim, ptr), "linkid", &u64);
1267 1263  
1268 1264                  if (getzonenamebyid(linkp->ll_zoneid, zname,
1269 1265                      sizeof (zname)) != -1) {
1270 1266                          ptr += write_str(ptr, BUFLEN(lim, ptr), "zone", zname);
1271 1267                  }
1272 1268          }
1273 1269          u64 = linkp->ll_class;
1274 1270          ptr += write_uint64(ptr, BUFLEN(lim, ptr), "class", &u64);
1275 1271          u64 = linkp->ll_media;
1276 1272          ptr += write_uint64(ptr, BUFLEN(lim, ptr), "media", &u64);
1277 1273  
1278 1274          /*
1279 1275           * The daemon does not keep any active link attribute. Only store the
1280 1276           * attributes if this request is for persistent configuration,
1281 1277           */
1282 1278          if (persist) {
1283 1279                  for (cur_p = linkp->ll_head; cur_p != NULL;
1284 1280                      cur_p = cur_p->lp_next) {
1285 1281                          ptr += translators[cur_p->lp_type].write_func(ptr,
1286 1282                              BUFLEN(lim, ptr), cur_p->lp_name, cur_p->lp_val);
1287 1283                  }
1288 1284          }
1289 1285  
1290 1286          if (ptr <= lim)
1291 1287                  (void) snprintf(buf, MAXLINELEN, "%s\n", tmpbuf);
1292 1288  }
1293 1289  
1294 1290  int
1295 1291  dlmgmt_delete_db_entry(dlmgmt_link_t *linkp, uint32_t flags)
1296 1292  {
1297 1293          return (dlmgmt_db_update(DLMGMT_DB_OP_DELETE, linkp->ll_link, linkp,
1298 1294              flags));
1299 1295  }
1300 1296  
1301 1297  int
1302 1298  dlmgmt_write_db_entry(const char *entryname, dlmgmt_link_t *linkp,
1303 1299      uint32_t flags)
1304 1300  {
1305 1301          int err;
1306 1302  
1307 1303          if (flags & DLMGMT_PERSIST) {
1308 1304                  if ((err = dlmgmt_db_update(DLMGMT_DB_OP_WRITE, entryname,
1309 1305                      linkp, DLMGMT_PERSIST)) != 0) {
1310 1306                          return (err);
1311 1307                  }
1312 1308          }
1313 1309  
1314 1310          if (flags & DLMGMT_ACTIVE) {
1315 1311                  if (((err = dlmgmt_db_update(DLMGMT_DB_OP_WRITE, entryname,
1316 1312                      linkp, DLMGMT_ACTIVE)) != 0) && (flags & DLMGMT_PERSIST)) {
1317 1313                          (void) dlmgmt_db_update(DLMGMT_DB_OP_DELETE, entryname,
1318 1314                              linkp, DLMGMT_PERSIST);
1319 1315                          return (err);
1320 1316                  }
1321 1317          }
1322 1318  
1323 1319          return (0);
1324 1320  }
1325 1321  
1326 1322  /*
1327 1323   * Upgrade properties that have link IDs as values to link names.  Because '.'
1328 1324   * is a valid linkname character, the port separater for link aggregations
1329 1325   * must be changed to ':'.
1330 1326   */
1331 1327  static void
1332 1328  linkattr_upgrade(dlmgmt_linkattr_t *attrp)
1333 1329  {
1334 1330          datalink_id_t   linkid;
1335 1331          char            *portidstr;
1336 1332          char            portname[MAXLINKNAMELEN + 1];
1337 1333          dlmgmt_link_t   *linkp;
1338 1334          char            *new_attr_val;
1339 1335          size_t          new_attr_sz;
1340 1336          boolean_t       upgraded = B_FALSE;
1341 1337  
1342 1338          if (strcmp(attrp->lp_name, "linkover") == 0 ||
1343 1339              strcmp(attrp->lp_name, "simnetpeer") == 0) {
1344 1340                  if (attrp->lp_type == DLADM_TYPE_UINT64) {
1345 1341                          linkid = (datalink_id_t)*(uint64_t *)attrp->lp_val;
1346 1342                          if ((linkp = link_by_id(linkid, GLOBAL_ZONEID)) == NULL)
1347 1343                                  return;
1348 1344                          new_attr_sz = strlen(linkp->ll_link) + 1;
1349 1345                          if ((new_attr_val = malloc(new_attr_sz)) == NULL)
1350 1346                                  return;
1351 1347                          (void) strcpy(new_attr_val, linkp->ll_link);
1352 1348                          upgraded = B_TRUE;
1353 1349                  }
1354 1350          } else if (strcmp(attrp->lp_name, "portnames") == 0) {
1355 1351                  /*
1356 1352                   * The old format for "portnames" was
1357 1353                   * "<linkid>.[<linkid>.]...".  The new format is
1358 1354                   * "<linkname>:[<linkname>:]...".
1359 1355                   */
1360 1356                  if (!isdigit(((char *)attrp->lp_val)[0]))
1361 1357                          return;
1362 1358                  new_attr_val = calloc(MAXLINKATTRVALLEN, sizeof (char));
1363 1359                  if (new_attr_val == NULL)
1364 1360                          return;
1365 1361                  portidstr = (char *)attrp->lp_val;
1366 1362                  while (*portidstr != '\0') {
1367 1363                          errno = 0;
1368 1364                          linkid = strtol(portidstr, &portidstr, 10);
1369 1365                          if (linkid == 0 || *portidstr != '.' ||
1370 1366                              (linkp = link_by_id(linkid, GLOBAL_ZONEID)) ==
1371 1367                              NULL) {
1372 1368                                  free(new_attr_val);
1373 1369                                  return;
1374 1370                          }
1375 1371                          (void) snprintf(portname, sizeof (portname), "%s:",
1376 1372                              linkp->ll_link);
1377 1373                          if (strlcat(new_attr_val, portname,
1378 1374                              MAXLINKATTRVALLEN) >= MAXLINKATTRVALLEN) {
1379 1375                                  free(new_attr_val);
1380 1376                                  return;
1381 1377                          }
1382 1378                          /* skip the '.' delimiter */
1383 1379                          portidstr++;
1384 1380                  }
1385 1381                  new_attr_sz = strlen(new_attr_val) + 1;
1386 1382                  upgraded = B_TRUE;
1387 1383          }
1388 1384  
1389 1385          if (upgraded) {
1390 1386                  attrp->lp_type = DLADM_TYPE_STR;
1391 1387                  attrp->lp_sz = new_attr_sz;
1392 1388                  free(attrp->lp_val);
1393 1389                  attrp->lp_val = new_attr_val;
1394 1390          }
1395 1391  }
1396 1392  
1397 1393  static void
1398 1394  dlmgmt_db_upgrade(dlmgmt_link_t *linkp)
1399 1395  {
1400 1396          dlmgmt_linkattr_t *attrp;
1401 1397  
1402 1398          for (attrp = linkp->ll_head; attrp != NULL; attrp = attrp->lp_next)
1403 1399                  linkattr_upgrade(attrp);
1404 1400  }
1405 1401  
1406 1402  static void
1407 1403  dlmgmt_db_phys_activate(dlmgmt_link_t *linkp)
1408 1404  {
1409 1405          linkp->ll_flags |= DLMGMT_ACTIVE;
1410 1406          (void) dlmgmt_write_db_entry(linkp->ll_link, linkp, DLMGMT_ACTIVE);
1411 1407  }
1412 1408  
1413 1409  static void
1414 1410  dlmgmt_db_walk(zoneid_t zoneid, datalink_class_t class, db_walk_func_t *func)
1415 1411  {
1416 1412          dlmgmt_link_t *linkp;
1417 1413  
1418 1414          for (linkp = avl_first(&dlmgmt_id_avl); linkp != NULL;
1419 1415              linkp = AVL_NEXT(&dlmgmt_id_avl, linkp)) {
1420 1416                  if (linkp->ll_zoneid == zoneid && (linkp->ll_class & class))
1421 1417                          func(linkp);
1422 1418          }
1423 1419  }
1424 1420  
1425 1421  /*
1426 1422   * Attempt to mitigate one of the deadlocks in the dlmgmtd architecture.
1427 1423   *
1428 1424   * dlmgmt_db_init() calls dlmgmt_process_db_req() which eventually gets to
1429 1425   * dlmgmt_zfop() which tries to fork, enter the zone and read the file.
1430 1426   * Because of the upcall architecture of dlmgmtd this can lead to deadlock
1431 1427   * with the following scenario:
1432 1428   *    a) the thread preparing to fork will have acquired the malloc locks
1433 1429   *       then attempt to suspend every thread in preparation to fork.
1434 1430   *    b) all of the upcalls will be blocked in door_ucred() trying to malloc()
1435 1431   *       and get the credentials of their caller.
1436 1432   *    c) we can't suspend the in-kernel thread making the upcall.
1437 1433   *
1438 1434   * Thus, we cannot serve door requests because we're blocked in malloc()
1439 1435   * which fork() owns, but fork() is in turn blocked on the in-kernel thread
1440 1436   * making the door upcall.  This is a fundamental architectural problem with
1441 1437   * any server handling upcalls and also trying to fork().
1442 1438   *
1443 1439   * To minimize the chance of this deadlock occuring, we check ahead of time to
1444 1440   * see if the file we want to read actually exists in the zone (which it almost
1445 1441   * never does), so we don't need fork in that case (i.e. rarely to never).
1446 1442   */
1447 1443  static boolean_t
1448 1444  zone_file_exists(char *zoneroot, char *filename)
1449 1445  {
1450 1446          struct stat     sb;
1451 1447          char            fname[MAXPATHLEN];
1452 1448  
1453 1449          (void) snprintf(fname, sizeof (fname), "%s/%s", zoneroot, filename);
1454 1450  
1455 1451          if (stat(fname, &sb) == -1)
1456 1452                  return (B_FALSE);
1457 1453  
1458 1454          return (B_TRUE);
1459 1455  }
1460 1456  
1461 1457  /*
1462 1458   * Initialize the datalink <link name, linkid> mapping and the link's
1463 1459   * attributes list based on the configuration file /etc/dladm/datalink.conf
1464 1460   * and the active configuration cache file
1465 1461   * /etc/svc/volatile/dladm/datalink-management:default.cache.
1466 1462   */
1467 1463  int
1468 1464  dlmgmt_db_init(zoneid_t zoneid, char *zoneroot)
1469 1465  {
1470 1466          dlmgmt_db_req_t *req;
1471 1467          int             err;
1472 1468          boolean_t       boot = B_FALSE;
1473 1469          char            tdir[MAXPATHLEN];
1474 1470          char            *path = cachefile;
1475 1471  
1476 1472          if ((req = dlmgmt_db_req_alloc(DLMGMT_DB_OP_READ, NULL,
1477 1473              DATALINK_INVALID_LINKID, zoneid, DLMGMT_ACTIVE, &err)) == NULL)
1478 1474                  return (err);
1479 1475  
1480 1476          /* Handle running in a non-native branded zone (i.e. has /native) */
1481 1477          if (zone_file_exists(zoneroot, "/native" DLMGMT_TMPFS_DIR)) {
1482 1478                  (void) snprintf(tdir, sizeof (tdir), "/native%s", cachefile);
1483 1479                  path = tdir;
1484 1480          }
1485 1481  
1486 1482          if (zone_file_exists(zoneroot, path)) {
1487 1483                  if ((err = dlmgmt_process_db_req(req)) != 0) {
1488 1484                          /*
1489 1485                           * If we get back ENOENT, that means that the active
1490 1486                           * configuration file doesn't exist yet, and is not an
1491 1487                           * error.  We'll create it down below after we've
  
    | 
      ↓ open down ↓ | 
    747 lines elided | 
    
      ↑ open up ↑ | 
  
1492 1488                           * loaded the persistent configuration.
1493 1489                           */
1494 1490                          if (err != ENOENT)
1495 1491                                  goto done;
1496 1492                          boot = B_TRUE;
1497 1493                  }
1498 1494          } else {
1499 1495                  boot = B_TRUE;
1500 1496          }
1501 1497  
1502      -        if (zone_file_exists(zoneroot, DLMGMT_PERSISTENT_DB_PATH)) {
1503      -                req->ls_flags = DLMGMT_PERSIST;
1504      -                err = dlmgmt_process_db_req(req);
1505      -                if (err != 0 && err != ENOENT)
1506      -                        goto done;
1507      -        }
     1498 +        req->ls_flags = DLMGMT_PERSIST;
     1499 +        err = dlmgmt_process_db_req(req);
     1500 +        if (err != 0 && err != ENOENT)
     1501 +                goto done;
1508 1502          err = 0;
1509 1503          if (rewrite_needed) {
1510 1504                  /*
1511 1505                   * First update links in memory, then dump the entire db to
1512 1506                   * disk.
1513 1507                   */
1514 1508                  dlmgmt_db_walk(zoneid, DATALINK_CLASS_ALL, dlmgmt_db_upgrade);
1515 1509                  req->ls_op = DLMGMT_DB_OP_WRITE;
1516 1510                  req->ls_linkid = DATALINK_ALL_LINKID;
1517 1511                  if ((err = dlmgmt_process_db_req(req)) != 0 &&
1518 1512                      err != EINPROGRESS)
1519 1513                          goto done;
1520 1514          }
1521 1515          if (boot) {
1522 1516                  dlmgmt_db_walk(zoneid, DATALINK_CLASS_PHYS,
1523 1517                      dlmgmt_db_phys_activate);
1524 1518          }
1525 1519  
1526 1520  done:
1527 1521          if (err == EINPROGRESS)
1528 1522                  err = 0;
1529 1523          else
1530 1524                  free(req);
1531 1525          return (err);
1532 1526  }
1533 1527  
1534 1528  /*
1535 1529   * Remove all links in the given zoneid.
1536 1530   *
1537 1531   * We do this work in two different passes. In the first pass, we remove any
1538 1532   * entry that hasn't been loaned and mark every entry that has been loaned as
1539 1533   * something that is going to be tombstomed. In the second pass, we drop the
1540 1534   * table lock for every entry and remove the tombstombed entry for our zone.
1541 1535   */
1542 1536  void
1543 1537  dlmgmt_db_fini(zoneid_t zoneid)
1544 1538  {
1545 1539          dlmgmt_link_t *linkp = avl_first(&dlmgmt_name_avl), *next_linkp;
1546 1540  
1547 1541          while (linkp != NULL) {
1548 1542                  next_linkp = AVL_NEXT(&dlmgmt_name_avl, linkp);
1549 1543                  if (linkp->ll_zoneid == zoneid) {
1550 1544                          boolean_t onloan = linkp->ll_onloan;
1551 1545  
1552 1546                          /*
1553 1547                           * Cleanup any VNICs that were loaned to the zone
1554 1548                           * before the zone goes away and we can no longer
  
    | 
      ↓ open down ↓ | 
    37 lines elided | 
    
      ↑ open up ↑ | 
  
1555 1549                           * refer to the VNIC by the name/zoneid.
1556 1550                           */
1557 1551                          if (onloan) {
1558 1552                                  (void) dlmgmt_delete_db_entry(linkp,
1559 1553                                      DLMGMT_ACTIVE);
1560 1554                                  linkp->ll_tomb = B_TRUE;
1561 1555                          } else {
1562 1556                                  (void) dlmgmt_destroy_common(linkp,
1563 1557                                      DLMGMT_ACTIVE | DLMGMT_PERSIST);
1564 1558                          }
1565      -
1566 1559                  }
1567 1560                  linkp = next_linkp;
1568 1561          }
1569 1562  
1570 1563  again:
1571 1564          linkp = avl_first(&dlmgmt_name_avl);
1572 1565          while (linkp != NULL) {
1573 1566                  vnic_ioc_delete_t ioc;
1574 1567  
1575 1568                  next_linkp = AVL_NEXT(&dlmgmt_name_avl, linkp);
1576 1569  
1577 1570                  if (linkp->ll_zoneid != zoneid) {
1578 1571                          linkp = next_linkp;
1579 1572                          continue;
1580 1573                  }
1581 1574                  ioc.vd_vnic_id = linkp->ll_linkid;
1582 1575                  if (linkp->ll_tomb != B_TRUE)
1583 1576                          abort();
1584 1577  
1585 1578                  /*
1586 1579                   * We have to drop the table lock while going up into the
1587 1580                   * kernel. If we hold the table lock while deleting a vnic, we
1588 1581                   * may get blocked on the mac perimeter and the holder of it may
1589 1582                   * want something from dlmgmtd.
1590 1583                   */
1591 1584                  dlmgmt_table_unlock();
1592 1585  
1593 1586                  if (ioctl(dladm_dld_fd(dld_handle),
1594 1587                      VNIC_IOC_DELETE, &ioc) < 0)
1595 1588                          dlmgmt_log(LOG_WARNING, "dlmgmt_db_fini "
1596 1589                              "delete VNIC ioctl failed %d %d",
1597 1590                              ioc.vd_vnic_id, errno);
1598 1591  
1599 1592                  /*
1600 1593                   * Even though we've dropped the lock, we know that nothing else
1601 1594                   * could have removed us. Therefore, it should be safe to go
1602 1595                   * through and delete ourselves, but do nothing else. We'll have
1603 1596                   * to restart iteration from the beginning. This can be painful.
1604 1597                   */
1605 1598                  dlmgmt_table_lock(B_TRUE);
1606 1599  
1607 1600                  (void) dlmgmt_destroy_common(linkp,
1608 1601                      DLMGMT_ACTIVE | DLMGMT_PERSIST);
1609 1602                  goto again;
1610 1603          }
1611 1604  
1612 1605  }
  
    | 
      ↓ open down ↓ | 
    37 lines elided | 
    
      ↑ open up ↑ | 
  
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX