OS-5591 Double flock(3C) causes undue block
OS-5585 fcntl(F_OFD_GETLK) should return EINVAL on bad parameters
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>
OS-4098 move open, close and fcntl into the kernel

*** 52,62 ****
  #include <sys/rctl.h>
  #include <sys/nbmlock.h>
  
  #include <sys/cmn_err.h>
  
! static int flock_check(vnode_t *, flock64_t *, offset_t, offset_t);
  static int flock_get_start(vnode_t *, flock64_t *, offset_t, u_offset_t *);
  static void fd_too_big(proc_t *);
  
  /*
   * File control.
--- 52,63 ----
  #include <sys/rctl.h>
  #include <sys/nbmlock.h>
  
  #include <sys/cmn_err.h>
  
! /* This is global so that it can be used by brand emulation. */
! int flock_check(vnode_t *, flock64_t *, offset_t, offset_t);
  static int flock_get_start(vnode_t *, flock64_t *, offset_t, u_offset_t *);
  static void fd_too_big(proc_t *);
  
  /*
   * File control.
*** 363,373 ****
                                  error = EINVAL;
                                  break;
                          }
                  }
  
!                 if (cmd == F_OFD_SETLK || cmd == F_OFD_SETLKW) {
                          /*
                           * TBD OFD-style locking is currently limited to
                           * covering the entire file.
                           */
                          if (bf.l_whence != 0 || bf.l_start != 0 ||
--- 364,375 ----
                                  error = EINVAL;
                                  break;
                          }
                  }
  
!                 if (cmd == F_OFD_GETLK || cmd == F_OFD_SETLK ||
!                     cmd == F_OFD_SETLKW) {
                          /*
                           * TBD OFD-style locking is currently limited to
                           * covering the entire file.
                           */
                          if (bf.l_whence != 0 || bf.l_start != 0 ||