--- old/usr/src/uts/common/io/ixgbe/ixgbe_main.c	Wed Jul 18 20:46:55 2012
+++ new/usr/src/uts/common/io/ixgbe/ixgbe_main.c	Wed Jul 18 20:46:54 2012
@@ -25,6 +25,8 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  */
 
 #include "ixgbe_sw.h"
@@ -291,6 +293,42 @@
 	| IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
 };
 
+static adapter_info_t ixgbe_X540_cap = {
+	128,		/* maximum number of rx queues */
+	1,		/* minimum number of rx queues */
+	128,		/* default number of rx queues */
+	64,		/* maximum number of rx groups */
+	1,		/* minimum number of rx groups */
+	1,		/* default number of rx groups */
+	128,		/* maximum number of tx queues */
+	1,		/* minimum number of tx queues */
+	8,		/* default number of tx queues */
+	15500,		/* maximum MTU size */
+	0xFF8,		/* maximum interrupt throttle rate */
+	0,		/* minimum interrupt throttle rate */
+	200,		/* default interrupt throttle rate */
+	64,		/* maximum total msix vectors */
+	16,		/* maximum number of ring vectors */
+	2,		/* maximum number of other vectors */
+	/* XXX KEBE ASKS, Do we care about X540's SDP3? */
+	(IXGBE_EICR_LSC
+	| IXGBE_EICR_GPI_SDP0
+	| IXGBE_EICR_GPI_SDP1
+	| IXGBE_EICR_GPI_SDP2
+	/* | IXGBE_EICR_GPI_SDP3 */), /* "other" interrupt types handled */
+
+	(IXGBE_SDP1_GPIEN
+	| IXGBE_SDP2_GPIEN
+	/* | IXGBE_SDP3_GPIEN */), /* "other" interrupt types enable mask */
+
+	/* XXX KEBE ASKS, SFP_PLUG capable?!? */
+	(IXGBE_FLAG_DCA_CAPABLE
+	| IXGBE_FLAG_RSS_CAPABLE
+	| IXGBE_FLAG_VMDQ_CAPABLE
+	| IXGBE_FLAG_RSC_CAPABLE
+	/* | IXGBE_FLAG_SFP_PLUG_CAPABLE */) /* capability flags */
+};
+
 /*
  * Module Initialization Functions.
  */
@@ -867,6 +905,15 @@
 		}
 		break;
 
+	case ixgbe_mac_X540:
+                IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
+                ixgbe->capab = &ixgbe_X540_cap;
+		/*
+		 * For now, X540 is all set in its capab structure.
+		 * As other X540 variants show up, things can change here.
+		 */
+		break; 
+
 	default:
 		IXGBE_DEBUGLOG_1(ixgbe,
 		    "adapter not supported in ixgbe_identify_hardware(): %d\n",
@@ -1200,8 +1247,8 @@
 	 * Setup default flow control thresholds - enable/disable
 	 * & flow control type is controlled by ixgbe.conf
 	 */
-	hw->fc.high_water = DEFAULT_FCRTH;
-	hw->fc.low_water = DEFAULT_FCRTL;
+	hw->fc.high_water[0] = DEFAULT_FCRTH;
+	hw->fc.low_water[0] = DEFAULT_FCRTL;
 	hw->fc.pause_time = DEFAULT_FCPAUSE;
 	hw->fc.send_xon = B_TRUE;
 
@@ -2096,7 +2143,7 @@
 	}
 	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
 
-	if (hw->mac.type == ixgbe_mac_82599EB) {
+	if (hw->mac.type >= ixgbe_mac_82599EB) {
 		reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
 		reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
 		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
@@ -2333,13 +2380,44 @@
 		 * Turn off relaxed ordering for head write back or it will
 		 * cause problems with the tx recycling
 		 */
-		reg_val = IXGBE_READ_REG(hw,
-		    IXGBE_DCA_TXCTRL(tx_ring->index));
-		reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
-		IXGBE_WRITE_REG(hw,
-		    IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
+#if 1
+		/* XXX KEBE ASKS --> Should we do what FreeBSD does? */
+		reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
+		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
+		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
+		reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
+		if (hw->mac.type == ixgbe_mac_82598EB) {
+			IXGBE_WRITE_REG(hw,
+			    IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
+		} else {
+			IXGBE_WRITE_REG(hw,
+			    IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
+		}
+#else
+		/* XXX KEBE ASKS --> Or should we do what we've always done? */
+		reg_val = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index));
+		reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
+		IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
+#endif
 	} else {
 		tx_ring->tbd_head_wb = NULL;
+#if 1
+		/*
+		 * XXX KEBE ASKS --> Should we do what FreeBSD does and
+		 * twiddle TXCTRL_DESC_WR0_EN off anyway?
+		 */
+		reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
+		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
+		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
+		reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
+		if (hw->mac.type == ixgbe_mac_82598EB) {
+			IXGBE_WRITE_REG(hw,
+			    IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
+		} else {
+			IXGBE_WRITE_REG(hw,
+			    IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
+		}
+#endif
 	}
 
 	tx_ring->tbd_head = 0;
@@ -2386,6 +2464,7 @@
 				break;
 
 			case ixgbe_mac_82599EB:
+			case ixgbe_mac_X540:
 				IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
 				    ring_mapping);
 				break;
@@ -2404,6 +2483,7 @@
 			break;
 
 		case ixgbe_mac_82599EB:
+		case ixgbe_mac_X540:
 			IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
 			break;
 
@@ -2420,13 +2500,23 @@
 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
 
 	/*
-	 * enable DMA for 82599 parts
+	 * enable DMA for 82599 and X540 parts
 	 */
-	if (hw->mac.type == ixgbe_mac_82599EB) {
+	if (hw->mac.type >= ixgbe_mac_82599EB) {
 	/* DMATXCTL.TE must be set after all Tx config is complete */
 		reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 		reg_val |= IXGBE_DMATXCTL_TE;
 		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
+#if 0
+		/* XXX KEBE SAYS - FreeBSD sets up MTQC.  Should we? */
+		/* Disable arbiter to set MTQC */
+		reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
+		reg_val |= IXGBE_RTTDCS_ARBDIS;
+		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
+		IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
+		reg_val &= ~IXGBE_RTTDCS_ARBDIS;
+		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
+#endif
 	}
 
 	/*
@@ -2526,6 +2616,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		/*
 		 * Enable VMDq-only.
 		 */
@@ -2619,6 +2710,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		/*
 		 * Enable RSS & Setup RSS Hash functions
 		 */
@@ -2664,7 +2756,7 @@
 	rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
 
-	if (hw->mac.type == ixgbe_mac_82599EB) {
+	if (hw->mac.type >= ixgbe_mac_82599EB) {
 		/*
 		 * Enable Virtualization and Replication.
 		 */
@@ -2839,7 +2931,7 @@
 	 * Update the multicast addresses to the MTA registers
 	 */
 	(void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
-	    ixgbe_mc_table_itr);
+	    ixgbe_mc_table_itr, TRUE);
 }
 
 /*
@@ -2875,6 +2967,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		/*
 		 * 82599 supports the following combination:
 		 * vmdq no. x rss no.
@@ -2883,6 +2976,8 @@
 		 * 1 x [1..16]
 		 * However 8 rss queue per pool (vmdq) is sufficient for
 		 * most cases.
+		 *
+		 * For now, treat X540 like the 82599.
 		 */
 		ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
 		if (ixgbe->num_rx_groups == 1) {
@@ -3045,7 +3140,7 @@
 	ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
 	    PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
 
-	/* Head Write Back not recommended for 82599 */
+	/* Head Write Back not recommended for 82599 and X540 */
 	if (hw->mac.type >= ixgbe_mac_82599EB) {
 		ixgbe->tx_head_wb_enable = B_FALSE;
 	}
@@ -3069,9 +3164,9 @@
 	}
 
 	/*
-	 * ixgbe LRO only been supported by 82599 now
+	 * ixgbe LRO only been supported by 82599 and X540 now
 	 */
-	if (hw->mac.type != ixgbe_mac_82599EB) {
+	if (hw->mac.type < ixgbe_mac_82599EB) {
 		ixgbe->lro_enable = B_FALSE;
 	}
 	ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
@@ -3099,11 +3194,11 @@
 	    ixgbe->capab->max_intr_throttle,
 	    ixgbe->capab->def_intr_throttle);
 	/*
-	 * 82599 requires the interupt throttling rate is
+	 * 82599 and X540 require the interupt throttling rate is
 	 * a multiple of 8. This is enforced by the register
 	 * definiton.
 	 */
-	if (hw->mac.type == ixgbe_mac_82599EB)
+	if (hw->mac.type >= ixgbe_mac_82599EB)
 		ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
 }
 
@@ -3229,7 +3324,7 @@
 		ixgbe->link_check_complete = B_TRUE;
 
 		/* Link is up, enable flow control settings */
-		(void) ixgbe_fc_enable(hw, 0);
+		(void) ixgbe_fc_enable(hw);
 
 		/*
 		 * The Link is up, check whether it was marked as down earlier
@@ -3748,6 +3843,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		gpie |= ixgbe->capab->other_gpie;
 
 		/* Enable RSC Delay 8us when LRO enabled  */
@@ -3941,6 +4037,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
 		reg |= (IXGBE_AUTOC_FLU |
 		    IXGBE_AUTOC_10G_KX4);
@@ -4159,6 +4256,7 @@
 				break;
 
 			case ixgbe_mac_82599EB:
+			case ixgbe_mac_X540:
 				ixgbe->eimc = IXGBE_82599_OTHER_INTR;
 				IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
 				break;
@@ -4252,6 +4350,7 @@
 			break;
 
 		case ixgbe_mac_82599EB:
+		case ixgbe_mac_X540:
 			ixgbe->eimc = IXGBE_82599_OTHER_INTR;
 			IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
 			break;
@@ -4331,6 +4430,7 @@
 				break;
 
 			case ixgbe_mac_82599EB:
+			case ixgbe_mac_X540:
 				ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
 				ixgbe_intr_other_work(ixgbe, eicr);
 				break;
@@ -4731,6 +4831,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		if (cause == -1) {
 			/* other causes */
 			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
@@ -4784,6 +4885,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		if (cause == -1) {
 			/* other causes */
 			index = (intr_alloc_entry & 1) * 8;
@@ -4833,6 +4935,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		if (cause == -1) {
 			/* other causes */
 			index = (intr_alloc_entry & 1) * 8;
@@ -4875,6 +4978,7 @@
 			return (sw_rx_index);
 
 		case ixgbe_mac_82599EB:
+		case ixgbe_mac_X540:
 			return (sw_rx_index * 2);
 
 		default:
@@ -4890,6 +4994,7 @@
 			return (hw_rx_index);
 
 		case ixgbe_mac_82599EB:
+		case ixgbe_mac_X540:
 			if (ixgbe->num_rx_groups > 32) {
 				hw_rx_index = (sw_rx_index /
 				    rx_ring_per_group) * 2 +
@@ -4994,6 +5099,7 @@
 		break;
 
 	case ixgbe_mac_82599EB:
+	case ixgbe_mac_X540:
 		for (v_idx = 0; v_idx < 64; v_idx++)
 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
