676 {
677 mblk_t *mp = NULL;
678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
679 RXQ_LOCK(rxq);
680 if (!iq->polling) {
681 mp = t4_ring_rx(rxq, iq->qsize/8);
682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 }
685 RXQ_UNLOCK(rxq);
686 if (mp != NULL)
687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 rxq->ring_gen_num);
689 }
690
691 /* Deals with interrupts on the given ingress queue */
692 /* ARGSUSED */
693 uint_t
694 t4_intr(caddr_t arg1, caddr_t arg2)
695 {
696 /* LINTED: E_BAD_PTR_CAST_ALIGN */
697 struct sge_iq *iq = (struct sge_iq *)arg2;
698 int state;
699
700 /* Right now receive polling is only enabled for MSI-X and
701 * when we have enough msi-x vectors i.e no interrupt forwarding.
702 */
703 if (iq->adapter->props.multi_rings) {
704 t4_intr_rx_work(iq);
705 } else {
706 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
707 if (state == IQS_IDLE) {
708 (void) service_iq(iq, 0);
709 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
710 }
711 }
712 return (DDI_INTR_CLAIMED);
713 }
714
715 /* Deals with error interrupts */
716 /* ARGSUSED */
759 membar_consumer();
760
761 m = NULL;
762 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
763 lq = be32_to_cpu(ctrl->pldbuflen_qid);
764 rss = (const void *)iq->cdesc;
765
766 switch (rsp_type) {
767 case X_RSPD_TYPE_FLBUF:
768
769 ASSERT(iq->flags & IQ_HAS_FL);
770
771 if (CPL_RX_PKT == rss->opcode) {
772 cpl = (void *)(rss + 1);
773 pkt_len = be16_to_cpu(cpl->len);
774
775 if (iq->polling && ((received_bytes + pkt_len) > budget))
776 goto done;
777
778 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
779 if (m == NULL) {
780 panic("%s: line %d.", __func__,
781 __LINE__);
782 }
783
784 iq->intr_next = iq->intr_params;
785 m->b_rptr += sc->sge.pktshift;
786 if (sc->params.tp.rx_pkt_encap)
787 /* It is enabled only in T6 config file */
788 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
789 else
790 err_vec = ntohs(cpl->err_vec);
791
792 csum_ok = cpl->csum_calc && !err_vec;
793
794 /* TODO: what about cpl->ip_frag? */
795 if (csum_ok && !cpl->ip_frag) {
796 mac_hcksum_set(m, 0, 0, 0, 0xffff,
797 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
798 HCK_IPV4_HDRCKSUM_OK);
799 rxq->rxcsum++;
800 }
801 rxq->rxpkts++;
802 rxq->rxbytes += pkt_len;
803 received_bytes += pkt_len;
804
805 *mblk_tail = m;
806 mblk_tail = &m->b_next;
807
808 break;
809 }
810
811 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
812 if (m == NULL) {
813 panic("%s: line %d.", __func__,
814 __LINE__);
815 }
816
817 case X_RSPD_TYPE_CPL:
818 ASSERT(rss->opcode < NUM_CPL_CMDS);
819 sc->cpl_handler[rss->opcode](iq, rss, m);
820 break;
821
822 default:
823 break;
824 }
825 iq_next(iq);
826 ++ndescs;
827 if (!iq->polling && (ndescs == budget))
828 break;
829 }
830
831 done:
832
833 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
834 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
835 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
844 add_fl_to_sfl(sc, fl);
845 }
846 return (mblk_head);
847 }
848
849 /*
850 * Deals with anything and everything on the given ingress queue.
851 */
852 static int
853 service_iq(struct sge_iq *iq, int budget)
854 {
855 struct sge_iq *q;
856 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
857 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
858 struct adapter *sc = iq->adapter;
859 struct rsp_ctrl *ctrl;
860 const struct rss_header *rss;
861 int ndescs = 0, limit, fl_bufs_used = 0;
862 int rsp_type;
863 uint32_t lq;
864 mblk_t *m;
865 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
866
867 limit = budget ? budget : iq->qsize / 8;
868
869 /*
870 * We always come back and check the descriptor ring for new indirect
871 * interrupts and other responses after running a single handler.
872 */
873 for (;;) {
874 while (is_new_response(iq, &ctrl)) {
875
876 membar_consumer();
877
878 m = NULL;
879 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
880 lq = be32_to_cpu(ctrl->pldbuflen_qid);
881 rss = (const void *)iq->cdesc;
882
883 switch (rsp_type) {
884 case X_RSPD_TYPE_FLBUF:
885
886 ASSERT(iq->flags & IQ_HAS_FL);
887
888 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
889 if (m == NULL) {
890 panic("%s: line %d.", __func__,
891 __LINE__);
892 }
893
894 /* FALLTHRU */
895 case X_RSPD_TYPE_CPL:
896
897 ASSERT(rss->opcode < NUM_CPL_CMDS);
898 sc->cpl_handler[rss->opcode](iq, rss, m);
899 break;
900
901 case X_RSPD_TYPE_INTR:
902
903 /*
904 * Interrupts should be forwarded only to queues
905 * that are not forwarding their interrupts.
906 * This means service_iq can recurse but only 1
907 * level deep.
908 */
909 ASSERT(budget == 0);
910
911 q = sc->sge.iqmap[lq - sc->sge.iq_start];
912 if (atomic_cas_uint(&q->state, IQS_IDLE,
951
952 if (STAILQ_EMPTY(&iql) != 0)
953 break;
954
955 /*
956 * Process the head only, and send it to the back of the list if
957 * it's still not done.
958 */
959 q = STAILQ_FIRST(&iql);
960 STAILQ_REMOVE_HEAD(&iql, link);
961 if (service_iq(q, q->qsize / 8) == 0)
962 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
963 else
964 STAILQ_INSERT_TAIL(&iql, q, link);
965 }
966
967 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
968 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
969
970 if (iq->flags & IQ_HAS_FL) {
971 int starved;
972
973 FL_LOCK(fl);
974 fl->needed += fl_bufs_used;
975 starved = refill_fl(sc, fl, fl->cap / 4);
976 FL_UNLOCK(fl);
977 if (starved != 0)
978 add_fl_to_sfl(sc, fl);
979 }
980
981 return (0);
982 }
983
984 #ifdef TCP_OFFLOAD_ENABLE
985 int
986 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
987 {
988 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
989 }
990
991 /*
1236 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1237 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1238
1239 iq->flags = 0;
1240 iq->adapter = sc;
1241 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1242 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1243 if (pktc_idx >= 0) {
1244 iq->intr_params |= F_QINTR_CNT_EN;
1245 iq->intr_pktc_idx = pktc_idx;
1246 }
1247 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1248 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1249 }
1250
1251 static inline void
1252 init_fl(struct sge_fl *fl, uint16_t qsize)
1253 {
1254
1255 fl->qsize = qsize;
1256 }
1257
1258 static inline void
1259 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1260 uint8_t tx_chan, uint16_t iqid)
1261 {
1262 struct sge *s = &sc->sge;
1263 uint32_t r;
1264
1265 ASSERT(tx_chan < NCHAN);
1266 ASSERT(eqtype <= EQ_TYPEMASK);
1267
1268 if (is_t5(sc->params.chip)) {
1269 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1270 r >>= S_QUEUESPERPAGEPF0 +
1271 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1272 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1273 }
1274
1275 eq->flags = eqtype & EQ_TYPEMASK;
1670 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1671 cntxt_id, sc->sge.neq - 1);
1672 sc->sge.eqmap[cntxt_id] = eq;
1673
1674 return (rc);
1675 }
1676
1677 static int
1678 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1679 {
1680 int rc, cntxt_id;
1681 struct fw_eq_eth_cmd c;
1682
1683 bzero(&c, sizeof (c));
1684
1685 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1686 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1687 V_FW_EQ_ETH_CMD_VFN(0));
1688 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1689 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1690 c.autoequiqe_to_viid = BE_32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
1691 c.fetchszm_to_iqid =
1692 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1693 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1694 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1695 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1696 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1697 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1698 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1699 c.eqaddr = BE_64(eq->ba);
1700
1701 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1702 if (rc != 0) {
1703 cxgb_printf(pi->dip, CE_WARN,
1704 "failed to create Ethernet egress queue: %d", rc);
1705 return (rc);
1706 }
1707 eq->flags |= EQ_ALLOCATED;
1708
1709 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1710 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
2313 sd = &fl->sdesc[i];
2314
2315 if (sd->rxb != NULL) {
2316 rxbuf_free(sd->rxb);
2317 sd->rxb = NULL;
2318 }
2319 }
2320 }
2321
2322 /*
2323 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2324 */
2325 static mblk_t *
2326 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2327 uint32_t len_newbuf, int *fl_bufs_used)
2328 {
2329 struct mblk_pair frame = {0};
2330 struct rxbuf *rxb;
2331 mblk_t *m = NULL;
2332 uint_t nbuf = 0, len, copy, n;
2333 uint32_t cidx, offset;
2334
2335 /*
2336 * The SGE won't pack a new frame into the current buffer if the entire
2337 * payload doesn't fit in the remaining space. Move on to the next buf
2338 * in that case.
2339 */
2340 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2341 fl->offset = 0;
2342 if (++fl->cidx == fl->cap)
2343 fl->cidx = 0;
2344 nbuf++;
2345 }
2346 cidx = fl->cidx;
2347 offset = fl->offset;
2348
2349 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2350 copy = (len <= fl->copy_threshold);
2351 if (copy != 0) {
2352 frame.head = m = allocb(len, BPRI_HI);
2353 if (m == NULL)
2354 return (NULL);
2355 }
2356
2357 while (len) {
2358 rxb = fl->sdesc[cidx].rxb;
2359 n = min(len, rxb->buf_size - offset);
2360
2361 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2362 DDI_DMA_SYNC_FORKERNEL);
2363
2364 if (copy != 0)
2365 bcopy(rxb->va + offset, m->b_wptr, n);
2366 else {
2367 m = desballoc((unsigned char *)rxb->va + offset, n,
2368 BPRI_HI, &rxb->freefunc);
2369 if (m == NULL) {
2370 freemsg(frame.head);
2371 return (NULL);
2372 }
2373 atomic_inc_uint(&rxb->ref_cnt);
2374 if (frame.head != NULL)
2375 frame.tail->b_cont = m;
2376 else
2377 frame.head = m;
2378 frame.tail = m;
2379 }
2380 m->b_wptr += n;
2381 len -= n;
2382 offset += roundup(n, sc->sge.fl_align);
2383 ASSERT(offset <= rxb->buf_size);
2384 if (offset == rxb->buf_size) {
2385 offset = 0;
2386 if (++cidx == fl->cap)
2387 cidx = 0;
2388 nbuf++;
2389 }
2390 }
3329 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3330 }
3331
3332 if (is_t4(sc->params.chip))
3333 v |= V_PIDX(ndesc);
3334 else
3335 v |= V_PIDX_T5(ndesc);
3336 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3337
3338 membar_producer();
3339
3340 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3341
3342 /*
3343 * Update pending count:
3344 * Deduct the number of descriptors posted
3345 */
3346 fl->pending -= ndesc * 8;
3347 }
3348
3349 /* ARGSUSED */
3350 static int
3351 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3352 mblk_t *m)
3353 {
3354 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3355 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3356 struct adapter *sc = iq->adapter;
3357 struct sge *s = &sc->sge;
3358 struct sge_txq *txq;
3359
3360 txq = (void *)s->eqmap[qid - s->eq_start];
3361 txq->qflush++;
3362 t4_mac_tx_update(txq->port, txq);
3363
3364 return (0);
3365 }
3366
3367 static int
3368 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3369 {
3370 struct adapter *sc = iq->adapter;
3371 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3372
3373 ASSERT(m == NULL);
3374
3375 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3376 const struct rss_header *rss2;
3377
3378 rss2 = (const struct rss_header *)&cpl->data[0];
3379 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3380 }
3381 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3382 }
3383
|
676 {
677 mblk_t *mp = NULL;
678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
679 RXQ_LOCK(rxq);
680 if (!iq->polling) {
681 mp = t4_ring_rx(rxq, iq->qsize/8);
682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 }
685 RXQ_UNLOCK(rxq);
686 if (mp != NULL)
687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 rxq->ring_gen_num);
689 }
690
691 /* Deals with interrupts on the given ingress queue */
692 /* ARGSUSED */
693 uint_t
694 t4_intr(caddr_t arg1, caddr_t arg2)
695 {
696 struct sge_iq *iq = (struct sge_iq *)arg2;
697 int state;
698
699 /* Right now receive polling is only enabled for MSI-X and
700 * when we have enough msi-x vectors i.e no interrupt forwarding.
701 */
702 if (iq->adapter->props.multi_rings) {
703 t4_intr_rx_work(iq);
704 } else {
705 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
706 if (state == IQS_IDLE) {
707 (void) service_iq(iq, 0);
708 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
709 }
710 }
711 return (DDI_INTR_CLAIMED);
712 }
713
714 /* Deals with error interrupts */
715 /* ARGSUSED */
758 membar_consumer();
759
760 m = NULL;
761 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
762 lq = be32_to_cpu(ctrl->pldbuflen_qid);
763 rss = (const void *)iq->cdesc;
764
765 switch (rsp_type) {
766 case X_RSPD_TYPE_FLBUF:
767
768 ASSERT(iq->flags & IQ_HAS_FL);
769
770 if (CPL_RX_PKT == rss->opcode) {
771 cpl = (void *)(rss + 1);
772 pkt_len = be16_to_cpu(cpl->len);
773
774 if (iq->polling && ((received_bytes + pkt_len) > budget))
775 goto done;
776
777 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
778 if (m == NULL)
779 goto done;
780
781 iq->intr_next = iq->intr_params;
782 m->b_rptr += sc->sge.pktshift;
783 if (sc->params.tp.rx_pkt_encap)
784 /* It is enabled only in T6 config file */
785 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
786 else
787 err_vec = ntohs(cpl->err_vec);
788
789 csum_ok = cpl->csum_calc && !err_vec;
790
791 /* TODO: what about cpl->ip_frag? */
792 if (csum_ok && !cpl->ip_frag) {
793 mac_hcksum_set(m, 0, 0, 0, 0xffff,
794 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
795 HCK_IPV4_HDRCKSUM_OK);
796 rxq->rxcsum++;
797 }
798 rxq->rxpkts++;
799 rxq->rxbytes += pkt_len;
800 received_bytes += pkt_len;
801
802 *mblk_tail = m;
803 mblk_tail = &m->b_next;
804
805 break;
806 }
807
808 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
809 if (m == NULL)
810 goto done;
811 /* FALLTHROUGH */
812
813 case X_RSPD_TYPE_CPL:
814 ASSERT(rss->opcode < NUM_CPL_CMDS);
815 sc->cpl_handler[rss->opcode](iq, rss, m);
816 break;
817
818 default:
819 break;
820 }
821 iq_next(iq);
822 ++ndescs;
823 if (!iq->polling && (ndescs == budget))
824 break;
825 }
826
827 done:
828
829 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
830 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
831 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
840 add_fl_to_sfl(sc, fl);
841 }
842 return (mblk_head);
843 }
844
845 /*
846 * Deals with anything and everything on the given ingress queue.
847 */
848 static int
849 service_iq(struct sge_iq *iq, int budget)
850 {
851 struct sge_iq *q;
852 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
853 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
854 struct adapter *sc = iq->adapter;
855 struct rsp_ctrl *ctrl;
856 const struct rss_header *rss;
857 int ndescs = 0, limit, fl_bufs_used = 0;
858 int rsp_type;
859 uint32_t lq;
860 int starved;
861 mblk_t *m;
862 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
863
864 limit = budget ? budget : iq->qsize / 8;
865
866 /*
867 * We always come back and check the descriptor ring for new indirect
868 * interrupts and other responses after running a single handler.
869 */
870 for (;;) {
871 while (is_new_response(iq, &ctrl)) {
872
873 membar_consumer();
874
875 m = NULL;
876 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
877 lq = be32_to_cpu(ctrl->pldbuflen_qid);
878 rss = (const void *)iq->cdesc;
879
880 switch (rsp_type) {
881 case X_RSPD_TYPE_FLBUF:
882
883 ASSERT(iq->flags & IQ_HAS_FL);
884
885 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
886 if (m == NULL) {
887 /*
888 * Rearm the iq with a
889 * longer-than-default timer
890 */
891 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
892 V_INGRESSQID((u32)iq->cntxt_id) |
893 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1)));
894 if (fl_bufs_used > 0) {
895 ASSERT(iq->flags & IQ_HAS_FL);
896 FL_LOCK(fl);
897 fl->needed += fl_bufs_used;
898 starved = refill_fl(sc, fl, fl->cap / 8);
899 FL_UNLOCK(fl);
900 if (starved)
901 add_fl_to_sfl(sc, fl);
902 }
903 return (0);
904 }
905
906 /* FALLTHRU */
907 case X_RSPD_TYPE_CPL:
908
909 ASSERT(rss->opcode < NUM_CPL_CMDS);
910 sc->cpl_handler[rss->opcode](iq, rss, m);
911 break;
912
913 case X_RSPD_TYPE_INTR:
914
915 /*
916 * Interrupts should be forwarded only to queues
917 * that are not forwarding their interrupts.
918 * This means service_iq can recurse but only 1
919 * level deep.
920 */
921 ASSERT(budget == 0);
922
923 q = sc->sge.iqmap[lq - sc->sge.iq_start];
924 if (atomic_cas_uint(&q->state, IQS_IDLE,
963
964 if (STAILQ_EMPTY(&iql) != 0)
965 break;
966
967 /*
968 * Process the head only, and send it to the back of the list if
969 * it's still not done.
970 */
971 q = STAILQ_FIRST(&iql);
972 STAILQ_REMOVE_HEAD(&iql, link);
973 if (service_iq(q, q->qsize / 8) == 0)
974 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
975 else
976 STAILQ_INSERT_TAIL(&iql, q, link);
977 }
978
979 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
980 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
981
982 if (iq->flags & IQ_HAS_FL) {
983
984 FL_LOCK(fl);
985 fl->needed += fl_bufs_used;
986 starved = refill_fl(sc, fl, fl->cap / 4);
987 FL_UNLOCK(fl);
988 if (starved != 0)
989 add_fl_to_sfl(sc, fl);
990 }
991
992 return (0);
993 }
994
995 #ifdef TCP_OFFLOAD_ENABLE
996 int
997 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
998 {
999 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
1000 }
1001
1002 /*
1247 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1248 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1249
1250 iq->flags = 0;
1251 iq->adapter = sc;
1252 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1253 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1254 if (pktc_idx >= 0) {
1255 iq->intr_params |= F_QINTR_CNT_EN;
1256 iq->intr_pktc_idx = pktc_idx;
1257 }
1258 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1259 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1260 }
1261
1262 static inline void
1263 init_fl(struct sge_fl *fl, uint16_t qsize)
1264 {
1265
1266 fl->qsize = qsize;
1267 fl->allocb_fail = 0;
1268 }
1269
1270 static inline void
1271 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1272 uint8_t tx_chan, uint16_t iqid)
1273 {
1274 struct sge *s = &sc->sge;
1275 uint32_t r;
1276
1277 ASSERT(tx_chan < NCHAN);
1278 ASSERT(eqtype <= EQ_TYPEMASK);
1279
1280 if (is_t5(sc->params.chip)) {
1281 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1282 r >>= S_QUEUESPERPAGEPF0 +
1283 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1284 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1285 }
1286
1287 eq->flags = eqtype & EQ_TYPEMASK;
1682 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1683 cntxt_id, sc->sge.neq - 1);
1684 sc->sge.eqmap[cntxt_id] = eq;
1685
1686 return (rc);
1687 }
1688
1689 static int
1690 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1691 {
1692 int rc, cntxt_id;
1693 struct fw_eq_eth_cmd c;
1694
1695 bzero(&c, sizeof (c));
1696
1697 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1698 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1699 V_FW_EQ_ETH_CMD_VFN(0));
1700 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1701 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1702 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1703 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1704 c.fetchszm_to_iqid =
1705 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1706 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1707 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1708 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1709 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1710 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1711 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1712 c.eqaddr = BE_64(eq->ba);
1713
1714 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1715 if (rc != 0) {
1716 cxgb_printf(pi->dip, CE_WARN,
1717 "failed to create Ethernet egress queue: %d", rc);
1718 return (rc);
1719 }
1720 eq->flags |= EQ_ALLOCATED;
1721
1722 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1723 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
2326 sd = &fl->sdesc[i];
2327
2328 if (sd->rxb != NULL) {
2329 rxbuf_free(sd->rxb);
2330 sd->rxb = NULL;
2331 }
2332 }
2333 }
2334
2335 /*
2336 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2337 */
2338 static mblk_t *
2339 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2340 uint32_t len_newbuf, int *fl_bufs_used)
2341 {
2342 struct mblk_pair frame = {0};
2343 struct rxbuf *rxb;
2344 mblk_t *m = NULL;
2345 uint_t nbuf = 0, len, copy, n;
2346 uint32_t cidx, offset, rcidx, roffset;
2347
2348 /*
2349 * The SGE won't pack a new frame into the current buffer if the entire
2350 * payload doesn't fit in the remaining space. Move on to the next buf
2351 * in that case.
2352 */
2353 rcidx = fl->cidx;
2354 roffset = fl->offset;
2355 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2356 fl->offset = 0;
2357 if (++fl->cidx == fl->cap)
2358 fl->cidx = 0;
2359 nbuf++;
2360 }
2361 cidx = fl->cidx;
2362 offset = fl->offset;
2363
2364 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2365 copy = (len <= fl->copy_threshold);
2366 if (copy != 0) {
2367 frame.head = m = allocb(len, BPRI_HI);
2368 if (m == NULL) {
2369 fl->allocb_fail++;
2370 cmn_err(CE_WARN,"%s: mbuf allocation failure "
2371 "count = %llu", __func__,
2372 (unsigned long long)fl->allocb_fail);
2373 fl->cidx = rcidx;
2374 fl->offset = roffset;
2375 return (NULL);
2376 }
2377 }
2378
2379 while (len) {
2380 rxb = fl->sdesc[cidx].rxb;
2381 n = min(len, rxb->buf_size - offset);
2382
2383 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2384 DDI_DMA_SYNC_FORKERNEL);
2385
2386 if (copy != 0)
2387 bcopy(rxb->va + offset, m->b_wptr, n);
2388 else {
2389 m = desballoc((unsigned char *)rxb->va + offset, n,
2390 BPRI_HI, &rxb->freefunc);
2391 if (m == NULL) {
2392 fl->allocb_fail++;
2393 cmn_err(CE_WARN,
2394 "%s: mbuf allocation failure "
2395 "count = %llu", __func__,
2396 (unsigned long long)fl->allocb_fail);
2397 if (frame.head)
2398 freemsgchain(frame.head);
2399 fl->cidx = rcidx;
2400 fl->offset = roffset;
2401 return (NULL);
2402 }
2403 atomic_inc_uint(&rxb->ref_cnt);
2404 if (frame.head != NULL)
2405 frame.tail->b_cont = m;
2406 else
2407 frame.head = m;
2408 frame.tail = m;
2409 }
2410 m->b_wptr += n;
2411 len -= n;
2412 offset += roundup(n, sc->sge.fl_align);
2413 ASSERT(offset <= rxb->buf_size);
2414 if (offset == rxb->buf_size) {
2415 offset = 0;
2416 if (++cidx == fl->cap)
2417 cidx = 0;
2418 nbuf++;
2419 }
2420 }
3359 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3360 }
3361
3362 if (is_t4(sc->params.chip))
3363 v |= V_PIDX(ndesc);
3364 else
3365 v |= V_PIDX_T5(ndesc);
3366 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3367
3368 membar_producer();
3369
3370 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3371
3372 /*
3373 * Update pending count:
3374 * Deduct the number of descriptors posted
3375 */
3376 fl->pending -= ndesc * 8;
3377 }
3378
3379 static void
3380 tx_reclaim_task(void *arg)
3381 {
3382 struct sge_txq *txq = arg;
3383
3384 TXQ_LOCK(txq);
3385 reclaim_tx_descs(txq, txq->eq.qsize);
3386 TXQ_UNLOCK(txq);
3387 }
3388
3389 /* ARGSUSED */
3390 static int
3391 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3392 mblk_t *m)
3393 {
3394 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3395 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3396 struct adapter *sc = iq->adapter;
3397 struct sge *s = &sc->sge;
3398 struct sge_eq *eq;
3399 struct sge_txq *txq;
3400
3401 txq = (void *)s->eqmap[qid - s->eq_start];
3402 eq = &txq->eq;
3403 txq->qflush++;
3404 t4_mac_tx_update(txq->port, txq);
3405
3406 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3407 (void *)txq, DDI_NOSLEEP);
3408
3409 return (0);
3410 }
3411
3412 static int
3413 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3414 {
3415 struct adapter *sc = iq->adapter;
3416 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3417
3418 ASSERT(m == NULL);
3419
3420 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3421 const struct rss_header *rss2;
3422
3423 rss2 = (const struct rss_header *)&cpl->data[0];
3424 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3425 }
3426 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3427 }
3428
|