jimzah/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2024-12-03 19:08:58 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Convert driver RX path over to using mbuf iovec

Browse Source
This commit is contained in:
Kip Macy 2007-04-08 15:04:19 +00:00
parent a8d9a363f5
commit c0a24dd4aa
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=168491
4 changed files with 107 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sys/dev/cxgb/cxgb_adapter.h
View File

@ -115,7 +115,7 @@ enum {
};
struct sge_lro_session {
struct t3_mbuf_hdr mh;
struct mbuf *m;
uint32_t seq;
uint16_t ip_len;
};
@ -154,7 +154,7 @@ struct sge_rspq {
uint32_t cntxt_id;
bus_dma_tag_t desc_tag;
bus_dmamap_t desc_map;
struct t3_mbuf_hdr mh;
struct mbuf *m;
struct mtx lock;
};
@ -403,7 +403,7 @@ int t3_encap(struct port_info *, struct mbuf **);
int t3_sge_init_sw(adapter_t *);
void t3_sge_deinit_sw(adapter_t *);
void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro);
void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad);
void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs);

123
sys/dev/cxgb/cxgb_lro.c
View File

@ -65,22 +65,17 @@ __FBSDID("$FreeBSD$");
#endif
#ifdef DEBUG
#define MBUF_HEADER_CHECK(mh) do { \
struct mbuf *head = mh->mh_head; \
struct mbuf *tail = mh->mh_tail; \
if (head->m_len == 0 || head->m_pkthdr.len == 0 \
|| (head->m_flags & M_PKTHDR) == 0) \
#define MBUF_HEADER_CHECK(m) do { \
if (m->m_len == 0 || m->m_pkthdr.len == 0 \
|| (m->m_flags & M_PKTHDR) == 0) \
panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \
head->m_len, head->m_pkthdr.len, head->m_flags); \
if ((head->m_flags & M_PKTHDR) == 0) \
m->m_len, m->m_pkthdr.len, m->m_flags); \
if (m->m_flags & M_PKTHDR) == 0) \
panic("first mbuf is not packet header - flags=0x%x\n", \
head->m_flags); \
if ((head == tail && head->m_len != head->m_pkthdr.len)) \
panic("len=%d pktlen=%d mismatch\n", \
head->m_len, head->m_pkthdr.len); \
if (head->m_len < ETHER_HDR_LEN || head->m_pkthdr.len < ETHER_HDR_LEN) \
m->m_flags); \
if (m->m_len < ETHER_HDR_LEN || m->m_pkthdr.len < ETHER_HDR_LEN) \
panic("packet too small len=%d pktlen=%d\n", \
head->m_len, head->m_pkthdr.len);\
m->m_len, m->m_pkthdr.len);\
} while (0)
#else
#define MBUF_HEADER_CHECK(m)
@ -100,7 +95,7 @@ static __inline int
lro_match_session(struct sge_lro_session *s,
struct ip *ih, struct tcphdr *th)
{
struct ip *sih = (struct ip *)(s->mh.mh_head->m_data + IPH_OFFSET);
struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET);
struct tcphdr *sth = (struct tcphdr *) (sih + 1);
/*
@ -121,7 +116,7 @@ lro_find_session(struct sge_lro *l, int idx, struct ip *ih, struct tcphdr *th)
while (active < l->num_active) {
s = lro_session(l, idx);
if (s->mh.mh_head) {
if (s->m) {
if (lro_match_session(s, ih, th)) {
l->last_s = s;
return s;
@ -179,39 +174,37 @@ can_lro_tcpsegment(struct tcphdr *th)
}
static __inline void
lro_new_session_init(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
lro_new_session_init(struct sge_lro_session *s, struct mbuf *m)
{
struct ip *ih = (struct ip *)(mh->mh_head->m_data + IPH_OFFSET);
struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET);
struct tcphdr *th = (struct tcphdr *) (ih + 1);
int ip_len = ntohs(ih->ip_len);
DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__,
s, mh->mh_head, mh->mh_tail);
DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
*&(s->mh) = *mh;
MBUF_HEADER_CHECK(mh);
s->m = m;
MBUF_HEADER_CHECK(m);
s->ip_len = ip_len;
s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2);
}
static void
lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m)
{
struct sge_lro *l = &qs->lro;
struct t3_mbuf_hdr *smh = &s->mh;
struct ip *ih = (struct ip *)(smh->mh_head->m_data + IPH_OFFSET);
struct mbuf *sm = s->m;
struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET);
DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__,
qs, s);
if (mh)
DPRINTF("mh->mh_head=%p, mh->mh_tail=%p)\n",
mh->mh_head, mh->mh_tail);
if (m)
DPRINTF("m=%p)\n", m);
else
DPRINTF("mh=NULL)\n");
DPRINTF("m=NULL)\n");
ih->ip_len = htons(s->ip_len);
ih->ip_sum = 0;
@ -219,15 +212,14 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf
MBUF_HEADER_CHECK(smh);
smh->mh_head->m_flags |= M_LRO;
t3_rx_eth(qs->port, &qs->rspq, smh->mh_head, 2);
sm->m_flags |= M_LRO;
t3_rx_eth(qs->port, &qs->rspq, sm, 2);
if (mh) {
*smh = *mh;
lro_new_session_init(s, mh);
if (m) {
s->m = m;
lro_new_session_init(s, m);
} else {
smh->mh_head = NULL;
smh->mh_tail = NULL;
s->m = NULL;
l->num_active--;
}
@ -235,23 +227,23 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf
}
static __inline struct sge_lro_session *
lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash)
lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
{
struct sge_lro *l = &qs->lro;
int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash);
struct sge_lro_session *s = lro_session(l, idx);
DPRINTF("%s(qs=%p, mh->mh_head=%p, mh->mh_tail=%p, rss_hash=0x%x)\n", __FUNCTION__,
qs, mh->mh_head, mh->mh_tail, rss_hash);
DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__,
qs, m, rss_hash);
if (__predict_true(!s->mh.mh_head))
if (__predict_true(!s->m))
goto done;
if (l->num_active > MAX_LRO_PER_QSET)
panic("MAX_LRO_PER_QSET exceeded");
if (l->num_active == MAX_LRO_PER_QSET) {
lro_flush_session(qs, s, mh);
lro_flush_session(qs, s, m);
qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
return s;
}
@ -259,24 +251,21 @@ lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash)
while (1) {
LRO_IDX_INC(idx);
s = lro_session(l, idx);
if (!s->mh.mh_head)
if (!s->m)
break;
}
done:
lro_new_session_init(s, mh);
lro_new_session_init(s, m);
l->num_active++;
return s;
return s;
}
static __inline int
lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
lro_update_session(struct sge_lro_session *s, struct mbuf *m)
{
struct mbuf *m = mh->mh_head;
struct t3_mbuf_hdr *smh = &s->mh;
struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(smh->mh_head->m_data + 2);
struct mbuf *sm = s->m;
struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2);
struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2);
struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET);
struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1);
@ -284,8 +273,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth);
DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__,
s, mh->mh_head, mh->mh_tail);
DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) {
return -1;
}
@ -295,7 +283,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
}
MBUF_HEADER_CHECK(smh);
th = (struct tcphdr *)(smh->mh_head->m_data + IPH_OFFSET + sizeof (struct ip));
th = (struct tcphdr *)(sm->m_data + IPH_OFFSET + sizeof (struct ip));
if (olen) {
uint32_t *ptr = (uint32_t *)(th + 1);
@ -315,7 +303,13 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
plen = ntohs(nih->ip_len) - tcpiphlen;
s->seq += plen;
s->ip_len += plen;
smh->mh_head->m_pkthdr.len += plen;
sm->m_pkthdr.len += plen;
/*
* XXX FIX ME
*
*
*/
#if 0
/* XXX this I *do not* understand */
@ -323,8 +317,8 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
skb_shinfo(s->skb)->gso_size = plen;
#endif
#if __FreeBSD_version > 700000
if (plen > smh->mh_head->m_pkthdr.tso_segsz)
smh->mh_head->m_pkthdr.tso_segsz = plen;
if (plen > sm->m_pkthdr.tso_segsz)
sm->m_pkthdr.tso_segsz = plen;
#endif
DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen));
m_adj(m, IPH_OFFSET + tcpiphlen);
@ -333,9 +327,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
skb_shinfo(s->skb)->frag_list = skb;
#endif
mh->mh_head->m_flags &= ~M_PKTHDR;
smh->mh_tail->m_next = mh->mh_head;
smh->mh_tail = mh->mh_tail;
#if 0
/*
@ -354,10 +346,9 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
}
void
t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro)
{
struct mbuf *m = mh->mh_head;
struct sge_qset *qs = rspq_to_qset(rq);
struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad);
struct ether_header *eh = (struct ether_header *)(cpl + 1);
@ -384,12 +375,12 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
if (__predict_false(!can_lro_tcpsegment(th))) {
goto no_lro;
} else if (__predict_false(!s)) {
s = lro_new_session(qs, mh, rss_hash);
s = lro_new_session(qs, m, rss_hash);
} else {
if (lro_update_session(s, mh)) {
lro_flush_session(qs, s, mh);
if (lro_update_session(s, m)) {
lro_flush_session(qs, s, m);
}
if (__predict_false(s->mh.mh_head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
lro_flush_session(qs, s, NULL);
}
}
@ -417,7 +408,7 @@ t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs)
s = lro_session(l, idx);
while (active < num_active) {
if (s->mh.mh_head) {
if (s->m) {
lro_flush_session(qs, s, NULL);
active++;
}

5
sys/dev/cxgb/cxgb_osdep.h
View File

@ -52,11 +52,6 @@ typedef struct adapter adapter_t;
struct sge_rspq;
struct t3_mbuf_hdr {
struct mbuf *mh_head;
struct mbuf *mh_tail;
};
#if __FreeBSD_version > 700030
#define INTR_FILTERS
#define FIRMWARE_LATEST

96
sys/dev/cxgb/cxgb_sge.c
View File

@ -67,6 +67,8 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/common/cxgb_t3_cpl.h>
#include <dev/cxgb/common/cxgb_firmware_exports.h>
#include <dev/cxgb/sys/mvec.h>
#define USE_GTS 0
#define SGE_RX_SM_BUF_SIZE 1536
@ -268,9 +270,8 @@ sgl_len(unsigned int n)
* Return a packet containing the immediate data of the given response.
*/
static __inline int
get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh)
get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl)
{
struct mbuf *m;
int len;
uint32_t flags = ntohl(resp->flags);
uint8_t sopeop = G_RSPD_SOP_EOP(flags);
@ -291,15 +292,14 @@ get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *m
switch (sopeop) {
case RSPQ_SOP_EOP:
mh->mh_head = mh->mh_tail = m;
m->m_pkthdr.len = len;
m->m_flags |= M_PKTHDR;
m->m_len = m->m_pkthdr.len = len;
m->m_flags |= M_PKTHDR;
memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE);
MH_ALIGN(m, IMMED_PKT_SIZE);
break;
case RSPQ_EOP:
m->m_flags &= ~M_PKTHDR;
mh->mh_head->m_pkthdr.len += len;
mh->mh_tail->m_next = m;
mh->mh_tail = m;
memcpy(cl, resp->imm_data, len);
m_iovappend(m, cl, MSIZE, len, 0);
break;
}
}
@ -1759,7 +1759,8 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad)
}
#endif
m->m_pkthdr.rcvif = ifp;
m_explode(m);
m_adj(m, sizeof(*cpl) + ethpad);
(*ifp->if_input)(ifp, m);
@ -1784,7 +1785,7 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad)
static int
get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m)
struct mbuf *m, struct rsp_desc *r)
{
unsigned int len_cq = ntohl(r->len_cq);
@ -1801,47 +1802,34 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(fl->entry_tag, sd->map);
m_cljset(m, sd->cl, fl->type);
m->m_len = len;
switch(sopeop) {
case RSPQ_SOP_EOP:
DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
mh->mh_head = mh->mh_tail = m;
m->m_pkthdr.len = len;
m_cljset(m, sd->cl, fl->type);
m->m_len = m->m_pkthdr.len = len;
m->m_flags |= M_PKTHDR;
ret = 1;
goto done;
break;
case RSPQ_NSOP_NEOP:
DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m));
m->m_flags &= ~M_PKTHDR;
if (mh->mh_tail == NULL) {
if (cxgb_debug)
printf("discarding intermediate descriptor entry\n");
m_freem(m);
break;
}
mh->mh_tail->m_next = m;
mh->mh_tail = m;
mh->mh_head->m_pkthdr.len += len;
ret = 0;
break;
case RSPQ_SOP:
DBG(DBG_RX, ("get_packet: SOP m %p\n", m));
m->m_pkthdr.len = len;
mh->mh_head = mh->mh_tail = m;
m->m_flags |= M_PKTHDR;
m_iovinit(m);
ret = 0;
break;
case RSPQ_EOP:
DBG(DBG_RX, ("get_packet: EOP m %p\n", m));
m->m_flags &= ~M_PKTHDR;
mh->mh_head->m_pkthdr.len += len;
mh->mh_tail->m_next = m;
mh->mh_tail = m;
ret = 1;
break;
}
m_iovappend(m, sd->cl, fl->buf_size, len, 0);
done:
if (++fl->cidx == fl->size)
fl->cidx = 0;
@ -1967,31 +1955,35 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
printf("async notification\n");
} else if (flags & F_RSPD_IMM_DATA_VALID) {
struct mbuf *m = NULL;
if (cxgb_debug)
printf("IMM DATA VALID\n");
if(get_imm_packet(adap, r, &rspq->mh) == 0) {
if (rspq->m == NULL)
rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
else
m = m_gethdr(M_NOWAIT, MT_DATA);
if (rspq->m == NULL || m == NULL) {
rspq->next_holdoff = NOMEM_INTR_DELAY;
budget_left--;
break;
} else {
eop = 1;
}
get_imm_packet(adap, r, rspq->m, m);
eop = 1;
rspq->imm_data++;
} else if (r->len_cq) {
int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
struct mbuf *m;
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL) {
log(LOG_WARNING, "failed to get mbuf for packet\n");
break;
if (rspq->m == NULL) {
rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
rspq->m->m_flags = 0;
} else if (rspq->m == NULL) {
log(LOG_WARNING, "failed to get mbuf for packet\n");
break;
}
ethpad = 2;
eop = get_packet(adap, drop_thresh, qs, &rspq->mh, r, m);
eop = get_packet(adap, drop_thresh, qs, rspq->m, r);
} else {
DPRINTF("pure response\n");
rspq->pure_rsps++;
@ -2016,14 +2008,14 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
}
if (eop) {
prefetch(rspq->mh.mh_head->m_data);
prefetch(rspq->mh.mh_head->m_data + L1_CACHE_BYTES);
prefetch(rspq->m->m_data);
prefetch(rspq->m->m_data + L1_CACHE_BYTES);
if (eth) {
t3_rx_eth_lro(adap, rspq, &rspq->mh, ethpad,
t3_rx_eth_lro(adap, rspq, rspq->m, ethpad,
rss_hash, rss_csum, lro);
rspq->mh.mh_tail = rspq->mh.mh_head = NULL;
rspq->m = NULL;
} else {
#ifdef notyet
if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT))
@ -2038,7 +2030,6 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
__refill_fl(adap, &qs->fl[0]);
__refill_fl(adap, &qs->fl[1]);
#endif
}
--budget_left;
}
@ -2158,11 +2149,18 @@ t3_intr_msix(void *data)
mtx_unlock(&rspq->lock);
}
/*
* broken by recent mbuf changes
*/
static int
t3_lro_enable(SYSCTL_HANDLER_ARGS)
{
adapter_t *sc;
int i, j, enabled, err, nqsets = 0;
#ifndef LRO_WORKING
return (0);
#endif
sc = arg1;
enabled = sc->sge.qs[0].lro.enabled;