/* * ng_source.c */ /*- * Copyright 2002 Sandvine Inc. * All rights reserved. * * Subject to the following obligations and disclaimer of warranty, use and * redistribution of this software, in source or object code forms, with or * without modifications are expressly permitted by Sandvine Inc.; provided, * however, that: * 1. Any and all reproductions of the source or object code must include the * copyright notice above and the following disclaimer of warranties; and * 2. No rights are granted, in any manner or form, to use Sandvine Inc. * trademarks, including the mark "SANDVINE" on advertising, endorsements, * or otherwise except as such appears in the above copyright notice or in * the software. * * THIS SOFTWARE IS BEING PROVIDED BY SANDVINE "AS IS", AND TO THE MAXIMUM * EXTENT PERMITTED BY LAW, SANDVINE MAKES NO REPRESENTATIONS OR WARRANTIES, * EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, INCLUDING WITHOUT LIMITATION, * ANY AND ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR * PURPOSE, OR NON-INFRINGEMENT. SANDVINE DOES NOT WARRANT, GUARANTEE, OR * MAKE ANY REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE * USE OF THIS SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY * OR OTHERWISE. IN NO EVENT SHALL SANDVINE BE LIABLE FOR ANY DAMAGES * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF SANDVINE IS ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. * * Author: Dave Chapeskie */ #include __FBSDID("$FreeBSD$"); /* * This node is used for high speed packet geneneration. It queues * all data recieved on it's 'input' hook and when told to start via * a control message it sends the packets out it's 'output' hook. In * this way this node can be preloaded with a packet stream which is * continuously sent. * * Currently it just copies the mbufs as required. It could do various * tricks to try and avoid this. Probably the best performance would * be achieved by modifying the appropriate drivers to be told to * self-re-enqueue packets (e.g. the if_bge driver could reuse the same * transmit descriptors) under control of this node; perhaps via some * flag in the mbuf or some such. The node would peak at an appropriate * ifnet flag to see if such support is available for the connected * interface. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define NG_SOURCE_INTR_TICKS 1 #define NG_SOURCE_DRIVER_IFQ_MAXLEN (4*1024) #define OFFSETOF(s, e) ((char *)&((s *)0)->e - (char *)((s *)0)) #define SIZEOFE(s, e) (sizeof(((s *)0)->e)) #define ENDOF(s, e) (OFFSETOF(s,e)+SIZEOFE(s,e)) #define IPSTART(aux) ((aux)->ipoffset) #define IPSUMOFF(aux) (IPSTART(aux)+OFFSETOF(struct ip,ip_sum)) #define IPSRCOFF(aux) (IPSTART(aux)+OFFSETOF(struct ip,ip_src)) #define IPDSTEND(aux) (IPSTART(aux)+ENDOF(struct ip,ip_dst)) #define mtod_off(m,off,t) ((t)(mtod((m),caddr_t)+(off))) /* Per hook info */ struct source_hookinfo { hook_p hook; }; /* Per node info */ struct privdata { node_p node; struct source_hookinfo input; struct source_hookinfo output; struct ng_source_stats stats; struct ifqueue snd_queue; /* packets to send */ /* The last_packet is zeroed in the constructor and is later set to * the last packet that is passed in for sending. This pointer is * used by the counter logic to determine whether or not the entire * list has been sent so that counters with the * NGM_SOURCE_INCREMENT_CNT_PER_LIST flag set are incremented after * the last packet has been sent */ struct mbuf *last_packet; struct ifnet *output_ifp; struct callout intr_ch; u_int64_t packets; /* packets to send */ u_int32_t queueOctets; u_int32_t octetsPerSec; u_int32_t maxOctetsPerTick; /* credit is extra amount for next tick, likely negative */ int32_t credit; /* lastIntTime is only used or updated for rate limiting */ struct timeval lastIntTime; struct ng_source_embed_cnt_info embed_counter[NG_SOURCE_COUNTERS]; struct ng_source_embed_info embed_chksum; struct ng_source_embed_info embed_timestamp; }; typedef struct privdata *sc_p; /* Node flags */ #define NG_SOURCE_ACTIVE (NGF_TYPE1) #define NG_SOURCE_SHUTDOWN_NOINPUT (NGF_TYPE2) /* * m_pkthdr.aux data. This stores per-packet information * when we first enqueue the packet so that we can avoid * scanning the packet everytime we need to modify and send it. * * If we modify the packet within [0,ip_end-1] we may need to modify * the IP header checksum (at [10], ip->p_sum). * If [9] (ip->ip_p) is modified we ignore UDP and TCP portions. * If [12,19] (ip->ip_src, ip->ip_dst) is modified we may need to * modify the UDP or TCP checksum (at udp_checksum_off or tcp_checksum_off). * If we modify [ip_end,upd_end-1] then the udp checksum may need to be * modified. * If we modify >=ip_end then the tcp checksum may need to be modified. * The chksum field is over the whole packet and updated with any * modification. When embedding this into the packet we need to take into * account the value that was at the location before embedding. */ struct ngsrc_tag { struct m_tag tag; int ip_end; /* end of IP header */ int udp_end; /* end of UDP datagram */ int udp_chksum_off; /* offset of UDP checksum field */ int tcp_chksum_off; /* offset of TCP checksum field */ u_int16_t chksum; /* packet checksum */ u_int16_t ipoffset; /* ip offset */ }; /* XXX hack to enable KASSERT localally */ #if 1 #undef KASSERT #define KASSERT(expr,msg) do { \ if (!(expr)) { \ printf msg ; \ panic msg; \ } \ } while(0) #endif /* Netgraph methods */ static ng_constructor_t ng_source_constructor; static ng_rcvmsg_t ng_source_rcvmsg; static ng_shutdown_t ng_source_rmnode; static ng_newhook_t ng_source_newhook; static ng_rcvdata_t ng_source_rcvdata; static ng_disconnect_t ng_source_disconnect; /* Other functions */ static void ng_source_intr(node_p, hook_p, void *, int); static int ng_source_request_output_ifp (sc_p); static void ng_source_clr_data (sc_p); static void ng_source_start (node_p); static void ng_source_set_active (node_p); static void ng_source_stop (sc_p); static int ng_source_send (sc_p, int, u_int32_t, int *, int *); static int ng_source_store_output_ifp(sc_p sc, struct ng_mesg *msg); static struct mbuf * ng_source_send_enqueue (sc_p, struct mbuf *); static u_int16_t new_chksum (u_int16_t, u_int16_t, u_int16_t); static void mbuf_chksum_update (struct mbuf*, u_int, caddr_t, u_int, u_int); static void ng_source_packet_mod (sc_p, const struct ngsrc_tag *, struct mbuf *, int, int, caddr_t, int); static int ng_source_dup_mod (sc_p, struct mbuf *, struct mbuf **); /* Parse type for timeval */ static const struct ng_parse_struct_field ng_source_timeval_type_fields[] = { { "tv_sec", &ng_parse_int32_type }, { "tv_usec", &ng_parse_int32_type }, { NULL } }; const struct ng_parse_type ng_source_timeval_type = { &ng_parse_struct_type, &ng_source_timeval_type_fields }; /* Parse type for struct ng_source_stats */ static const struct ng_parse_struct_field ng_source_stats_type_fields[] = NG_SOURCE_STATS_TYPE_INFO; static const struct ng_parse_type ng_source_stats_type = { &ng_parse_struct_type, &ng_source_stats_type_fields }; /* Parse type for struct ng_source_embed_info */ static const struct ng_parse_struct_field ng_source_embed_type_fields[] = NG_SOURCE_EMBED_TYPE_INFO; static const struct ng_parse_type ng_source_embed_type = { &ng_parse_struct_type, &ng_source_embed_type_fields }; /* Parse type for struct ng_source_embed_cnt_info */ static const struct ng_parse_struct_field ng_source_embed_cnt_type_fields[] = NG_SOURCE_EMBED_CNT_TYPE_INFO; static const struct ng_parse_type ng_source_embed_cnt_type = { &ng_parse_struct_type, &ng_source_embed_cnt_type_fields }; /* List of commands and how to convert arguments to/from ASCII */ static const struct ng_cmdlist ng_source_cmds[] = { { NGM_SOURCE_COOKIE, NGM_SOURCE_GET_STATS, "getstats", NULL, &ng_source_stats_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_CLR_STATS, "clrstats", NULL, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_GETCLR_STATS, "getclrstats", NULL, &ng_source_stats_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_START, "start", &ng_parse_uint64_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_STOP, "stop", NULL, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_CLR_DATA, "clrdata", NULL, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_SHUTDOWN_MODE, "shutdownmode", &ng_parse_uint32_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_SET_MAX_OCTET_RATE, "setmaxoctetrate", &ng_parse_uint32_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_GET_MAX_OCTET_RATE, "getmaxoctetrate", NULL, &ng_parse_uint32_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_SET_COUNTER, "setcounter", &ng_source_embed_cnt_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_GET_COUNTER, "getcounter", &ng_parse_uint8_type, &ng_source_embed_cnt_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_SET_CHKSUM, "setchksum", &ng_source_embed_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_GET_CHKSUM, "getchksum", NULL, &ng_source_embed_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_SET_TIMESTAMP, "settimestamp", &ng_source_embed_type, NULL }, { NGM_SOURCE_COOKIE, NGM_SOURCE_GET_TIMESTAMP, "gettimestamp", NULL, &ng_source_embed_type }, { NGM_SOURCE_COOKIE, NGM_SOURCE_START_NOW, "start_now", &ng_parse_uint64_type, NULL }, { 0 } }; /* Netgraph type descriptor */ static struct ng_type ng_source_typestruct = { .version = NG_ABI_VERSION, .name = NG_SOURCE_NODE_TYPE, .constructor = ng_source_constructor, .rcvmsg = ng_source_rcvmsg, .shutdown = ng_source_rmnode, .newhook = ng_source_newhook, .rcvdata = ng_source_rcvdata, .disconnect = ng_source_disconnect, .cmdlist = ng_source_cmds, }; NETGRAPH_INIT(source, &ng_source_typestruct); static int ng_source_set_autosrc(sc_p, u_int32_t); static u_int us_per_tick; /* * Used to keep track of ng_sources' total mbuf usage, which is the total * mbufs used by all ng_source nodes. */ int ng_source_mbuf_usage = 0; /* * Node constructor */ static int ng_source_constructor(node_p node) { sc_p sc; int i; sc = malloc(sizeof(*sc), M_NETGRAPH, M_NOWAIT | M_ZERO); if (sc == NULL) return (ENOMEM); NG_NODE_SET_PRIVATE(node, sc); sc->node = node; /* XXX: This needs to be set more intelligently */ sc->snd_queue.ifq_maxlen = 2048; ng_callout_init(&sc->intr_ch); us_per_tick = NG_SOURCE_INTR_TICKS * 1000000 / hz; /* User-specified peak rate */ sc->maxOctetsPerTick = 0; /* by default no limiting */ sc->octetsPerSec = 0; sc->credit = 0; /* Embedded counter defaults */ for (i = 0; i < NG_SOURCE_COUNTERS; ++ i) { sc->embed_counter[i].index = i; sc->embed_counter[i].width = 4; sc->embed_counter[i].increment = 1; sc->embed_counter[i].max_val = 0xffffffff; } return (0); } /* * Add a hook */ static int ng_source_newhook(node_p node, hook_p hook, const char *name) { sc_p sc; sc = NG_NODE_PRIVATE(node); KASSERT(sc != NULL, ("%s: null node private", __func__)); if (strcmp(name, NG_SOURCE_HOOK_INPUT) == 0) { sc->input.hook = hook; NG_HOOK_SET_PRIVATE(hook, &sc->input); } else if (strcmp(name, NG_SOURCE_HOOK_OUTPUT) == 0) { sc->output.hook = hook; NG_HOOK_SET_PRIVATE(hook, &sc->output); sc->output_ifp = 0; bzero(&sc->stats, sizeof(sc->stats)); sc->stats.minEnqueuedPackets = 1000000; /* extreme value */ } else return (EINVAL); return (0); } /* * Receive a control message */ static int ng_source_rcvmsg(node_p node, item_p item, hook_p lasthook) { sc_p sc; struct ng_mesg *resp = NULL; int error = 0; struct ng_mesg *msg; sc = NG_NODE_PRIVATE(node); NGI_GET_MSG(item, msg); KASSERT(sc != NULL, ("%s: null node private", __func__)); switch (msg->header.typecookie) { case NGM_SOURCE_COOKIE: if (msg->header.flags & NGF_RESP) { error = EINVAL; break; } switch (msg->header.cmd) { case NGM_SOURCE_GET_STATS: case NGM_SOURCE_CLR_STATS: case NGM_SOURCE_GETCLR_STATS: { struct ng_source_stats *stats; if (msg->header.cmd != NGM_SOURCE_CLR_STATS) { NG_MKRESPONSE(resp, msg, sizeof(*stats), M_NOWAIT); if (resp == NULL) { error = ENOMEM; goto done; } sc->stats.queueOctets = sc->queueOctets; sc->stats.queueFrames = sc->snd_queue.ifq_len; if ((sc->node->nd_flags & NG_SOURCE_ACTIVE) && !timevalisset(&sc->stats.endTime)) { getmicrotime(&sc->stats.elapsedTime); timevalsub(&sc->stats.elapsedTime, &sc->stats.startTime); } stats = (struct ng_source_stats *)resp->data; bcopy(&sc->stats, stats, sizeof(* stats)); } if (msg->header.cmd != NGM_SOURCE_GET_STATS) bzero(&sc->stats, sizeof(sc->stats)); } break; case NGM_SOURCE_START: { u_int64_t packets = *(u_int64_t *)msg->data; if (sc->output.hook == NULL) { printf("%s: start on node with no output hook\n" , __func__); error = EINVAL; break; } if (packets == 0) { printf("%s: start with zero packet count\n", __func__); error = EINVAL; break; } if (sc->snd_queue.ifq_len == 0) { printf("%s: start with no queued packets\n", __func__); error = EINVAL; break; } /* * TODO depending on packets and snd_queue.ifq_len * we should duplicate the send queue to avoid * individual packet reference counts from getting * too high. */ sc->packets = packets; ng_source_start(node); } break; case NGM_SOURCE_START_NOW: { u_int64_t packets = *(u_int64_t *)msg->data; if (sc->output.hook == NULL) { printf("%s: start on node with no output hook\n" , __func__); error = EINVAL; break; } if (sc->node->nd_flags & NG_SOURCE_ACTIVE) { error = EBUSY; break; } /* TODO validation of packets */ sc->packets = packets; sc->output_ifp = NULL; sc->node->nd_flags |= NG_SOURCE_ACTIVE; timevalclear(&sc->stats.elapsedTime); timevalclear(&sc->stats.endTime); getmicrotime(&sc->stats.startTime); ng_callout(&sc->intr_ch, node, NULL, 0, ng_source_intr, sc, 0); } break; case NGM_SOURCE_STOP: ng_source_stop(sc); break; case NGM_SOURCE_CLR_DATA: ng_source_clr_data(sc); break; case NGM_SOURCE_SHUTDOWN_MODE: { u_int32_t old_mode = !!(sc->node->nd_flags & NG_SOURCE_SHUTDOWN_NOINPUT); u_int32_t new_mode = *(u_int32_t *)msg->data; if (new_mode && !old_mode) { sc->node->nd_flags |= NG_SOURCE_SHUTDOWN_NOINPUT; if (sc->input.hook == NULL) ng_rmnode_self(node); } else if (!new_mode && old_mode) { sc->node->nd_flags &= ~NG_SOURCE_SHUTDOWN_NOINPUT; } } break; case NGM_SOURCE_SET_MAX_OCTET_RATE: { int octetsPerSec = *(u_int32_t *)msg->data; sc->octetsPerSec = octetsPerSec; sc->maxOctetsPerTick = NG_SOURCE_INTR_TICKS * octetsPerSec / hz; if (octetsPerSec != 0 && sc->maxOctetsPerTick == 0) sc->maxOctetsPerTick = 1; sc->credit = 0; #ifdef DEBUG_NG_SOURCE printf("Octets per tick: %d*%d/%d = %u\n", NG_SOURCE_INTR_TICKS, octetsPerSec, hz, sc->maxOctetsPerTick ); #endif } break; case NGM_SOURCE_GET_MAX_OCTET_RATE: { NG_MKRESPONSE(resp, msg, sizeof(u_int32_t), M_DONTWAIT); if (resp == NULL) { error = ENOMEM; goto done; } *(u_int32_t *)resp->data = sc->octetsPerSec; } break; case NGM_SOURCE_SET_COUNTER: { struct ng_source_embed_cnt_info *embed; embed = (struct ng_source_embed_cnt_info *)msg->data; #ifdef DEBUG_NG_SOURCE printf ("cnt.index = %d\n", embed->index); printf ("cnt.flags = 0x%b\n", embed->flags, NGM_SOURCE_FLAGS_PRINT); printf ("cnt.offset = 0x%04x = %d\n", embed->offset, embed->offset); printf ("cnt.width = %d\n", embed->width); printf ("cnt.next_val = %d\n", embed->next_val); printf ("cnt.min_val = %d\n", embed->min_val); printf ("cnt.max_val = %d\n", embed->max_val); printf ("cnt.increment = %d\n", embed->increment); #endif bcopy(embed, &sc->embed_counter[embed->index], sizeof(*embed)); } break; case NGM_SOURCE_GET_COUNTER: { u_int8_t index = *(u_int8_t *)msg->data; struct ng_source_embed_cnt_info *embed; NG_MKRESPONSE(resp, msg, sizeof(*embed), M_DONTWAIT); if (resp == NULL) { error = ENOMEM; goto done; } embed = (struct ng_source_embed_cnt_info *)resp->data; bcopy(&sc->embed_counter[index], embed, sizeof(*embed)); } break; case NGM_SOURCE_SET_CHKSUM: { struct ng_source_embed_info *embed; embed = (struct ng_source_embed_info *)msg->data; bcopy(embed, &sc->embed_chksum, sizeof(*embed)); } break; case NGM_SOURCE_GET_CHKSUM: { struct ng_source_embed_info *embed; NG_MKRESPONSE(resp, msg, sizeof(*embed), M_DONTWAIT); if (resp == NULL) { error = ENOMEM; goto done; } embed = (struct ng_source_embed_info *)resp->data; bcopy(&sc->embed_chksum, embed, sizeof(*embed)); } break; case NGM_SOURCE_SET_TIMESTAMP: { struct ng_source_embed_info *embed; embed = (struct ng_source_embed_info *)msg->data; bcopy(embed, &sc->embed_timestamp, sizeof(*embed)); } break; case NGM_SOURCE_GET_TIMESTAMP: { struct ng_source_embed_info *embed; NG_MKRESPONSE(resp, msg, sizeof(*embed), M_DONTWAIT); if (resp == NULL) { error = ENOMEM; goto done; } embed = (struct ng_source_embed_info *)resp->data; bcopy(&sc->embed_timestamp, embed, sizeof(*embed)); } break; default: error = EINVAL; break; } break; case NGM_ETHER_COOKIE: if (!(msg->header.flags & NGF_RESP)) { error = EINVAL; break; } switch (msg->header.cmd) { /* * Got ether ifp back from output hook. Store it away and * actually start sending. */ case NGM_ETHER_GET_IFINDEX: if (ng_source_store_output_ifp(sc, msg) == 0) { ng_source_set_active(node); } break; default: error = EINVAL; } break; default: error = EINVAL; break; } done: /* Take care of synchronous response, if any */ NG_RESPOND_MSG(error, node, item, resp); /* Free the message and return */ NG_FREE_MSG(msg); return (error); } /* * Receive data on a hook * * If data comes in the input hook, enqueue it on the send queue. * If data comes in the output hook, discard it. */ static int ng_source_rcvdata(hook_p hook, item_p item) { sc_p sc; struct source_hookinfo *hinfo; int error = 0; struct mbuf *m; sc = NG_NODE_PRIVATE(NG_HOOK_NODE(hook)); NGI_GET_M(item, m); NG_FREE_ITEM(item); hinfo = NG_HOOK_PRIVATE(hook); struct mbuf *mbuf_counter = m; int num_new_mbufs = 0; KASSERT(sc != NULL, ("%s: null node private", __func__)); KASSERT(hinfo != NULL, ("%s: null hook info", __func__)); while (mbuf_counter != NULL) { ++num_new_mbufs; mbuf_counter = mbuf_counter->m_next; } /* Which hook? */ if (hinfo == &sc->output) { /* discard */ NG_FREE_M(m); return (error); } KASSERT(hinfo == &sc->input, ("%s: no hook!", __func__)); if ((m->m_flags & M_PKTHDR) == 0) { printf("%s: mbuf without PKTHDR\n", __func__); NG_FREE_M(m); return (EINVAL); } if (sc->node->nd_flags & NG_SOURCE_ACTIVE) { printf("%s: data recieved while active\n", __func__); NG_FREE_M(m); return (EBUSY); } /* Check ifq_maxlen here */ if (num_new_mbufs + sc->snd_queue.ifq_len > sc->snd_queue.ifq_maxlen) { NG_FREE_M(m); return(ENOBUFS); } /* enque packet */ ng_source_send_enqueue (sc, m); ng_source_mbuf_usage += num_new_mbufs; return (0); } /* * Shutdown processing */ static int ng_source_rmnode(node_p node) { sc_p sc; sc = NG_NODE_PRIVATE(node); KASSERT(sc != NULL, ("%s: null node private", __func__)); ng_source_stop(sc); ng_source_clr_data(sc); NG_NODE_SET_PRIVATE(node, NULL); NG_NODE_UNREF(node); FREE(sc, M_NETGRAPH); return (0); } /* * Hook disconnection */ static int ng_source_disconnect(hook_p hook) { struct source_hookinfo *hinfo; sc_p sc; hinfo = NG_HOOK_PRIVATE(hook); sc = NG_NODE_PRIVATE(NG_HOOK_NODE(hook)); KASSERT(sc != NULL, ("%s: null node private", __func__)); hinfo->hook = NULL; if (NG_NODE_NUMHOOKS(NG_HOOK_NODE(hook)) == 0 || hinfo == &sc->output || (sc->node->nd_flags & NG_SOURCE_SHUTDOWN_NOINPUT)) ng_rmnode_self(NG_HOOK_NODE(hook)); return (0); } /* * * Ask out neighbour on the output hook side to send us it's interface * information. */ static int ng_source_request_output_ifp(sc_p sc) { struct ng_mesg *msg; int error = 0; sc->output_ifp = NULL; /* Ask the attached node for the connected interface's index */ NG_MKMESSAGE(msg, NGM_ETHER_COOKIE, NGM_ETHER_GET_IFINDEX, 0, M_NOWAIT); if (msg == NULL) return (ENOBUFS); NG_SEND_MSG_HOOK(error, sc->node, msg, sc->output.hook, 0); return (error); } /* * Set sc->output_ifp to point to the the struct ifnet of the interface * reached via our output hook. */ static int ng_source_store_output_ifp(sc_p sc, struct ng_mesg *msg) { struct ifnet *ifp; u_int32_t if_index; if (msg->header.arglen < sizeof(u_int32_t)) return (EINVAL); if_index = *(u_int32_t *)msg->data; /* Could use ifindex2ifnet[if_index] except that we have no * way of verifying if_index is valid since if_indexlim is * local to if_attach() */ IFNET_RLOCK(); TAILQ_FOREACH(ifp, &ifnet, if_link) { if (ifp->if_index == if_index) break; } IFNET_RUNLOCK(); if (ifp == NULL) { printf("%s: can't find interface %d\n", __func__, if_index); return (EINVAL); } sc->output_ifp = ifp; #if 1 /* XXX mucking with a drivers ifqueue size is ugly but we need it * to queue a lot of packets to get close to line rate on a gigabit * interface with small packets. * XXX we should restore the original value at stop or disconnect */ /* XXX locking? splimp() removed. */ if (ifp->if_snd.ifq_maxlen < NG_SOURCE_DRIVER_IFQ_MAXLEN) { printf("ng_source: changing ifq_maxlen from %d to %d\n", ifp->if_snd.ifq_maxlen, NG_SOURCE_DRIVER_IFQ_MAXLEN); ifp->if_snd.ifq_maxlen = NG_SOURCE_DRIVER_IFQ_MAXLEN; } #endif return (0); } /* * Set the attached ethernet node's ethernet source address override flag. */ static int ng_source_set_autosrc(sc_p sc, u_int32_t flag) { struct ng_mesg *msg; int error = 0; NG_MKMESSAGE(msg, NGM_ETHER_COOKIE, NGM_ETHER_SET_AUTOSRC, sizeof (u_int32_t), M_NOWAIT); if (msg == NULL) return(ENOBUFS); *(u_int32_t *)msg->data = flag; NG_SEND_MSG_HOOK(error, sc->node, msg, sc->output.hook, 0); return (error); } /* * Clear out the data we've queued */ static void ng_source_clr_data (sc_p sc) { struct mbuf *m; struct mbuf *mbuf_counter; for (;;) { int num_old_mbufs = 0; _IF_DEQUEUE(&sc->snd_queue, m); if (m == NULL) break; mbuf_counter = m; while(mbuf_counter != NULL) { ++num_old_mbufs; mbuf_counter = mbuf_counter->m_next; } NG_FREE_M(m); ng_source_mbuf_usage -= num_old_mbufs; } sc->queueOctets = 0; sc->last_packet = 0; } /* * Start sending queued data out the output hook */ static void ng_source_start (node_p node) { sc_p sc = NG_NODE_PRIVATE(node); KASSERT(sc->output.hook != NULL, ("%s: output hook unconnected", __func__)); #ifdef DEBUG_NG_SOURCE printf("%s: octets per sec: %u, total packets: %llu\n", __FUNCTION__, sc->octetsPerSec, sc->packets); #endif /* * If NGM_SOURCE_START msg recieved again, after we've gotten * output_ifp, activate and start sending packets again. */ if ((sc->node->nd_flags & NG_SOURCE_ACTIVE) == 0) { if (sc->output_ifp == NULL) ng_source_request_output_ifp(sc); else ng_source_set_active(node); } } /* * reset stats, call ng_callout, and set NG_SOURCE_ACTIVE */ static void ng_source_set_active (node_p node) { sc_p sc = NG_NODE_PRIVATE(node); ng_source_set_autosrc(sc, 0); sc->node->nd_flags |= NG_SOURCE_ACTIVE; sc->stats.interrupts = 0; sc->stats.slowInterrupts = 0; timevalclear(&sc->stats.elapsedTime); timevalclear(&sc->stats.endTime); getmicrotime(&sc->stats.startTime); sc->lastIntTime = sc->stats.startTime; ng_callout(&sc->intr_ch, node, NULL, 0, ng_source_intr, sc, 0); } /* * Stop sending queued data out the output hook */ static void ng_source_stop (sc_p sc) { if (sc->node->nd_flags & NG_SOURCE_ACTIVE) { ng_uncallout(&sc->intr_ch, sc->node); sc->node->nd_flags &= ~NG_SOURCE_ACTIVE; getmicrotime(&sc->stats.endTime); sc->stats.elapsedTime = sc->stats.endTime; timevalsub(&sc->stats.elapsedTime, &sc->stats.startTime); /* XXX should set this to the initial value instead */ ng_source_set_autosrc(sc, 1); } #ifdef DEBUG_NG_SOURCE printf("%llu total interrupts, %llu slow interrupts\n", sc->stats.interrupts, sc->stats.slowInterrupts ); #endif } /* * While active called every NG_SOURCE_INTR_TICKS ticks. * Sends as many packets as the interface connected to our * output hook is able to enqueue. */ static void ng_source_intr(node_p node, hook_p hook, void *arg1, int arg2) { const sc_p sc = (sc_p) arg1; struct ifaltq *ifq; int packets; int32_t maxOctets; KASSERT(sc != NULL, ("%s: null node private", __func__)); if (sc->packets == 0 || sc->output.hook == NULL || (sc->node->nd_flags & NG_SOURCE_ACTIVE) == 0) { ng_source_stop(sc); return; } sc->stats.interrupts++; if( sc->octetsPerSec == 0 ) { /* no-limit case */ maxOctets = 0; } else { struct timeval now, elapsed; int64_t usec_passed; getmicrotime(&now); elapsed = now; /* lastIntTime will be the last time we sent anything */ timevalsub(&elapsed, &sc->lastIntTime); /* do correct calculation even if rate is very slow */ usec_passed = elapsed.tv_sec * 1000000 + elapsed.tv_usec; /* This *could* roll over if usec_passed was large and the * rate were large. This is unlikely unless the box is so * busy that a large number of ticks have passed. */ maxOctets = (int64_t)sc->octetsPerSec * (int64_t)usec_passed / 1000000 + sc->credit; /* maybe negative */ /* sc->credit may be negative, meaning we are in debt. * Re-schedule timer for later when we will recalculate. */ if ( maxOctets <= 0 ) { /* Could reschedule several ticks in the future * depending on rate, But in this case the box isn't * very busy anyhow. */ ng_callout(&sc->intr_ch, node, NULL, NG_SOURCE_INTR_TICKS, ng_source_intr, sc, 0); return; } /* if rate is very high, box may not be able to keep up. * Put a 2-tick limit on this case. * not the small case, since > 1 pkt/tick */ if( sc->maxOctetsPerTick > 10000 && maxOctets > 2 * sc->maxOctetsPerTick ) { maxOctets = 2 * sc->maxOctetsPerTick; sc->stats.slowInterrupts++; } /* LastIntTime is only updated if maxOctets > 0. * Otherwise, it will have rounded-down to zero and we will * re-calculate next time. */ sc->lastIntTime = now; } if (sc->output_ifp != NULL) { /* if_snd changed ifqueue -> if_altq */ ifq = &sc->output_ifp->if_snd; packets = ifq->ifq_maxlen - ifq->ifq_len; } else packets = sc->snd_queue.ifq_len; ng_source_send(sc, packets, maxOctets, &packets, &sc->credit); if (sc->packets == 0) ng_source_stop(sc); else ng_callout(&sc->intr_ch, node, NULL, NG_SOURCE_INTR_TICKS, ng_source_intr, sc, 0); if (packets < sc->stats.minEnqueuedPackets) sc->stats.minEnqueuedPackets = packets; if (packets > sc->stats.maxEnqueuedPackets) sc->stats.maxEnqueuedPackets = packets; } /* * Send packets out our output hook * If maxOctets=0, no limiting will occur. */ static int ng_source_send (sc_p sc, int maxPackets, u_int32_t maxOctets, int *sent_p, int *credit) { struct ifqueue tmp_queue; struct mbuf *m, *m2; int sent = 0; int error = 0; u_int32_t octets = 0; KASSERT(sc != NULL, ("%s: null node private", __func__)); KASSERT(maxPackets >= 0, ("%s: negative maxPackets param", __func__)); KASSERT(sc->node->nd_flags & NG_SOURCE_ACTIVE, ("%s: inactive node", __func__)); if ((u_int64_t)maxPackets > sc->packets) maxPackets = sc->packets; /* Copy the required number of packets to a temporary queue */ bzero (&tmp_queue, sizeof (tmp_queue)); for (sent = 0; error == 0 && sent < maxPackets && (maxOctets <= 0 || octets < maxOctets); ++sent) { _IF_DEQUEUE(&sc->snd_queue, m); if (m == NULL) break; /* duplicate and modify the packet */ error = ng_source_dup_mod (sc, m, &m2); if (error) { if (error == ENOBUFS) { _IF_PREPEND(&sc->snd_queue, m); } else { _IF_ENQUEUE(&sc->snd_queue, m); } break; } KASSERT (m2 != NULL, ("%s: m2 is NULL", __func__)); /* re-enqueue the original packet for us */ _IF_ENQUEUE(&sc->snd_queue, m); /* queue the copy for sending at smplimp */ _IF_ENQUEUE(&tmp_queue, m2); octets += m->m_pkthdr.len; } /* calculate the surplus. Usually negative unless there * was an error */ *credit = maxOctets - octets; /* * Now take the packets on the temporary queue and send * them out our output hook at splimp() and splnet(). */ error = 0; sent = 0; for (;;) { _IF_DEQUEUE(&tmp_queue, m2); if (m2 == NULL) break; if (error == 0) { ++sent; sc->stats.outFrames++; sc->stats.outOctets += m2->m_pkthdr.len; NG_SEND_DATA_ONLY(error, sc->output.hook, m2); } else { NG_FREE_M(m2); } } sc->packets -= sent; if (sent_p != NULL) *sent_p = sent; return (error); } /* * Incremental checksum update. Endian independant. * RFC 1624: HC' = ~(~HC + ~m + m') */ static __inline u_int16_t new_chksum (u_int16_t oldsum, u_int16_t oldval, u_int16_t newval) { u_int32_t newsum = (~oldsum&0xffff) + (~oldval&0xffff) + newval; newsum = (newsum & 0xffff) + (newsum>>16); newsum = (newsum & 0xffff) + (newsum>>16); KASSERT((newsum & 0xffff) == newsum, ("%s: newsum=0x%x", __func__, newsum)); newsum = ~newsum & 0xffff; return ((u_int16_t)newsum); } /* * Incremental checksum update. * RFC 1624, RFC 3022 4.2 */ static void mbuf_chksum_update (struct mbuf *m, u_int chksum_offset, caddr_t new_data, u_int offset, u_int len) { u_int32_t x; u_int16_t *oldp, *newp; u_int16_t old, new; if (len == 0) return; x = *mtod_off(m, chksum_offset, u_int16_t *); #ifdef DEBUG_NG_SOURCE_CHKSUM printf("%s: off=0x%02x, len=%d, data=<%*D>\n", __func__, offset, len, len, (u_char*)new_data, " "); printf("%s: old chksum=0x%04x\n", __func__, x&0xffff); #endif x = ~x & 0xffff; oldp = mtod_off(m, offset, u_int16_t*); newp = (u_int16_t*)new_data; if (offset & 1) { old = ntohs(*(u_int8_t *)oldp); new = ntohs(*(u_int8_t *)newp); #ifdef DEBUG_NG_SOURCE_CHKSUM printf("%s: leading byte, old=0x%x, new=0x%x\n", __func__, old, new); #endif x += (~old & 0xffff) + new; --len; oldp = (u_int16_t*)((caddr_t)oldp + 1); newp = (u_int16_t*)((caddr_t)newp + 1); #ifdef DEBUG_NG_SOURCE_CHKSUM printf("%s: oldp=%p, newp=%p\n", __func__, oldp, newp); #endif } for (; len >= 2; len -= 2) { x += (~(*oldp) & 0xffff) + *newp; ++oldp; ++newp; } if (len > 0) { old = ntohs(*(u_int8_t *)oldp << 8); new = ntohs(*(u_int8_t *)newp << 8); #ifdef DEBUG_NG_SOURCE_CHKSUM printf("%s: trailing byte, old=0x%x, new=0x%x\n", __func__, old, new); #endif x += (~old & 0xffff) + new; --len; } KASSERT(len == 0, ("%s: len!=0 (0x%x)", __func__, len)); x = (x & 0xffff) + (x >> 16); x = (x & 0xffff) + (x >> 16); KASSERT((x & 0xffff) == x, ("%s: x=0x%x", __func__, x)); #ifdef DEBUG_NG_SOURCE_CHKSUM printf("%s: new chksum=0x%04x\n", __func__, ~x&0xffff); #endif *mtod_off(m, chksum_offset, u_int16_t *) = ~x & 0xffff; } /* * Extract information from the packet and enqueue the * info+packet on the send queue. The info is recorded in an m_tag. * Note, aux mbufs are moved to the copy by m_copypacket() and m_dup() * and are not copied/duplicated. */ static struct mbuf * ng_source_send_enqueue (sc_p sc, struct mbuf *m) { struct mbuf *n; struct ngsrc_tag *aux; struct ether_header *eh; struct ip *ip = NULL; struct udphdr *udp = NULL; struct tcphdr *tcp = NULL; KASSERT (m->m_flags & M_PKTHDR, ("%s: packet without PKTHDR", __func__)); /* * For effieciency make sure the whole packet is available * in a single mbuf as one contiguous buffer. * Usually this will already be the case. */ if (m->m_pkthdr.len > m->m_len) { /* * Not in a single mbuf, we can make it so as a side * effect of m_dup() as long as it's <= MCLBYTES, * which should be the case as long as we're not * using jumbo frames. */ if (m->m_pkthdr.len > MCLBYTES) { printf ("%s: packet size %d > MCLBYTES\n", __func__, m->m_pkthdr.len); m_freem(m); return (NULL); } n = m_dup(m, M_DONTWAIT); m_freem(m); if (n == NULL) return (n); m = n; KASSERT(m->m_pkthdr.len <= m->m_len, ("%s: after m_dup pkthdr.len=%d, m_len=%d", __func__, m->m_pkthdr.len, m->m_len)); } /* * Get an m_tag to store some packet * specific info to help us out later. */ aux = (struct ngsrc_tag *)m_tag_alloc(NGM_SOURCE_COOKIE, NG_SOURCE_TAG, sizeof(struct ngsrc_tag) - sizeof(struct m_tag), M_NOWAIT); if (aux == NULL) { m_freem(m); return (NULL); } bzero((char*)aux + sizeof(struct m_tag), sizeof(struct ngsrc_tag) - sizeof(struct m_tag)); aux->udp_chksum_off = -1; aux->tcp_chksum_off = -1; aux->ipoffset = sizeof(struct ether_header); eh = mtod(m, struct ether_header *); if (m->m_len >= sizeof(*eh) && ntohs(eh->ether_type) == ETHERTYPE_IP) { if (m->m_len >= sizeof(*ip) + IPSTART(aux)) { ip = mtod_off(m, IPSTART(aux), struct ip *); aux->ip_end = IPSTART(aux) + (ip->ip_hl<<2); } } else if (m->m_len >= (sizeof(*eh)+4) && ntohs(eh->ether_type) == ETHERTYPE_VLAN) { aux->ipoffset = sizeof(struct ether_header) + 4; if (m->m_len >= sizeof(*ip) + IPSTART(aux)) { ip = mtod_off(m, IPSTART(aux), struct ip *); aux->ip_end = IPSTART(aux) + (ip->ip_hl<<2); } } /* * Pre-calculate packet checksum. Note the checksum isn't to the point * we can directly stuff it into the packet yet because the place * we're going to stick it hasn't been zeroed yet. We leave this to * be fixed up later so that the checksum offset can be changed on * the fly without making us re-calculate the whole checksum again. */ aux->chksum = in_cksum (m, m->m_pkthdr.len); if (ip != NULL) { /* Ignore IP header chksum */ aux->chksum = new_chksum (aux->chksum, ip->ip_sum, 0); switch (ip->ip_p) { case IPPROTO_UDP: if (m->m_len < sizeof(*udp) + aux->ip_end) break; udp = mtod_off(m, aux->ip_end, struct udphdr *); aux->udp_end = aux->ip_end + udp->uh_ulen; aux->udp_chksum_off = (caddr_t)&udp->uh_sum - mtod(m, caddr_t); /* Ignore UDP chksum */ aux->chksum = new_chksum (aux->chksum, udp->uh_sum, 0); break; case IPPROTO_TCP: if (m->m_len < sizeof(*tcp) + aux->ip_end) break; tcp = mtod_off(m, aux->ip_end, struct tcphdr *); aux->tcp_chksum_off = (caddr_t)&tcp->th_sum - mtod(m, caddr_t); /* Ignore TCP chksum */ aux->chksum = new_chksum (aux->chksum, tcp->th_sum, 0); break; } } m_tag_prepend(m, aux); /* XXX should we check IF_QFULL() ? */ _IF_ENQUEUE(&sc->snd_queue, m); sc->queueOctets += m->m_pkthdr.len; /* Store the mbuf pointer now so that we always have the * 'latest packet' that was added to the list. This is * required for counters that should only be updated after the * last packet has been sent */ sc->last_packet = m; return (m); } /* * Modify packet in 'm' by changing 'len' bytes starting at 'offset' * to data in 'cp'. Info in 'sc' and 'aux' is used to determine what * checksums need to be updated as well. * * The packet data in 'm' must be in a contiguous buffer in a single mbuf. * * XXX does not handle modifications that cross boundaries (e.g. if half * the change is in the IP header and half beyond the IP then chksum * will be wrong). If such a modification is detected it could split * the modification and recursively call itself for each part. */ static void ng_source_packet_mod (sc_p sc, const struct ngsrc_tag *aux, struct mbuf *m, int offset, int len, caddr_t cp, int flags) { KASSERT(sc != NULL, ("%s: sc is NULL", __func__)); KASSERT(aux != NULL, ("%s: aux is NULL", __func__)); KASSERT(m != NULL, ("%s: m is NULL", __func__)); KASSERT(cp != NULL, ("%s: cp is NULL", __func__)); KASSERT(offset >= 0, ("%s: bad offset %d", __func__, offset)); KASSERT(len >= 0, ("%s: bad len %d", __func__, len)); KASSERT(m->m_flags & M_PKTHDR, ("%s: m mbuf without PKTHDR", __func__)); KASSERT(m->m_pkthdr.len <= m->m_len, ("%s: pkthdr.len=%d, m_len=%d", __func__, m->m_pkthdr.len, m->m_len)); #ifdef DEBUG_NG_SOURCE_EMBED printf("ng_source_packet_mod flags=0x%b\n", flags, NGM_SOURCE_FLAGS_PRINT); printf("ng_source_packet_mod offset=0x%04x, len=%d, data=<%*D>\n", offset, len, len, (u_char *)cp, " "); #endif if (len == 0) return; /* Can't modify beyond end of packet... */ /* TODO should extend packet by adding 0 padding */ if (offset + len > m->m_len) return; /* If we're doing packet chksum... */ if (sc->embed_chksum.flags & NGM_SOURCE_EMBED_ENABLE) { u_int16_t *sum_p, oldval, newval; /* XXX assume IP,UDP & TCP checksum fields weren't changed */ oldval = *mtod_off(m, sc->embed_chksum.offset, u_int16_t *); mbuf_chksum_update (m, sc->embed_chksum.offset, cp, offset, len); newval = *mtod_off(m, sc->embed_chksum.offset, u_int16_t *); /* XXX assume sc->embed_chksum is in UDP/TCP payload */ if (sc->embed_chksum.flags & NGM_SOURCE_MODIFY_UDP_CHKSUM && aux->udp_chksum_off > 0) { sum_p = mtod_off(m, aux->udp_chksum_off, u_int16_t *); KASSERT((caddr_t)(sum_p) + sizeof(*sum_p) < m->m_data + m->m_len, ("%s: ptr past mbuf", __func__)); *sum_p = new_chksum (*sum_p, oldval, newval); } if (sc->embed_chksum.flags & NGM_SOURCE_MODIFY_TCP_CHKSUM && aux->tcp_chksum_off > 0) { sum_p = mtod_off(m, aux->tcp_chksum_off, u_int16_t *); KASSERT((caddr_t)(sum_p) + sizeof(*sum_p) < m->m_data + m->m_len, ("%s: ptr past mbuf", __func__)); *sum_p = new_chksum (*sum_p, oldval, newval); } /* * If the modification was only to get the embeded * checksum in then we're done. */ if (offset == sc->embed_chksum.offset && len == sizeof(u_int16_t)) return; } /* If we're doing IP chksum and the mod is in the IP header... */ if ((flags & NGM_SOURCE_MODIFY_IP_CHKSUM) && offset >= IPSTART(aux) && offset < aux->ip_end) { mbuf_chksum_update (m, IPSUMOFF(aux), cp, offset, len); } /* * If we're doing UDP chksum and pkt is UDP and * (mod is in UDP or mod is in IP pseudo hdr)... */ if ((flags & NGM_SOURCE_MODIFY_UDP_CHKSUM) && aux->udp_chksum_off >= 0 && ((offset >= aux->ip_end && offset < aux->udp_end) || (offset >= IPSRCOFF(aux) && offset < IPDSTEND(aux)))) { mbuf_chksum_update (m, aux->udp_chksum_off, cp, offset, len); } /* * If we're doing TCP chksum and pkt is TCP and * (mod is in TCP or mod is in IP pseudo hdr)... */ if ((flags & NGM_SOURCE_MODIFY_TCP_CHKSUM) && aux->tcp_chksum_off >= 0 && (offset >= aux->ip_end || (offset >= IPSRCOFF(aux) && offset < IPDSTEND(aux)))) { mbuf_chksum_update (m, aux->tcp_chksum_off, cp, offset, len); } bcopy (cp, mtod_off(m, offset, caddr_t), len); } static int ng_source_dup_mod (sc_p sc, struct mbuf *m0, struct mbuf **m_ptr) { struct mbuf *m; struct ng_source_embed_cnt_info *cnt; struct ng_source_embed_info *sum, *ts; const struct ngsrc_tag *aux; int error = 0; int modify; int i; KASSERT(sc != NULL, ("%s: sc is NULL", __func__)); KASSERT(m0 != NULL, ("%s: m0 is NULL", __func__)); KASSERT(m_ptr != NULL, ("%s: m_ptr is NULL", __func__)); /* Are we going to modify packets? */ modify = (sc->embed_timestamp.flags & NGM_SOURCE_EMBED_ENABLE || sc->embed_chksum.flags & NGM_SOURCE_EMBED_ENABLE); for (i = 0; !modify && i < NG_SOURCE_COUNTERS; ++i) modify = sc->embed_counter[i].flags & NGM_SOURCE_EMBED_ENABLE; /* duplicate the packet */ if (modify) m = m_dup(m0, M_DONTWAIT); else m = m_copypacket(m0, M_DONTWAIT); if (m == NULL) { error = ENOBUFS; goto done; } *m_ptr = m; if (!modify) goto done; /* modify the copied packet for sending */ KASSERT(M_WRITABLE(m), ("%s: packet not writable", __func__)); KASSERT(m->m_pkthdr.len <= m->m_len, ("%s: after m_dup pkthdr.len=%d, m_len=%d", __func__, m->m_pkthdr.len, m->m_len)); /* Note: aux is attached to m0, do NOT modify! */ aux = (struct ngsrc_tag *)m_tag_locate(m0, NGM_SOURCE_COOKIE, NG_SOURCE_TAG, NULL); KASSERT(aux != NULL, ("%s: no tag", __func__)); sum = &sc->embed_chksum; ts = &sc->embed_timestamp; if (sum->flags & NGM_SOURCE_EMBED_ENABLE) { /* recalculate with the chksum field as 0 and stuff it in */ u_int16_t zero = 0; ng_source_packet_mod (sc, aux, m, sum->offset, sizeof zero, (caddr_t)&zero, sum->flags); } for (i = 0; i < NG_SOURCE_COUNTERS; ++i) { cnt = &sc->embed_counter[i]; if (cnt->flags & NGM_SOURCE_EMBED_ENABLE) { caddr_t cp; u_int32_t val, oldVal = 0; /* Add the counter to the packet, or overwrite */ if (cnt->flags & NGM_SOURCE_ADD_COUNTER_TO_PKT) { switch (cnt->width) { case 1: oldVal = ntohs( *(mtod_off(m, cnt->offset, u_int8_t *)) << 8); break; case 2: oldVal = ntohs( *(mtod_off(m, cnt->offset, u_int16_t *))); break; case 4: oldVal = ntohl( *(mtod_off(m, cnt->offset, u_int32_t *))); break; } val = htonl(oldVal + cnt->next_val); } else { val = htonl(cnt->next_val); } cp = (caddr_t)&val; cp += sizeof(val) - cnt->width; ng_source_packet_mod (sc, aux, m, cnt->offset, cnt->width, cp, cnt->flags); /* Increment the counter every packet unless the * increment_cnt_per_list flag is set... in which case * increment the counter only after the last packet is * sent */ if ( (cnt->flags & NGM_SOURCE_INCREMENT_CNT_PER_LIST) == 0 || sc->last_packet == m0 ) { cnt->next_val += cnt->increment; /* XXX TODO, handle wraparound */ if (cnt->increment > 0 && cnt->next_val > cnt->max_val) cnt->next_val = cnt->min_val; else if (cnt->increment < 0 && cnt->next_val < cnt->min_val) cnt->next_val = cnt->max_val; } } } if (ts->flags & NGM_SOURCE_EMBED_ENABLE) { struct timeval now; getmicrotime(&now); now.tv_sec = htonl(now.tv_sec); now.tv_usec = htonl(now.tv_usec); ng_source_packet_mod (sc, aux, m, ts->offset, sizeof now, (caddr_t)&now, ts->flags); } done: return (error); }