## UPGW关键log设计 ### rte_log基本流程和功能验证 #### 初始化流程 DPDK代码中广泛使用RTE_INIT宏进行设备驱动或者RTE模块等的初始化工作，其核心是RTE_INIT_PRIO宏，定义在文件rte_common.h中。如下可见，RTE_INIT_PRIO宏的实现，实际为附带GCC编译属性的函数定义。此处用到两个属性，constructor和used。其中后一个used比较简单，向GCC编译器表明此函数的有用性，即使函数没有被任何地方引用，也不要警告。constructor构造属性，类似C++的构造函数，其标注的函数将在主函数（main）之前执行，相关代码位于目标文件的.ctors区（section）。 ``` #define RTE_INIT_PRIO(func, prio) \ static void __attribute__((constructor(RTE_PRIO(prio)), used)) func(void) ``` 另外RTE_PRIO宏指定了constructor的优先级，即函数执行的顺序，值越小优先级越高。目前定义了4个优先级：RTE_PRIORITY_LOG、RTE_PRIORITY_BUS、RTE_PRIORITY_CLASS和RTE_PRIORITY_LAST。 ``` #define RTE_PRIORITY_LOG 101 #define RTE_PRIORITY_BUS 110 #define RTE_PRIORITY_CLASS 120 #define RTE_PRIORITY_LAST 65535 #define RTE_PRIO(prio) \ RTE_PRIORITY_ ## prio ``` 其中RTE_INIT宏封装了RTE_PRIORITY_LAST优先级的函数。RTE_REGISTER_BUS宏封装了RTE_PRIORITY_BUS优先级的函数。而RTE_REGISTER_CLASS宏封装了RTE_PRIORITY_CLASS优先级的函数。目前DPDK中RTE_PRIORITY_LOG优先级的函数仅有一个rte_log_init，优先级最高，最先初始化（函数所在文件：lib/librte_eal/common/eal_common_log.c）。 ``` RTE_INIT_PRIO(rte_log_init, LOG) { uint32_t i; rte_log_set_global_level(RTE_LOG_DEBUG); rte_logs.dynamic_types = calloc(RTE_LOGTYPE_FIRST_EXT_ID, sizeof(struct rte_log_dynamic_type)); if (rte_logs.dynamic_types == NULL) return; /* register legacy log types */ for (i = 0; i < RTE_DIM(logtype_strings); i++) __rte_log_register(logtype_strings[i].logtype, logtype_strings[i].log_id); rte_logs.dynamic_types_len = RTE_LOGTYPE_FIRST_EXT_ID; } ``` rte_logs为全局变量，rte_log_init函数首先调用rte_log_set_global_level函数对全局的一个日志等级进行配置，配置为RTE_LOG_DEBUG。 ``` struct rte_log_dynamic_type { const char *name; uint32_t loglevel; }; /** The rte_log structure. */ struct rte_logs { uint32_t type; /**< Bitfield with enabled logs. */ uint32_t level; /**< Log level. */ FILE *file; /**< Output file set by rte_openlog_stream, or NULL. */ size_t dynamic_types_len; struct rte_log_dynamic_type *dynamic_types; }; /* global log structure */ struct rte_logs rte_logs = { .type = ~0, .level = RTE_LOG_DEBUG, .file = NULL, }; /* Set global log level */ void rte_log_set_global_level(uint32_t level) { rte_logs.level = (uint32_t)level; } ``` 对于各日志模块，再调用__rte_log_register函数逐个进行等级配置，默认为RTE_LOG_INFO，下面是系统已定义的一些日志模块，**也可以用户自定义添加，见后续说明，同时各模块的日志等级也能在程序启动时通过命令行“--log-level=log_type:log_level”更改**。 ``` static const struct logtype logtype_strings[] = { {RTE_LOGTYPE_EAL, "lib.eal"}, {RTE_LOGTYPE_MALLOC, "lib.malloc"}, {RTE_LOGTYPE_RING, "lib.ring"}, {RTE_LOGTYPE_MEMPOOL, "lib.mempool"}, {RTE_LOGTYPE_TIMER, "lib.timer"}, {RTE_LOGTYPE_PMD, "pmd"}, {RTE_LOGTYPE_HASH, "lib.hash"}, {RTE_LOGTYPE_LPM, "lib.lpm"}, {RTE_LOGTYPE_KNI, "lib.kni"}, {RTE_LOGTYPE_ACL, "lib.acl"}, {RTE_LOGTYPE_POWER, "lib.power"}, {RTE_LOGTYPE_METER, "lib.meter"}, {RTE_LOGTYPE_SCHED, "lib.sched"}, {RTE_LOGTYPE_PORT, "lib.port"}, {RTE_LOGTYPE_TABLE, "lib.table"}, {RTE_LOGTYPE_PIPELINE, "lib.pipeline"}, {RTE_LOGTYPE_MBUF, "lib.mbuf"}, {RTE_LOGTYPE_CRYPTODEV, "lib.cryptodev"}, {RTE_LOGTYPE_EFD, "lib.efd"}, {RTE_LOGTYPE_EVENTDEV, "lib.eventdev"}, {RTE_LOGTYPE_GSO, "lib.gso"}, {RTE_LOGTYPE_USER1, "user1"}, {RTE_LOGTYPE_USER2, "user2"}, {RTE_LOGTYPE_USER3, "user3"}, {RTE_LOGTYPE_USER4, "user4"}, {RTE_LOGTYPE_USER5, "user5"}, {RTE_LOGTYPE_USER6, "user6"}, {RTE_LOGTYPE_USER7, "user7"}, {RTE_LOGTYPE_USER8, "user8"} }; static int __rte_log_register(const char *name, int id) { char *dup_name = strdup(name); if (dup_name == NULL) return -ENOMEM; rte_logs.dynamic_types[id].name = dup_name; rte_logs.dynamic_types[id].loglevel = RTE_LOG_INFO; return id; } ``` 日志等级如下，值越大等级越高，打印的信息越多。 ``` #define RTE_LOG_EMERG 1U /**< System is unusable. */ #define RTE_LOG_ALERT 2U /**< Action must be taken immediately. */ #define RTE_LOG_CRIT 3U /**< Critical conditions. */ #define RTE_LOG_ERR 4U /**< Error conditions. */ #define RTE_LOG_WARNING 5U /**< Warning conditions. */ #define RTE_LOG_NOTICE 6U /**< Normal but significant condition. */ #define RTE_LOG_INFO 7U /**< Informational. */ #define RTE_LOG_DEBUG 8U /**< Debug-level messages. */ ``` #### 日志输出逻辑 dpdk的各日志模块封装了rte_log函数，rte_log的核心是rte_vlog函数，首先要判断日志等级与全局日志等级的关系，如果大于全局日志等级，则不输出，再判断与该模块定义日志等级的关系，若大于该模块日志等级，同样不输出，在输出的情况下，是输出到rte_logs.file里，即下面的default_log_stream，**default_log_stream初始值是在rte_eal_init里面配置的**。 ``` #define RTE_LOG(l, t, ...) \ rte_log(RTE_LOG_ ## l, \ RTE_LOGTYPE_ ## t, # t ": " __VA_ARGS__) int rte_log(uint32_t level, uint32_t logtype, const char *format, ...) { va_list ap; int ret; va_start(ap, format); ret = rte_vlog(level, logtype, format, ap); va_end(ap); return ret; } int rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) { int ret; FILE *f = rte_logs.file; if (f == NULL) { f = default_log_stream; if (f == NULL) { /* * Grab the current value of stderr here, rather than * just initializing default_log_stream to stderr. This * ensures that we will always use the current value * of stderr, even if the application closes and * reopens it. */ f = stderr; } } if (level > rte_logs.level) { return 0; } if (logtype >= rte_logs.dynamic_types_len) { return -1; } if (level > rte_logs.dynamic_types[logtype].loglevel) { return 0; } /* save loglevel and logtype in a global per-lcore variable */ RTE_PER_LCORE(log_cur_msg).loglevel = level; RTE_PER_LCORE(log_cur_msg).logtype = logtype; ret = vfprintf(f, format, ap); fflush(f); return ret; } ``` #### 日志输出位置 在rte_eal_log_init里面，log_stream被配置为标准输出stdout，log_stream又被赋值给default_log_stream，这样dpdk的日志默认被输出到stdout上，若想改为其它位置可以在rte_eal_init之后**通过rte_openlog_stream重定向到其它文件中**。 ``` int rte_eal_init(int argc, char **argv) { if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0) { rte_eal_init_alert("Cannot init logging."); rte_errno = ENOMEM; rte_atomic32_clear(&run_once); return -1; } } int rte_eal_log_init(const char *id, int facility) { FILE *log_stream; log_stream = fopencookie(NULL, "w+", console_log_func); if (log_stream == NULL) return -1; openlog(id, LOG_NDELAY | LOG_PID, facility); eal_log_set_default(log_stream); return 0; } static cookie_io_functions_t console_log_func = { .write = console_log_write, }; static ssize_t console_log_write(__attribute__((unused)) void *c, const char *buf, size_t size) { ssize_t ret; /* write on stdout */ ret = fwrite(buf, 1, size, stdout); fflush(stdout); /* Syslog error levels are from 0 to 7, so subtract 1 to convert */ syslog(rte_log_cur_msg_loglevel() - 1, "%.*s", (int)size, buf); return ret; } void eal_log_set_default(FILE *default_log) { default_log_stream = default_log; #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG RTE_LOG(NOTICE, EAL, "Debug dataplane logs available - lower performance\n"); #endif } ``` #### 修改对应模块日志等级 以l2fwd程序作为验证基础，如下，dpdk程序中已定义该log模块为RTE_LOGTYPE_USER1，可以在上面的模块表找到对应关系{RTE_LOGTYPE_USER1, "user1"}。 ``` #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 ``` 编译后运行“./build/l2fwd -l 1 -n 4 -- -q 1 -p 1”命令启动l2fwd，因为该模块的默认输出的最高日志等级为RTE_LOG_INFO（7）， 代码中的相关日志也是以INFO的等级输出，可看到屏幕上有下面的打印： ``` Port statistics ==================================== Statistics for port 0 ------------------------------ Packets sent: 175 Packets received: 175 Packets dropped: 0 Aggregate statistics =============================== Total packets sent: 175 Total packets received: 175 Total packets dropped: 0 ==================================================== Port statistics ==================================== Statistics for port 0 ------------------------------ Packets sent: 175 Packets received: 175 Packets dropped: 0 Aggregate statistics =============================== Total packets sent: 175 Total packets received: 175 Total packets dropped: 0 ==================================================== ``` 但当用“./build/l2fwd -l 1 -n 4 --log-level="user1":6 -- -q 1 -p 1” 命令启动程序后，就看不到日志输出了，因为将该模块输出的最高日志等级改为了RTE_LOG_NOTICE（6），所以INFO等级的日志就不能输出，验证成功。 #### 重定向功能验证 在rte_eal_init后面，添加下列函数，可将日志输出到/var/log/rte.log文件中，验证成功。 ``` FILE *fp = fopen("/var/log/rte.log", "w+"); if (fp) { rte_openlog_stream(fp); } ``` #### 自定义log_type验证 若新加一个日志模块，需要做以下操作，下列示例也是在l2fwd中完成修改： 1. 定义新的log_type，在rte_eal_init之前完成初始化。 ``` int logtype_l2fwd; logtype_l2fwd = rte_log_register("l2fwd"); #其实是封装了__rte_log_register if (logtype_l2fwd >= 0) { rte_log_set_level(logtype_l2fwd, RTE_LOG_INFO); #默认最高日志等级为RTE_LOG_INFO } ``` 2. 封装rte_log函数，预置log类型为logtype_l2fwd，输出该模块的log时直接调用LOG_L2FWD接口。 ``` #define LOG_L2FWD(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, logtype_l2fwd, "%s(): " fmt, \ __func__, ## args) ``` 3. 增加该自定义log后验证了解析--log-level参数和重定向功能，同样ok。 ### UPGW的log模块设计 如下，设置NES的log模块，初始化日志等级为INFO，eal初始化后将日志重定向到/var/log/rte.log文件中。 ``` int logtype_nes; logtype_nes = rte_log_register("NES"); if (logtype_nes >= 0) { rte_log_set_level(logtype_nes, RTE_LOG_INFO); } else { NES_LOG("logtype_nes register error\n"); return NES_FAIL; } eal_args = rte_eal_init(argc, argv); FILE *fp = fopen("/var/log/rte.log", "w+"); if (fp) { rte_openlog_stream(fp); } ``` 设计NES_LOG的接口如下，每条日志的通用字段是日志等级和打印这条日志的文件、函数和行数，其它为可选参数。 ``` #define NES_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, logtype_nes, "["#level"] "fmt" %s -- %s() -- %d\n", \ ## args, __FILE__, __func__, __LINE__) ``` ### LOG具体实现 目前日志的实现思路是打印一条流从进入到送出的整个流程，细化下来就是这条流在经过recv模块、scatter、flow和send模块时，都留下相关的日志输出，为跟踪代码流程和排错提供参考。 #### recv模块日志 在nes_dev_t结构体中增加port_rx变量，标识收包index，收到报文后，将更新后的值赋予mbuf中的udata64（用户数据）变量，这样该端口收到的每个mbuf都是唯一标识的，它在传递的过程也不会改变，日志中记录该线程为nes_io、模块为recv，同时记录端口名字和index。 ``` typedef struct nes_dev_s { uint64_t port_rx; /* 增加该变量记录端口的收包index */ } nes_dev_t; self->port_rx++; self->rx_pkts[i]->udata64 = self->port_rx; /* 在mbuf中标识该报文 */ NES_LOG(DEBUG, "thread:nes_io, module:recv, portname:%s, index:%llu", self->name, self->rx_pkts[i]->udata64); ``` #### scatter模块日志 scatter根据收包的端口不通，流程也会不同。 * upstream口 1. 判断为NO ETHER_TYPE_IPv4类型的报文，直接送入engress的队列，不再经过flow模块，日志内容如下，记录线程为nes_io、模块为scatter、方向为FROM UPSTR、协议类型为NO ETHER_TYPE_IPv4，同时记录端口名字、index和将要发往的ring。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:%s", self->name, self->rx_pkts[i]->udata64, ring->ring->name); ``` 2. 判断为SCTP类型的报文，送往NIS_PORT_UPSTR_SCTP队列，日志如下，这里因为已经解析到了ip地址，所以将源目ip也记录了下来。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:SCTP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` 3. 判断为IP类型的报文，送往NTS_PORT_UPSTR_IP队列，日志如下。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` 4. 判断是LTE WITHOUT UDP类型的报文，直接送入engress的队列，不再经过flow模块，日志如下。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:LTE WITHOUT UDP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` 5. 判断是GTPU类型的报文，送往NTS_PORT_UPSTR_GTPU队列，日志如下。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:GTPU, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` 6. 判断是GTPC 类型的报文，送往NIS_PORT_UPSTR_GTPC 队列，日志如下。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:GTPC, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` 7. 判断是LTE WITH UDP类型的报文，送往engress队列，不再经过flow模块，日志如下。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:LTE WITH UDP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name); ``` * downstream口 与上文报文分类一致，只是报文方向改为了FROM DWSTR，日志不再赘述。 * lbp口 报文进入NTS_LBP_ANY队列，在此模块没有解析ip，在flow模块进行了解析，若enq_burst一次没有发送完成，记录上面的日志，module为scatter less，若一次发送完成记录下面的日志，module为scatter，如下所示。 ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter less, portname:%s, index:%llu, traffic-direction:FROM LBP, to_ring:%s", self->name, self->rx_pkts[j]->udata64, rx_ring->ring->name); ``` ``` NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, traffic-direction:FROM LBP, to_ring:%s", self->name, self->rx_pkts[j]->udata64, rx_ring->ring->name); ``` #### flow模块日志 * 处理NTS_PORT_UPSTR_IP队列报文 1. 判断是否命中DNS规则，若命中日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO DNS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, ring->ring->name); ``` 2. 判断是否命中LBP规则，这条日志不能区分UPSTR和DWSTR，IP和GTPU，可与前面日志联系进行区分，若命中日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:TO LBP, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, entry->dst_ring->ring->name); ``` 3. 其它情况往engress口转发。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name); ``` * 处理NTS_PORT_UPSTR_GTPU队列报文 1. 判断是否有NONE_N3_N9_EXTENSION_FORWARD标志，若有往engress口转发，日志如下： ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:NONE_N3_N9, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name); ``` 2. 判断是否命中DNS规则，若命中日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO DNS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, ring->ring->name); ``` 3. 判断是否命中LBP规则，若命中日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:TO LBP, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, entry->dst_ring->ring->name); ``` 4. 其它情况往engress口转发。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name); ``` * 处理NTS_PORT_DWSTR_IP队列报文 与上面处理NTS_PORT_UPSTR_IP队列报文类型，只是不再命中DNS规则，同时报文方向改为了FROM DWSTR。 * 处理NTS_PORT_DWSTR_GTPU队列报文 与上面处理NTS_PORT_UPSTR_GTPU队列报文类型，只是不再命中DNS规则，同时报文方向改为了FROM DWSTR。 * 处理NTS_LBP_ANY队列报文 1. 若是arp request报文，修改为arp reply报文后发送给源端口，日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO LBP, proto-type:ARP REPLY, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, src_ip, dst_ip, dst_ring->ring->name); ``` 2. 若是ip报文，根据命中learning表项的情况往对应的engress口发出，日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO ENGRESS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[idx]->port].device->name, mbufs[idx]->udata64, src_ip, dst_ip, egress_ring->ring->name); ``` 3. 若是gtpu报文，根据命中learning表项的情况往对应的engress口发出，日志如下。 ``` NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO ENGRESS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[idx]->port].device->name, mbufs[idx]->udata64, src_ip, dst_ip, egress_ring->ring->name); ``` #### send模块日志 * send 1. self->tx_buffer中还保存有未发完的报文，记录下面的日志，from portname代表从哪个端口进入的报文，from ring代表从什么队列取到的包，可以和前面的日志进行联系，再加上index就可以一一对应。 ``` NES_LOG(DEBUG, "thread:nes_io, module:send remaining, from portname:%s, to portname:%s, index:%llu, from ring:%s", devices_tab[self->tx_buffer[j]->port].device->name, self->name, self->tx_buffer[j]->udata64, tx_ring->ring->name); ``` 2. tx_burst未一次发送完成记录下面的日志。 ``` NES_LOG(DEBUG, "thread:nes_io, module:send less, from portname:%s, to portname:%s, index:%llu, from ring:%s", devices_tab[buf[j]->port].device->name, self->name, buf[j]->udata64, tx_ring->ring->name); ``` 3. tx_burst发送完成记录下面的日志。 ``` NES_LOG(DEBUG, "thread:nes_io, module:send, from portname:%s, to portname:%s, index:%llu, from ring:%s", devices_tab[buf[j]->port].device->name, self->name, buf[j]->udata64, tx_ring->ring->name); ``` * send mtu 与上面基本一致，只是module:send xxx改为了module:send mtu xxx，不再重复记录。 ### UPGW日志模块测试 #### 逻辑拓扑 如下所示，nr0和up0直连，nr1和up1直连，upf0和down0直连，upf1和down1直连，dn0和lbp0直连，dn1和lbp1直连。 ![逻辑拓扑](../../_static/logic_upgw.png) #### 配置文件 ``` [PORT0] name = 0000:00:04.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:04.0 traffic-type = IP traffic-direction = upstream egress-port = 1 [PORT1] name = 0000:00:05.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:05.0 traffic-type = IP traffic-direction = downstream egress-port = 0 [PORT2] name = 0000:00:06.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:06.0 traffic-type = IP traffic-direction = lbp egress-port = 0 lbp-mac = fa:16:3e:c0:70:fa [PORT3] name = 0000:00:0e.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:0e.0 traffic-type = IP traffic-direction = upstream egress-port = 4 [PORT4] name = 0000:00:0f.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:0f.0 traffic-type = IP traffic-direction = downstream egress-port = 3 [PORT5] name = 0000:00:10.0 description = 82540EM Gigabit Ethernet Controller pci-address = 0000:00:10.0 traffic-type = IP traffic-direction = lbp egress-port = 3 lbp-mac = fa:16:3e:a7:02:cc [VM common] max = 32 number = 2 vhost-dev = /var/lib/appliance/nts/qemu/usvhost-1 [NES_SERVER] ;ctrl_socket = /var/lib/appliance/nts/control-socket ;ctrl_ip = 127.0.0.1 ctrl_ip = 0.0.0.0 ctrl_port = 8080 [KNI] max = 32 [REDIS_SERVER] host = 127.0.0.1 port = 6379 ``` #### 测试项1-nr0向upf0进行ping包 截取日志，如下所示，IP包为四条log记录，ARP包为三条log记录。 ``` [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:1, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:1, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:1, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:1, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:2, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:2, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:2, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:2, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:3, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:3, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:3, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:3, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:3, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:3, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:4, traffic-direction:FROM DWSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 219 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:4, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:4, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 217 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:4, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 ``` #### 测试项2-upf0向nr0进行ping包 截取日志，如下所示，IP包为四条log记录，ARP包为三条log记录。 ``` [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:1, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:1, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:1, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:1, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:2, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:2, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:2, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:2, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:3, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:3, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:3, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:3, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:3, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:3, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:4, traffic-direction:FROM DWSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 219 [DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:4, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:4, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 217 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:4, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 ``` #### 测试项3-配置route后nr0向upf0进行ping包 route规则为： ``` route add fa:16:3e:c0:70:fa ace4d210-3d92-422b-83d7-11a0de50fea5 prio:98,srv_ip:192.168.30.116/32,encap_proto:noencap,dst_lbp_port:2 ``` 截取日志，如下所示，进入up0的IP包被卸载到lbp0。 ``` [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:TO LBP, to_ring:PORT_2_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_packet_edit_enq() -- 435 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:06.0, index:1, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418 [DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:06.0, index:2, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:TO LBP, to_ring:PORT_2_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_packet_edit_enq() -- 435 ``` #### 测试项4-配置route后dn0向nr0进行ping包 截取日志，如下所示，首先有个ARP代答的记录，然后lbp0的流量被送到了up0。 ``` [DEBUG] thread:nes_io, module:recv, portname:0000:00:06.0, index:12 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:06.0, index:12, traffic-direction:FROM LBP, to_ring:NTS_LBP_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_lbp() -- 504 [DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:12, srcip:192.168.40.84, dstip:192.168.20.74, traffic-direction:FROM LBP, proto-type:ARP REQUEST /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_vm() -- 1209 [DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:12, srcip:192.168.20.74, dstip:192.168.40.84, traffic-direction:FROM LBP TO LBP, proto-type:ARP REPLY, to_ring:PORT_2_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_edge_arp_edit() -- 1110 [DEBUG] thread:nes_io, module:send, from portname:0000:00:06.0, to portname:0000:00:06.0, index:12, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 [DEBUG] thread:nes_io, module:recv, portname:0000: 00:06.0, index:13 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211 [DEBUG] thread:nes_io, module:scatter, portname:0000:00:06.0, index:13, traffic-direction:FROM LBP, to_ring:NTS_LBP_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_lbp() -- 504 [DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:13, srcip:192.168.40.84, dstip:192.168.20.74, traffic-direction:FROM LBP TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_vm() -- 1242 [DEBUG] thread:nes_io, module:send, from portname:0000:00:06.0, to portname:0000:00:04.0, index:13, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267 ```