From 7932f7ff3593e52da1c0624e2b12c5a2d8fd58be Mon Sep 17 00:00:00 2001 From: ywc689 Date: Fri, 26 Jul 2019 20:27:51 +0800 Subject: [PATCH] ctrl: dpvs_msg allocate memory from mempool --- include/ctrl.h | 4 ++++ src/ctrl.c | 35 ++++++++++++++++++++++++++--------- src/ipv6/ipv6_ctrl.c | 2 +- src/ipvs/ip_vs_conn.c | 2 +- src/ipvs/ip_vs_stats.c | 6 +++++- src/neigh.c | 4 +++- src/netif.c | 2 +- src/sa_pool.c | 2 +- src/tc/tc_ctrl.c | 2 +- 9 files changed, 43 insertions(+), 16 deletions(-) diff --git a/include/ctrl.h b/include/ctrl.h index 18ac72598..2fa8c7df4 100644 --- a/include/ctrl.h +++ b/include/ctrl.h @@ -177,6 +177,10 @@ int msg_master_process(int step); /* Master lcore msg loop */ /* Slave lcore msg process loop */ int msg_slave_process(int step); /* Slave lcore msg loop */ +/* allocator for msg reply data */ +void *msg_reply_alloc(int size); +void msg_reply_free(void *mptr); + /* debug utility */ int msg_type_table_print(char *buf, int len); /* print msg_type table on all configured lcores */ int msg_dump(const struct dpvs_msg *msg, char *buf, int len); diff --git a/src/ctrl.c b/src/ctrl.c index a3edd553c..d7f7bf703 100644 --- a/src/ctrl.c +++ b/src/ctrl.c @@ -24,6 +24,7 @@ #include #include "ctrl.h" #include "netif.h" +#include "mempool.h" #include "parser/parser.h" /////////////////////////////////// lcore msg /////////////////////////////////////////// @@ -33,6 +34,7 @@ uint64_t slave_lcore_mask; /* bit-wise enabled lcores */ uint8_t slave_lcore_nb; /* slave lcore number */ lcoreid_t master_lcore; /* master lcore id */ +struct dpvs_mempool *msg_pool; /* memory pool for msg */ struct netif_lcore_loop_job ctrl_lcore_job; @@ -323,24 +325,32 @@ int msg_type_mc_unregister(const struct dpvs_msg_type *msg_type) return EDPVS_OK; } +void *msg_reply_alloc(int size) +{ + return dpvs_mempool_get(msg_pool, size); +} + +void msg_reply_free(void *mptr) +{ + return dpvs_mempool_put(msg_pool, mptr); +} + struct dpvs_msg* msg_make(msgid_t type, uint32_t seq, msg_mode_t mode, lcoreid_t cid, uint32_t len, const void *data) { + int total_len; struct dpvs_msg *msg; - uint32_t flags; - msg = rte_zmalloc("msg", sizeof(struct dpvs_msg) + len, RTE_CACHE_LINE_SIZE); + total_len = sizeof(struct dpvs_msg) + len; + msg = dpvs_mempool_get(msg_pool, total_len); if (unlikely(NULL == msg)) return NULL; + memset(msg, 0, total_len); rte_spinlock_init(&msg->lock); - flags = get_msg_flags(msg); - if (flags) - RTE_LOG(WARNING, MSGMGR, "dirty msg flags: %d\n", flags); - msg->type = type; msg->seq = seq; msg->mode = mode; @@ -350,7 +360,6 @@ struct dpvs_msg* msg_make(msgid_t type, uint32_t seq, rte_memcpy(msg->data, data, len); msg->reply.data = NULL; msg->reply.len = 0; - assert(0 == flags); rte_atomic16_init(&msg->refcnt); rte_atomic16_inc(&msg->refcnt); @@ -402,10 +411,10 @@ int msg_destroy(struct dpvs_msg **pmsg) } if (msg->reply.data) { - rte_free(msg->reply.data); + msg_reply_free(msg->reply.data); msg->reply.len = 0; } - rte_free(msg); + dpvs_mempool_put(msg_pool, msg); *pmsg = NULL; msg_debug_free(); @@ -1001,6 +1010,11 @@ static inline int msg_init(void) return EDPVS_NOTSUPP; } + /* msg_pool uses about 10MB memory */ + msg_pool = dpvs_mempool_create("mp_msg", 32, 65536, 1024); + if (!msg_pool) + return EDPVS_NOMEM; + /* per-lcore msg type array init */ for (ii = 0; ii < DPVS_MAX_LCORE; ii++) { for (jj = 0; jj < DPVS_MSG_LEN; jj++) { @@ -1020,6 +1034,7 @@ static inline int msg_init(void) rte_socket_id(), RING_F_SC_DEQ); if (unlikely(NULL == msg_ring[ii])) { RTE_LOG(ERR, MSGMGR, "%s: fail to create msg ring\n", __func__); + dpvs_mempool_destroy(msg_pool); return EDPVS_DPDKAPIFAIL; } } @@ -1031,6 +1046,7 @@ static inline int msg_init(void) ctrl_lcore_job.type = NETIF_LCORE_JOB_LOOP; if ((ret = netif_lcore_loop_job_register(&ctrl_lcore_job)) < 0) { RTE_LOG(ERR, MSGMGR, "%s: fail to register ctrl func on slave lcores\n", __func__); + dpvs_mempool_destroy(msg_pool); return ret; } @@ -1058,6 +1074,7 @@ static inline int msg_term(void) /* per-lcore msg queue */ for (ii= 0; ii < DPVS_MAX_LCORE; ii++) rte_ring_free(msg_ring[ii]); + dpvs_mempool_destroy(msg_pool); return EDPVS_OK; } diff --git a/src/ipv6/ipv6_ctrl.c b/src/ipv6/ipv6_ctrl.c index 613757d7c..ec6d9a7e4 100644 --- a/src/ipv6/ipv6_ctrl.c +++ b/src/ipv6/ipv6_ctrl.c @@ -34,7 +34,7 @@ static int ip6_msg_get_stats(struct dpvs_msg *msg) struct inet_stats *stats; assert(msg); - stats = rte_zmalloc(NULL, sizeof(*stats), 0); + stats = msg_reply_alloc(sizeof(*stats)); if (!stats) return EDPVS_NOMEM; diff --git a/src/ipvs/ip_vs_conn.c b/src/ipvs/ip_vs_conn.c index d89e7dd65..a46a78e66 100644 --- a/src/ipvs/ip_vs_conn.c +++ b/src/ipvs/ip_vs_conn.c @@ -1476,7 +1476,7 @@ static int conn_get_msgcb_slave(struct dpvs_msg *msg) reply_len = sizeof(struct ip_vs_conn_array) + sizeof(ipvs_conn_entry_t); else reply_len = sizeof(struct ip_vs_conn_array); - reply_data = rte_zmalloc("get_conns", reply_len, 0); + reply_data = msg_reply_alloc(reply_len); if (unlikely(!reply_data)) { dp_vs_conn_put(conn); return EDPVS_NOMEM; diff --git a/src/ipvs/ip_vs_stats.c b/src/ipvs/ip_vs_stats.c index 4d00e02b9..17a80f7d9 100644 --- a/src/ipvs/ip_vs_stats.c +++ b/src/ipvs/ip_vs_stats.c @@ -131,6 +131,7 @@ void dp_vs_zero_stats(struct dp_vs_stats* stats) static int get_stats_uc_cb(struct dpvs_msg *msg) { + char *reply; struct dp_vs_stats **src; lcoreid_t cid; assert(msg); @@ -140,7 +141,10 @@ static int get_stats_uc_cb(struct dpvs_msg *msg) return EDPVS_INVAL; } src = (struct dp_vs_stats **)msg->data; - char *reply = rte_malloc(NULL, sizeof(struct dp_vs_stats), RTE_CACHE_LINE_SIZE); + + reply = msg_reply_alloc(sizeof(struct dp_vs_stats)); + if (unlikely(!reply)) + return EDPVS_NOMEM; rte_memcpy(reply, &((*src)[cid]), sizeof(struct dp_vs_stats)); msg->reply.len = sizeof(struct dp_vs_stats); msg->reply.data = (void *)reply; diff --git a/src/neigh.c b/src/neigh.c index 216ffebce..ce810c42c 100644 --- a/src/neigh.c +++ b/src/neigh.c @@ -856,7 +856,9 @@ static int get_neigh_uc_cb(struct dpvs_msg *msg) len = sizeof(struct dp_vs_neigh_conf_array) + sizeof(struct dp_vs_neigh_conf) * neigh_nums[cid]; - array = rte_zmalloc("neigh_array", len, RTE_CACHE_LINE_SIZE); + array = msg_reply_alloc(len); + if (unlikely(!array)) + return EDPVS_NOMEM; neigh_fill_array(dev, cid, array, neigh_nums[cid]); msg->reply.len = len; diff --git a/src/netif.c b/src/netif.c index 21e4544e2..9764cdaef 100644 --- a/src/netif.c +++ b/src/netif.c @@ -4266,7 +4266,7 @@ static int lcore_stats_msg_cb(struct dpvs_msg *msg) msg->mode != DPVS_MSG_UNICAST)) return EDPVS_INVAL; - reply_data = rte_malloc(NULL, sizeof(struct netif_lcore_stats), RTE_CACHE_LINE_SIZE); + reply_data = msg_reply_alloc(sizeof(struct netif_lcore_stats)); if (unlikely(!reply_data)) return EDPVS_NOMEM; diff --git a/src/sa_pool.c b/src/sa_pool.c index 35da2fcde..b9e01f946 100644 --- a/src/sa_pool.c +++ b/src/sa_pool.c @@ -819,7 +819,7 @@ static int sa_msg_get_stats(struct dpvs_msg *msg) ptr = msg->data; ifa = *(struct inet_ifaddr **)ptr; - stats = rte_zmalloc(NULL, sizeof(*stats), 0); + stats = msg_reply_alloc(sizeof(*stats)); if (!stats) return EDPVS_NOMEM; diff --git a/src/tc/tc_ctrl.c b/src/tc/tc_ctrl.c index d4e977c24..9d67c5a11 100644 --- a/src/tc/tc_ctrl.c +++ b/src/tc/tc_ctrl.c @@ -414,7 +414,7 @@ static int tc_msg_get_stats(struct dpvs_msg *msg) ptr = msg->data; qsch = *(struct Qsch **)ptr; - st = rte_zmalloc(NULL, sizeof(*st), 0); + st = msg_reply_alloc(sizeof(*st)); if (!st) return EDPVS_NOMEM;