From 818f3139d446f82e92c0d42767bf9492164f05c1 Mon Sep 17 00:00:00 2001 From: Sukchan Lee Date: Sun, 13 Sep 2020 22:04:43 -0400 Subject: [PATCH] memory optimization [#549] 1. number of packet buffer : 65,536 -> 32,768 2. Packet SDU Size : 8,192 -> 2,048 3. Stabilize test program --- configs/open5gs/sgwu.yaml.in | 6 ++-- configs/open5gs/upf.yaml.in | 6 ++-- docker/docker-compose.yml | 2 +- docker/ubuntu/latest/dev/Dockerfile | 8 ++++- .../guide/02-building-open5gs-from-sources.md | 5 +++ lib/app/ogs-context.c | 2 +- lib/core/ogs-3gpp-types.h | 1 + lib/pfcp/xact.c | 36 +++++++++---------- src/sgwu/event.h | 2 -- src/sgwu/gtp-path.c | 6 ++-- src/sgwu/gtp-path.h | 4 --- src/sgwu/sgwu-sm.c | 12 ------- src/upf/gtp-path.c | 19 +++++++--- tests/app/5gc-init.c | 14 ++++---- tests/app/app-init.c | 10 ++---- tests/app/epc-init.c | 11 +++--- tests/common/application.c | 2 -- 17 files changed, 72 insertions(+), 74 deletions(-) diff --git a/configs/open5gs/sgwu.yaml.in b/configs/open5gs/sgwu.yaml.in index 6b12dcab9..82f20b8ec 100644 --- a/configs/open5gs/sgwu.yaml.in +++ b/configs/open5gs/sgwu.yaml.in @@ -122,9 +122,9 @@ max: # big: 8 # # o Memory of Packet Buffering in UPF/SGW -# - Maximum Number of packet(SDU size = 8Kbytes) pool in UPF/SGW -# - UPF/SGW Memory Usage : 65536 * 8Kbytes = 512Mbytes +# - Maximum Number of packet(SDU size = 2Kbytes) pool in UPF/SGW +# - UPF/SGW Memory Usage : 32768 * 2Kbytes = 64Mbytes # -# packet: 65536 +# packet: 32768 # pool: diff --git a/configs/open5gs/upf.yaml.in b/configs/open5gs/upf.yaml.in index 083778c7c..e78b9a986 100644 --- a/configs/open5gs/upf.yaml.in +++ b/configs/open5gs/upf.yaml.in @@ -210,8 +210,8 @@ max: # big: 8 # # o Memory of Packet Buffering in UPF/SGW -# - Maximum Number of packet(SDU size = 8Kbytes) pool in UPF/SGW -# - UPF/SGW Memory Usage : 65536 * 8Kbytes = 512Mbytes +# - Maximum Number of packet(SDU size = 2Kbytes) pool in UPF/SGW +# - UPF/SGW Memory Usage : 32768 * 2Kbytes = 64Mbytes # -# packet: 65536 +# packet: 32768 pool: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 33a3764d4..1ea56b75c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -57,7 +57,7 @@ services: environment: - DB_URI=mongodb://mongodb/open5gs - DISPLAY=$DISPLAY - # - DISPLAY=docker.for.mac.localhost:0 + # - DISPLAY=host.docker.internal:0 cap_add: - NET_ADMIN devices: diff --git a/docker/ubuntu/latest/dev/Dockerfile b/docker/ubuntu/latest/dev/Dockerfile index 8beba0d3d..6902fc59c 100644 --- a/docker/ubuntu/latest/dev/Dockerfile +++ b/docker/ubuntu/latest/dev/Dockerfile @@ -20,10 +20,16 @@ RUN apt-get update && \ vim \ sudo \ iputils-ping \ - wireshark \ net-tools && \ apt-get clean +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + sudo add-apt-repository ppa:wireshark-dev/stable -y && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y wireshark + COPY setup.sh /root ARG username=acetcom diff --git a/docs/_docs/guide/02-building-open5gs-from-sources.md b/docs/_docs/guide/02-building-open5gs-from-sources.md index f2748021b..c0920d39d 100644 --- a/docs/_docs/guide/02-building-open5gs-from-sources.md +++ b/docs/_docs/guide/02-building-open5gs-from-sources.md @@ -57,6 +57,11 @@ $ meson build --prefix=`pwd`/install $ ninja -C build ``` +Please free up enough memory space on the VM and run the test program. + +The test program has been run on [VirtualBox - CPU: 1, Memory: 4.00 GB] and [Docker for Mac - CPU: 2, Memory: 2.00 GB] with default setting (max.ue: 4,096, pool.packet: 32,768). +{: .notice--danger} + Check whether the compilation is correct. ```bash $ ./build/tests/attach/attach ## EPC Only diff --git a/lib/app/ogs-context.c b/lib/app/ogs-context.c index 0cb9b84d9..d564e3d60 100644 --- a/lib/app/ogs-context.c +++ b/lib/app/ogs-context.c @@ -179,7 +179,7 @@ static void app_context_prepare(void) self.max.gnb = MAX_NUM_OF_GNB; self.max.ue = MAX_NUM_OF_UE; -#define MAX_NUM_OF_PACKET_POOL 65536 +#define MAX_NUM_OF_PACKET_POOL 32768 self.pool.packet = MAX_NUM_OF_PACKET_POOL; ogs_pkbuf_default_init(&self.pool.defconfig); diff --git a/lib/core/ogs-3gpp-types.h b/lib/core/ogs-3gpp-types.h index 8b86c3dd3..7a80c48bb 100644 --- a/lib/core/ogs-3gpp-types.h +++ b/lib/core/ogs-3gpp-types.h @@ -37,6 +37,7 @@ extern "C" { #define OGS_MAX_NUM_OF_PACKET_FILTER 16 #define OGS_MAX_SDU_LEN 8192 +#define OGS_MAX_PKT_LEN 2048 #define OGS_PLMN_ID_LEN 3 #define OGS_MAX_PLMN_ID_BCD_LEN 6 diff --git a/lib/pfcp/xact.c b/lib/pfcp/xact.c index 99ee20be7..2ffd4b17f 100644 --- a/lib/pfcp/xact.c +++ b/lib/pfcp/xact.c @@ -188,7 +188,7 @@ int ogs_pfcp_xact_update_tx(ogs_pfcp_xact_t *xact, switch (stage) { case PFCP_XACT_INITIAL_STAGE: if (xact->step != 0) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, hdesc->type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } @@ -201,7 +201,7 @@ int ogs_pfcp_xact_update_tx(ogs_pfcp_xact_t *xact, case PFCP_XACT_FINAL_STAGE: if (xact->step != 2) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, hdesc->type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } @@ -221,19 +221,19 @@ int ogs_pfcp_xact_update_tx(ogs_pfcp_xact_t *xact, case PFCP_XACT_INTERMEDIATE_STAGE: case PFCP_XACT_FINAL_STAGE: if (xact->step != 1) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, hdesc->type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } break; default: - ogs_error("invalid stage[%d]", stage); + ogs_error("invalid stage[%d] type[%d]", stage, hdesc->type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } } else { - ogs_error("invalid org[%d]", xact->org); + ogs_error("invalid org[%d] type[%d]", xact->org, hdesc->type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } @@ -296,7 +296,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) ogs_pkbuf_t *pkbuf = NULL; if (xact->step != 2 && xact->step != 3) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); ogs_pkbuf_free(pkbuf); return OGS_ERROR; } @@ -335,7 +335,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) } if (xact->step != 1) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); return OGS_ERROR; } @@ -347,7 +347,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) case PFCP_XACT_FINAL_STAGE: if (xact->step != 1) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); return OGS_ERROR; } break; @@ -363,7 +363,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) ogs_pkbuf_t *pkbuf = NULL; if (xact->step != 1 && xact->step != 2) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); return OGS_ERROR; } @@ -401,7 +401,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) } if (xact->step != 0) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); return OGS_ERROR; } if (xact->tm_holding) @@ -416,7 +416,7 @@ int ogs_pfcp_xact_update_rx(ogs_pfcp_xact_t *xact, uint8_t type) case PFCP_XACT_FINAL_STAGE: if (xact->step != 2) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); return OGS_ERROR; } @@ -470,7 +470,7 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) switch (stage) { case PFCP_XACT_INITIAL_STAGE: if (xact->step != 1) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } @@ -488,7 +488,7 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) case PFCP_XACT_FINAL_STAGE: if (xact->step != 2 && xact->step != 3) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } @@ -500,7 +500,7 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) break; default: - ogs_error("invalid stage[%d]", stage); + ogs_error("invalid stage[%d] type[%d]", stage, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } @@ -513,7 +513,7 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) case PFCP_XACT_INTERMEDIATE_STAGE: if (xact->step != 2) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } @@ -525,7 +525,7 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) case PFCP_XACT_FINAL_STAGE: if (xact->step != 2 && xact->step != 3) { - ogs_error("invalid step[%d]", xact->step); + ogs_error("invalid step[%d] type[%d]", xact->step, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } @@ -537,12 +537,12 @@ int ogs_pfcp_xact_commit(ogs_pfcp_xact_t *xact) break; default: - ogs_error("invalid stage[%d]", stage); + ogs_error("invalid stage[%d] type[%d]", stage, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } } else { - ogs_error("invalid org[%d]", xact->org); + ogs_error("invalid org[%d] type[%d]", xact->org, type); ogs_pfcp_xact_delete(xact); return OGS_ERROR; } diff --git a/src/sgwu/event.h b/src/sgwu/event.h index b83dce2a5..e64f0ae19 100644 --- a/src/sgwu/event.h +++ b/src/sgwu/event.h @@ -35,8 +35,6 @@ typedef struct sgwu_bearer_s sgwu_bearer_t; typedef enum { SGWU_EVT_BASE = OGS_FSM_USER_SIG, - SGWU_EVT_LO_DLDATA_NOTI, - SGWU_EVT_SXA_MESSAGE, SGWU_EVT_SXA_TIMER, SGWU_EVT_SXA_NO_HEARTBEAT, diff --git a/src/sgwu/gtp-path.c b/src/sgwu/gtp-path.c index 1063361eb..623cd3de7 100644 --- a/src/sgwu/gtp-path.c +++ b/src/sgwu/gtp-path.c @@ -43,9 +43,9 @@ static void _gtpv1_u_recv_cb(short when, ogs_socket_t fd, void *data) ogs_assert(fd != INVALID_SOCKET); - pkbuf = ogs_pkbuf_alloc(packet_pool, OGS_MAX_SDU_LEN); + pkbuf = ogs_pkbuf_alloc(packet_pool, OGS_MAX_PKT_LEN); ogs_assert(pkbuf); - ogs_pkbuf_put(pkbuf, OGS_MAX_SDU_LEN); + ogs_pkbuf_put(pkbuf, OGS_MAX_PKT_LEN); size = ogs_recvfrom(fd, pkbuf->data, pkbuf->len, 0, &from); if (size <= 0) { @@ -181,7 +181,7 @@ int sgwu_gtp_open(void) ogs_pkbuf_config_t config; memset(&config, 0, sizeof config); - config.cluster_8192_pool = ogs_app()->pool.packet; + config.cluster_2048_pool = ogs_app()->pool.packet; packet_pool = ogs_pkbuf_pool_create(&config); diff --git a/src/sgwu/gtp-path.h b/src/sgwu/gtp-path.h index 78dde1c3c..3865691c8 100644 --- a/src/sgwu/gtp-path.h +++ b/src/sgwu/gtp-path.h @@ -29,10 +29,6 @@ extern "C" { int sgwu_gtp_open(void); void sgwu_gtp_close(void); -#if 0 -void sgwu_gtp_send_end_marker(sgwu_tunnel_t *s1u_tunnel); -#endif - #ifdef __cplusplus } #endif diff --git a/src/sgwu/sgwu-sm.c b/src/sgwu/sgwu-sm.c index 5b18605e2..6de326c1e 100644 --- a/src/sgwu/sgwu-sm.c +++ b/src/sgwu/sgwu-sm.c @@ -45,8 +45,6 @@ void sgwu_state_operational(ogs_fsm_t *s, sgwu_event_t *e) ogs_pfcp_node_t *node = NULL; ogs_pfcp_xact_t *xact = NULL; - sgwu_bearer_t *bearer = NULL; - sgwu_sm_debug(e); ogs_assert(s); @@ -67,16 +65,6 @@ void sgwu_state_operational(ogs_fsm_t *s, sgwu_event_t *e) sgwu_pfcp_close(); sgwu_gtp_close(); break; - case SGWU_EVT_LO_DLDATA_NOTI: - ogs_assert(e); - - bearer = e->bearer; - ogs_assert(bearer); - -#if 0 - sgwu_s11_handle_lo_dldata_notification(bearer); -#endif - break; case SGWU_EVT_SXA_MESSAGE: ogs_assert(e); recvbuf = e->pkbuf; diff --git a/src/upf/gtp-path.c b/src/upf/gtp-path.c index f0780e937..635f0b7ce 100644 --- a/src/upf/gtp-path.c +++ b/src/upf/gtp-path.c @@ -41,6 +41,8 @@ #define UPF_GTP_HANDLED 1 +static ogs_pkbuf_pool_t *packet_pool = NULL; + static void upf_gtp_handle_multicast(ogs_pkbuf_t *recvbuf); static int upf_gtp_handle_slaac(upf_sess_t *sess, ogs_pkbuf_t *recvbuf); static int upf_gtp_send_router_advertisement( @@ -53,10 +55,10 @@ static void _gtpv1_tun_recv_cb(short when, ogs_socket_t fd, void *data) ogs_pfcp_pdr_t *pdr = NULL; ogs_pfcp_user_plane_report_t report; - recvbuf = ogs_pkbuf_alloc(NULL, OGS_MAX_SDU_LEN); + recvbuf = ogs_pkbuf_alloc(packet_pool, OGS_MAX_PKT_LEN); ogs_assert(recvbuf); ogs_pkbuf_reserve(recvbuf, OGS_GTPV1U_5GC_HEADER_LEN); - ogs_pkbuf_put(recvbuf, OGS_MAX_SDU_LEN-OGS_GTPV1U_5GC_HEADER_LEN); + ogs_pkbuf_put(recvbuf, OGS_MAX_PKT_LEN-OGS_GTPV1U_5GC_HEADER_LEN); n = ogs_read(fd, recvbuf->data, recvbuf->len); if (n <= 0) { @@ -102,9 +104,9 @@ static void _gtpv1_u_recv_cb(short when, ogs_socket_t fd, void *data) ogs_assert(fd != INVALID_SOCKET); - pkbuf = ogs_pkbuf_alloc(NULL, OGS_MAX_SDU_LEN); + pkbuf = ogs_pkbuf_alloc(NULL, OGS_MAX_PKT_LEN); ogs_assert(pkbuf); - ogs_pkbuf_put(pkbuf, OGS_MAX_SDU_LEN); + ogs_pkbuf_put(pkbuf, OGS_MAX_PKT_LEN); size = ogs_recvfrom(fd, pkbuf->data, pkbuf->len, 0, &from); if (size <= 0) { @@ -256,6 +258,13 @@ int upf_gtp_open(void) ogs_sock_t *sock = NULL; int rc; + ogs_pkbuf_config_t config; + memset(&config, 0, sizeof config); + + config.cluster_2048_pool = ogs_app()->pool.packet; + + packet_pool = ogs_pkbuf_pool_create(&config); + ogs_list_for_each(&upf_self()->gtpu_list, node) { sock = ogs_gtp_server(node); ogs_assert(sock); @@ -333,6 +342,8 @@ void upf_gtp_close(void) ogs_pollset_remove(dev->poll); ogs_closesocket(dev->fd); } + + ogs_pkbuf_pool_destroy(packet_pool); } static void upf_gtp_handle_multicast(ogs_pkbuf_t *recvbuf) diff --git a/tests/app/5gc-init.c b/tests/app/5gc-init.c index a732e3908..da7ec65f0 100644 --- a/tests/app/5gc-init.c +++ b/tests/app/5gc-init.c @@ -49,27 +49,26 @@ int app_initialize(const char *const argv[]) if (ogs_app()->parameter.no_nrf == 0) nrf_thread = test_child_create("nrf", argv_out); + if (ogs_app()->parameter.no_udr == 0) udr_thread = test_child_create("udr", argv_out); if (ogs_app()->parameter.no_udm == 0) udm_thread = test_child_create("udm", argv_out); if (ogs_app()->parameter.no_ausf == 0) ausf_thread = test_child_create("ausf", argv_out); + if (ogs_app()->parameter.no_upf == 0) upf_thread = test_child_create("upf", argv_out); - - /* Wait for PFCP association */ - ogs_msleep(300); - if (ogs_app()->parameter.no_smf == 0) smf_thread = test_child_create("smf", argv_out); + if (ogs_app()->parameter.no_amf == 0) amf_thread = test_child_create("amf", argv_out); /* * Wait for all sockets listening * - * If freeDiameter is not used, it uses a delay of less than 1 second. + * If freeDiameter is not used, it uses a delay of less than 4 seconds. */ ogs_msleep(1500); @@ -78,14 +77,15 @@ int app_initialize(const char *const argv[]) void app_terminate(void) { - ogs_msleep(300); - if (amf_thread) ogs_thread_destroy(amf_thread); + if (smf_thread) ogs_thread_destroy(smf_thread); if (upf_thread) ogs_thread_destroy(upf_thread); + if (ausf_thread) ogs_thread_destroy(ausf_thread); if (udm_thread) ogs_thread_destroy(udm_thread); if (udr_thread) ogs_thread_destroy(udr_thread); + if (nrf_thread) ogs_thread_destroy(nrf_thread); } diff --git a/tests/app/app-init.c b/tests/app/app-init.c index c76bfc483..419820559 100644 --- a/tests/app/app-init.c +++ b/tests/app/app-init.c @@ -71,13 +71,11 @@ int app_initialize(const char *const argv[]) if (ogs_app()->parameter.no_sgwu == 0) sgwu_thread = test_child_create("sgwu", argv_out); - /* Wait for PFCP association */ - ogs_msleep(300); - if (ogs_app()->parameter.no_smf == 0) smf_thread = test_child_create("smf", argv_out); if (ogs_app()->parameter.no_sgwc == 0) sgwc_thread = test_child_create("sgwc", argv_out); + if (ogs_app()->parameter.no_mme == 0) mme_thread = test_child_create("mme", argv_out); if (ogs_app()->parameter.no_amf == 0) @@ -86,7 +84,7 @@ int app_initialize(const char *const argv[]) /* * Wait for all sockets listening * - * Note that at least 1 second is needed if freeDiameter is running. + * Note that at least 4 seconds are needed if freeDiameter is running. */ ogs_msleep(5000); @@ -95,10 +93,9 @@ int app_initialize(const char *const argv[]) void app_terminate(void) { - ogs_msleep(300); - if (amf_thread) ogs_thread_destroy(amf_thread); if (mme_thread) ogs_thread_destroy(mme_thread); + if (sgwc_thread) ogs_thread_destroy(sgwc_thread); if (smf_thread) ogs_thread_destroy(smf_thread); @@ -111,7 +108,6 @@ void app_terminate(void) if (hss_thread) ogs_thread_destroy(hss_thread); if (pcrf_thread) ogs_thread_destroy(pcrf_thread); - if (nrf_thread) ogs_thread_destroy(nrf_thread); } diff --git a/tests/app/epc-init.c b/tests/app/epc-init.c index 119835cd4..2d118f60c 100644 --- a/tests/app/epc-init.c +++ b/tests/app/epc-init.c @@ -60,20 +60,18 @@ int app_initialize(const char *const argv[]) if (ogs_app()->parameter.no_sgwu == 0) sgwu_thread = test_child_create("sgwu", argv_out); - /* Wait for PFCP association */ - ogs_msleep(300); - if (ogs_app()->parameter.no_smf == 0) smf_thread = test_child_create("smf", argv_out); if (ogs_app()->parameter.no_sgwc == 0) sgwc_thread = test_child_create("sgwc", argv_out); + if (ogs_app()->parameter.no_mme == 0) mme_thread = test_child_create("mme", argv_out); /* * Wait for all sockets listening * - * Note that at least 1 second is needed if freeDiameter is running. + * Note that at least 4 seconds are needed if freeDiameter is running. */ ogs_msleep(5000); @@ -82,13 +80,14 @@ int app_initialize(const char *const argv[]) void app_terminate(void) { - ogs_msleep(300); - if (mme_thread) ogs_thread_destroy(mme_thread); + if (sgwc_thread) ogs_thread_destroy(sgwc_thread); if (smf_thread) ogs_thread_destroy(smf_thread); + if (sgwu_thread) ogs_thread_destroy(sgwu_thread); if (upf_thread) ogs_thread_destroy(upf_thread); + if (hss_thread) ogs_thread_destroy(hss_thread); if (pcrf_thread) ogs_thread_destroy(pcrf_thread); if (nrf_thread) ogs_thread_destroy(nrf_thread); diff --git a/tests/common/application.c b/tests/common/application.c index a8c29c326..0e90c6b40 100644 --- a/tests/common/application.c +++ b/tests/common/application.c @@ -72,8 +72,6 @@ void test_app_run(int argc, const char *const argv[], rv = test_context_parse_config(); ogs_assert(rv == OGS_OK); - - ogs_msleep(500); /* Wait for listening all sockets */ } #define MAX_CHILD_PROCESS 16