From 28e40a0f1b9048768958344e213c69994be671f5 Mon Sep 17 00:00:00 2001 From: Pau Espin Pedrol Date: Tue, 7 Jun 2022 22:51:02 +0200 Subject: [PATCH] Initial metrics support based on Prometheus (#1571) * Initial metrics support based on Prometheus This commit introduces initial support for metrics in open5gs. The metrics code is added as libogsmetrics (lib/metrics/), with a well defined opaque API to manage different types of metrics, allowing for different implementations for different technologies to scrap the metrics (placed as lib/metrics//. The implementation is right now selected at build time, in order to be able to opt-out the related dependencies for users not interested in the features. 2 implementations are already provided in this commit to start with: * void: Default implementation. Empty stubs, acts as a NOOP. * prometheus: open5gs processes become Prometheus servers, offering states through an http server to the Prometheus scrappers. Relies on libprom (prometheus-client-ci [1] project) to track the metrics and format them during export, and libmicrohttpd to make the export possible through HTTP. [1] https://github.com/digitalocean/prometheus-client-c The prometheus-client-c is not well maintained nowadays in upstream, and furthermore it uses a quite peculiar mixture of build systems (autolib on the main dir, cmake for libprom in a subdir). This makes it difficult to have it widely available in distros, and difficult to find it if it is installed in the system. Hence, the best is to include it as a meson subproject like we already do for freeDiameter. An open5gs fork is requried in order to have an extra patch adding a top-level CMakeList.txt in order to be able to includ eit from open5gs's meson build. Furthermore, this allows adding bugfixes to the subproject if any are found in the future. * [SMF] Initial metrics support * [SMF] Add metrics at gtp_node level * docs: Add tutorial documenting metrics with Prometheus --- .gitignore | 1 + configs/open5gs/smf.yaml.in | 14 + docs/_docs/tutorial/04-metrics-prometheus.md | 134 ++++++ docs/_pages/docs.md | 5 +- lib/app/ogs-context.c | 5 +- lib/app/ogs-context.h | 4 + lib/gtp/context.h | 1 + lib/meson.build | 1 + lib/metrics/context.c | 22 + lib/metrics/context.h | 72 +++ lib/metrics/meson.build | 66 +++ lib/metrics/ogs-metrics.h | 45 ++ lib/metrics/prometheus/context.c | 482 +++++++++++++++++++ lib/metrics/void/context.c | 99 ++++ meson_options.txt | 1 + src/smf/context.c | 62 ++- src/smf/context.h | 1 + src/smf/event.h | 3 +- src/smf/gtp-path.c | 39 +- src/smf/gtp-path.h | 9 + src/smf/init.c | 10 + src/smf/meson.build | 4 + src/smf/metrics.c | 196 ++++++++ src/smf/metrics.h | 70 +++ src/smf/smf-sm.c | 34 +- subprojects/prometheus-client-c.wrap | 7 + 26 files changed, 1346 insertions(+), 41 deletions(-) create mode 100644 docs/_docs/tutorial/04-metrics-prometheus.md create mode 100644 lib/metrics/context.c create mode 100644 lib/metrics/context.h create mode 100644 lib/metrics/meson.build create mode 100644 lib/metrics/ogs-metrics.h create mode 100644 lib/metrics/prometheus/context.c create mode 100644 lib/metrics/void/context.c create mode 100644 meson_options.txt create mode 100644 src/smf/metrics.c create mode 100644 src/smf/metrics.h create mode 100644 subprojects/prometheus-client-c.wrap diff --git a/.gitignore b/.gitignore index 1783da8be..de384f138 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # This directory is fetched during first build and is present in this directory subprojects/freeDiameter subprojects/libtins +subprojects/prometheus-client-c subprojects/usrsctp webui/.next diff --git a/configs/open5gs/smf.yaml.in b/configs/open5gs/smf.yaml.in index 47f898a30..d43639490 100644 --- a/configs/open5gs/smf.yaml.in +++ b/configs/open5gs/smf.yaml.in @@ -563,3 +563,17 @@ max: # handover: # duration: 500 time: + +# +# metrics: +# +# +# +# o Metrics Server(http://:9090) +# metrics: +# addr: 0.0.0.0 +# port: 9090 +# +metrics: + addr: 0.0.0.0 + port: 9090 diff --git a/docs/_docs/tutorial/04-metrics-prometheus.md b/docs/_docs/tutorial/04-metrics-prometheus.md new file mode 100644 index 000000000..8e36bb14b --- /dev/null +++ b/docs/_docs/tutorial/04-metrics-prometheus.md @@ -0,0 +1,134 @@ +--- +title: Metrics with Prometheus +--- + +#### 0. Introduction + +This tutorial explains how to export open5gs metrics to Prometheus, which can in +turn be used to visualize or export them to other systems such as Grafana or +StatsD. + +When this method is used, any open5gs program exporting metrics becomes a +Prometheus server, which is basically an HTTP server serving Prometheus data to +the Prometheus scrapper. + +Note: Only open5gs-smfd supports exporting metrics so far, though other may +hopefully follow soon. + +#### 1. Enable Prometheus support during build + +Open5GS programs use a generic internal API available in libogsmetrics. This +library implements the API based on configuration passed during open5gs build +time. By default, the library will be built using the `void` implementation, +which is basically a NO-OP implementation. + +In order to use the Prometheus, the `prometheus` metrics implementation needs to +be selected at build time: + +``` +meson configure -Dmetrics_impl=prometheus build +``` + +This will enable building the implementation under lib/metrics/prometheus/, +which uses: + +* prometheus-client-c project (libprom): To generate the Prometheus expected + output format of the metrics +* libmicrohttpd: To server the content generated by libprom as an HTTP server + +The `prometheus-client-c` project is not currently well maintained, and uses a +weird mixture of build systems, which makes it difficult to make it available in +most Linux distributions. As a result, a fork of the project is available under +Open5GS GitHub namespace, with an extra patch applied making it possible to +include it as a subproject, which will be fetched and built automatically when +building the prometheus libmetrics implementation. + +#### 2. Configuring for runtime + +By default the created Prometheus HTTP server will be listening on `0.0.0.0` +port `9090`. +This can be configured under the following config file options: + +``` +# +# metrics: +# +# +# +# o Metrics Server(http://:9090) +# metrics: +# addr: 0.0.0.0 +# port: 9090 +# +metrics: + addr: 0.0.0.0 + port: 9090 +``` + +Note: You may want to change the default IP address or port if you are running +the Prometheus scrapper in the same host, since it will also spawn its own +Prometheus server also in port 9090, which will collide. + +#### 3. Manual visualization + +Simply open the web browser at the following URL (changing IP address and port +as configured in previous section): +``` +http://127.0.0.1:9090/metrics +``` + +Note: URL `metrics/` (with a slash at the end) will not work. + +You should see some output similar to this one below: +``` +# HELP ues_active Active User Equipments +# TYPE ues_active gauge +ues_active 2 + +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1024 + +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1 + +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total gauge +process_cpu_seconds_total 0 + +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 3156643840 + +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 402433 + +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 23 +``` + +#### 3. Integration with Prometheus scrapper + +Sample Prometheus scrapper configuration (`~/prometheus.yml`): +``` +global: + scrape_interval: 10s + +scrape_configs: + - job_name: open5gs-smfd + static_configs: + - targets: ["192.168.1.140:9091"] +``` + +Where `192.168.1.140:9091` is the IP address and port where `open5gs-smfd` is +serving its metrics, as configured in above sections. + +The Prometheus scrapper can be easily started from a docker container: +``` +docker run -p 9090:9090 -v /prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus +``` + +Then open your browser to be able to visualize the data: `http://localhost:9090/graph` diff --git a/docs/_pages/docs.md b/docs/_pages/docs.md index 31cd4566d..86e502eeb 100644 --- a/docs/_pages/docs.md +++ b/docs/_pages/docs.md @@ -13,6 +13,7 @@ head_inline: "" - [Your First LTE](tutorial/01-your-first-lte) - [VoLTE Setup with Kamailio IMS and Open5GS](tutorial/02-VoLTE-setup) - [Dockerized VoLTE Setup](tutorial/03-VoLTE-dockerized) + - [Metrics with Prometheus](tutorial/04-metrics-prometheus) - Troubleshooting - [Simple Issues](troubleshoot/01-simple-issues) @@ -26,10 +27,10 @@ head_inline: "" - [MacOSX(Intel)](platform/06-macosx-intel) - [FreeBSD](platform/07-freebsd) - [Alpine](platform/08-alpine) - + - Hardware Specific Notes - [eNodeBs/gNodeBs tested on Open5GS](hardware/01-genodebs) - + - @infinitydon - [Open5GS on Amazon Elastic Kubernetes Service](https://aws.amazon.com/blogs/opensource/open-source-mobile-core-network-implementation-on-amazon-elastic-kubernetes-service/) - [Kubernetes Open5GS Deployment](https://dev.to/infinitydon/virtual-4g-simulation-using-kubernetes-and-gns3-3b7k?fbclid=IwAR1p99h13a-mCfejanbBQe0H0-jp5grXkn5mWf1WrTHf47UtegB2-UHGGZQ) diff --git a/lib/app/ogs-context.c b/lib/app/ogs-context.c index a76d02f28..f7e3ad5d9 100644 --- a/lib/app/ogs-context.c +++ b/lib/app/ogs-context.c @@ -226,9 +226,12 @@ static void app_context_prepare(void) */ self.time.handover.duration = ogs_time_from_msec(300); + /* Size of internal metrics pool (amount of ogs_metrics_spec_t) */ + self.metrics.max_specs = 512; + regenerate_all_timer_duration(); } - + static int app_context_validation(void) { if (self.parameter.no_ipv4 == 1 && diff --git a/lib/app/ogs-context.h b/lib/app/ogs-context.h index d0db2cd16..217f89646 100644 --- a/lib/app/ogs-context.h +++ b/lib/app/ogs-context.h @@ -175,6 +175,10 @@ typedef struct ogs_app_context_s { } handover; } time; + + struct metrics { + uint64_t max_specs; + } metrics; } ogs_app_context_t; int ogs_app_context_init(void); diff --git a/lib/gtp/context.h b/lib/gtp/context.h index 080a37cc6..079cb5647 100644 --- a/lib/gtp/context.h +++ b/lib/gtp/context.h @@ -65,6 +65,7 @@ typedef struct ogs_gtp_context_s { * PGW gateway. Some of members may not be used by the specific type of node */ typedef struct ogs_gtp_node_s { ogs_lnode_t node; /* A node of list_t */ + void *data_ptr; /* Can be used by app */ ogs_sockaddr_t *sa_list; /* Socket Address List Candidate */ diff --git a/lib/meson.build b/lib/meson.build index e3405bbd5..94b3ff0a1 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -21,6 +21,7 @@ subdir('core') subdir('ipfw') subdir('crypt') subdir('app') +subdir('metrics') subdir('sctp') subdir('tun') subdir('dbi') diff --git a/lib/metrics/context.c b/lib/metrics/context.c new file mode 100644 index 000000000..24fea8d5c --- /dev/null +++ b/lib/metrics/context.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + * + * This file is part of Open5GS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "ogs-metrics.h" + +int __ogs_metrics_domain; diff --git a/lib/metrics/context.h b/lib/metrics/context.h new file mode 100644 index 000000000..ce8b1819b --- /dev/null +++ b/lib/metrics/context.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + * + * This file is part of Open5GS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if !defined(OGS_METRICS_INSIDE) && !defined(OGS_METRICS_COMPILATION) +#error "This header cannot be included directly." +#endif + +#ifndef OGS_METRICS_CONTEXT_H +#define OGS_METRICS_CONTEXT_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum ogs_metrics_metric_type_s { + OGS_METRICS_METRIC_TYPE_COUNTER, + OGS_METRICS_METRIC_TYPE_GAUGE, +} ogs_metrics_metric_type_t; + +typedef struct ogs_metrics_context_s ogs_metrics_context_t; +void ogs_metrics_context_init(void); +void ogs_metrics_context_open(ogs_metrics_context_t *ctx); +void ogs_metrics_context_close(ogs_metrics_context_t *ctx); +void ogs_metrics_context_final(void); +ogs_metrics_context_t *ogs_metrics_self(void); +int ogs_metrics_context_parse_config(void); + +typedef struct ogs_metrics_spec_s ogs_metrics_spec_t; +ogs_metrics_spec_t *ogs_metrics_spec_new( + ogs_metrics_context_t *ctx, ogs_metrics_metric_type_t type, + const char *name, const char *description, + int initial_val, unsigned int num_labels, const char **labels); +void ogs_metrics_spec_free(ogs_metrics_spec_t *spec); + +typedef struct ogs_metrics_inst_s ogs_metrics_inst_t; +ogs_metrics_inst_t *ogs_metrics_inst_new( + ogs_metrics_spec_t *spec, + unsigned int num_labels, const char **label_values); +void ogs_metrics_inst_free(ogs_metrics_inst_t *inst); +void ogs_metrics_inst_set(ogs_metrics_inst_t *inst, int val); +void ogs_metrics_inst_reset(ogs_metrics_inst_t *inst); +void ogs_metrics_inst_add(ogs_metrics_inst_t *inst, int val); +static inline void ogs_metrics_inst_inc(ogs_metrics_inst_t *inst) +{ + ogs_metrics_inst_add(inst, 1); +} +static inline void ogs_metrics_inst_dec(ogs_metrics_inst_t *inst) +{ + ogs_metrics_inst_add(inst, -1); +} + +#ifdef __cplusplus +} +#endif + +#endif /* OGS_METRICS_CONTEXT_H */ diff --git a/lib/metrics/meson.build b/lib/metrics/meson.build new file mode 100644 index 000000000..ca210b6ac --- /dev/null +++ b/lib/metrics/meson.build @@ -0,0 +1,66 @@ +# Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + +# This file is part of Open5GS. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +libmetrics_file_list = ''' + ogs-metrics.h + context.h + context.c +''' +libmetrics_dependencies = [libcore_dep, libapp_dep] + + +metrics_impl_optval = get_option('metrics_impl') + +if metrics_impl_optval == 'prometheus' + # Note: This requires meson >= 0.51.0: + # 0.47.0: {'check arg in run_command'} + # 0.50.0: {'CMake Module'} + # 0.51.0: {'subproject'} + + libmicrohttpd_dep = dependency('libmicrohttpd', version: '>=0.9.40') + + cmake = import('cmake') + prometheus_client_c_proj = cmake.subproject('prometheus-client-c') + # generated cmake subproject seems to include + # open5gs/subprojects/prometheus-client-c/__CMake_build as include, which + # doesn't exist and fail: + missing_include_dir = join_paths(meson.current_source_dir(), '../../subprojects/prometheus-client-c/__CMake_build') + run_command('mkdir', '-p', missing_include_dir, check: true) + libprom_dep = prometheus_client_c_proj.dependency('prom') + + libmetrics_dependencies = libmetrics_dependencies + [libprom_dep, libmicrohttpd_dep] + libmetrics_file_list = libmetrics_file_list + ' prometheus/context.c' +else + libmetrics_file_list = libmetrics_file_list + ' void/context.c' +endif + +libmetrics_sources = files(libmetrics_file_list.split()) + +libmetrics_inc = include_directories('.') + +libmetrics = library('ogsmetrics', + sources : libmetrics_sources, + version : libogslib_version, + c_args : '-DOGS_METRICS_COMPILATION', + include_directories : [libmetrics_inc, libinc], + dependencies : libmetrics_dependencies, + install : true) + +libmetrics_dep = declare_dependency( + link_with : libmetrics, + include_directories : [libmetrics_inc, libinc], + dependencies : libmetrics_dependencies) diff --git a/lib/metrics/ogs-metrics.h b/lib/metrics/ogs-metrics.h new file mode 100644 index 000000000..aec696f7a --- /dev/null +++ b/lib/metrics/ogs-metrics.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + * + * This file is part of Open5GS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef OGS_METRICS_H +#define OGS_METRICS_H + +#include "ogs-core.h" +#include "ogs-app.h" + +#define OGS_METRICS_INSIDE + +#include "metrics/context.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#undef OGS_METRICS_INSIDE + +extern int __ogs_metrics_domain; + +#undef OGS_LOG_DOMAIN +#define OGS_LOG_DOMAIN __ogs_metrics_domain + +#ifdef __cplusplus +} +#endif + +#endif /* OGS_METRICS_H */ diff --git a/lib/metrics/prometheus/context.c b/lib/metrics/prometheus/context.c new file mode 100644 index 000000000..d74daf1dc --- /dev/null +++ b/lib/metrics/prometheus/context.c @@ -0,0 +1,482 @@ +/* + * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + * + * This file is part of Open5GS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "ogs-metrics.h" + +#include /* AI_PASSIVE */ +#include "prom.h" +#include "microhttpd.h" + +#define DEFAULT_PROMETHEUS_HTTP_PORT 9090 +#define MAX_LABELS 8 + +typedef struct ogs_metrics_context_s { + //uint32_t port; /* METRICS local port */ + + //ogs_sock_t *metricsc_sock; /* METRICS IPv4 Socket */ + ogs_socknode_t node; + + ogs_list_t spec_list; + struct MHD_Daemon *mhd_server; +} ogs_metrics_context_t; + +typedef struct ogs_metrics_spec_s { + ogs_metrics_context_t *ctx; /* backpointer */ + ogs_list_t entry; /* included in ogs_metrics_context_t */ + ogs_metrics_metric_type_t type; + char *name; + char *description; + int initial_val; + ogs_list_t inst_list; /* list of ogs_metrics_instance_t */ + unsigned int num_labels; + char *labels[MAX_LABELS]; + prom_metric_t *prom; +} ogs_metrics_spec_t; + +typedef struct ogs_metrics_inst_s { + ogs_metrics_spec_t *spec; /* backpointer */ + ogs_list_t entry; /* included in ogs_metrics_spec_t spec */ + unsigned int num_labels; + char *label_values[MAX_LABELS]; +} ogs_metrics_inst_t; + +static ogs_metrics_context_t self; +static int context_initialized = 0; +static OGS_POOL(metrics_spec_pool, ogs_metrics_spec_t); + +void ogs_metrics_context_init(void) +{ + ogs_assert(context_initialized == 0); + + ogs_log_install_domain(&__ogs_metrics_domain, "metrics", ogs_core()->log.level); + + ogs_pool_init(&metrics_spec_pool, ogs_app()->metrics.max_specs); + + /* Initialize METRICS context */ + memset(&self, 0, sizeof(ogs_metrics_context_t)); + ogs_list_init(&self.spec_list); + prom_collector_registry_default_init(); + + context_initialized = 1; +} + +void ogs_metrics_context_final(void) +{ + ogs_metrics_spec_t *spec = NULL, *next = NULL; + ogs_assert(context_initialized == 1); + + if (self.mhd_server) + ogs_metrics_context_close(&self); + + ogs_list_for_each_entry_safe(&self.spec_list, next, spec, entry) { + ogs_metrics_spec_free(spec); + } + prom_collector_registry_destroy(PROM_COLLECTOR_REGISTRY_DEFAULT); + + if (self.node.addr) { + ogs_freeaddrinfo(self.node.addr); + self.node.addr = NULL; + } + + ogs_pool_final(&metrics_spec_pool); + + context_initialized = 0; +} + +ogs_metrics_context_t *ogs_metrics_self(void) +{ + return &self; +} + +int ogs_metrics_context_parse_config(void) +{ + int family = AF_UNSPEC; + const char *hostname = NULL; + uint16_t port = DEFAULT_PROMETHEUS_HTTP_PORT; + ogs_sockaddr_t *addr = NULL; + yaml_document_t *document = NULL; + ogs_yaml_iter_t root_iter; + const char *v; + + document = ogs_app()->document; + ogs_assert(document); + + ogs_yaml_iter_init(&root_iter, document); + while (ogs_yaml_iter_next(&root_iter)) { + const char *root_key = ogs_yaml_iter_key(&root_iter); + ogs_assert(root_key); + if (!strcmp(root_key, "metrics")) { + ogs_yaml_iter_t local_iter; + ogs_yaml_iter_recurse(&root_iter, &local_iter); + while (ogs_yaml_iter_next(&local_iter)) { + const char *local_key = ogs_yaml_iter_key(&local_iter); + if (!strcmp(local_key, "addr")) { + if ((v = ogs_yaml_iter_value(&local_iter))) + hostname = v; + } else if (!strcmp(local_key, "port")) { + if ((v = ogs_yaml_iter_value(&local_iter))) + port = atoi(v); + } + } + } + } + ogs_assert(OGS_OK == + ogs_addaddrinfo(&addr, family, hostname, port, AI_PASSIVE)); + if (self.node.addr) + ogs_freeaddrinfo(self.node.addr); + ogs_assert(OGS_OK == ogs_copyaddrinfo(&self.node.addr, addr)); + ogs_freeaddrinfo(addr); + return OGS_OK; +} + +static void mhd_server_run(short when, ogs_socket_t fd, void *data) +{ + struct MHD_Daemon *mhd_daemon = data; + + ogs_assert(mhd_daemon); + MHD_run(mhd_daemon); +} + +static void mhd_server_notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) +{ + struct MHD_Daemon *mhd_daemon = NULL; + MHD_socket mhd_socket = INVALID_SOCKET; + + const union MHD_ConnectionInfo *mhd_info = NULL; + struct { + ogs_poll_t *read; + } poll; + + switch (toe) { + case MHD_CONNECTION_NOTIFY_STARTED: + mhd_info = MHD_get_connection_info( + connection, MHD_CONNECTION_INFO_DAEMON); + ogs_assert(mhd_info); + mhd_daemon = mhd_info->daemon; + ogs_assert(mhd_daemon); + + mhd_info = MHD_get_connection_info( + connection, MHD_CONNECTION_INFO_CONNECTION_FD); + ogs_assert(mhd_info); + mhd_socket = mhd_info->connect_fd; + ogs_assert(mhd_socket != INVALID_SOCKET); + + poll.read = ogs_pollset_add(ogs_app()->pollset, + OGS_POLLIN, mhd_socket, mhd_server_run, mhd_daemon); + ogs_assert(poll.read); + *socket_context = poll.read; + break; + case MHD_CONNECTION_NOTIFY_CLOSED: + poll.read = *socket_context; + if (poll.read) + ogs_pollset_remove(poll.read); + break; + } +} + +#if MHD_VERSION >= 0x00097001 +typedef enum MHD_Result _MHD_Result; +#else +typedef int _MHD_Result; +#endif + +static _MHD_Result mhd_server_access_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, const char *version, + const char *upload_data, size_t *upload_data_size, void **con_cls) { + + const char *buf; + struct MHD_Response *rsp; + int ret; + + if (strcmp(method, "GET") != 0) { + buf = "Invalid HTTP Method\n"; + rsp = MHD_create_response_from_buffer(strlen(buf), (void *)buf, MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_response(connection, MHD_HTTP_BAD_REQUEST, rsp); + MHD_destroy_response(rsp); + return ret; + } + if (strcmp(url, "/") == 0) { + buf = "OK\n"; + rsp = MHD_create_response_from_buffer(strlen(buf), (void *)buf, MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_response(connection, MHD_HTTP_OK, rsp); + MHD_destroy_response(rsp); + return ret; + } + if (strcmp(url, "/metrics") == 0) { + buf = prom_collector_registry_bridge(PROM_COLLECTOR_REGISTRY_DEFAULT); + rsp = MHD_create_response_from_buffer(strlen(buf), (void *)buf, MHD_RESPMEM_MUST_FREE); + ret = MHD_queue_response(connection, MHD_HTTP_OK, rsp); + MHD_destroy_response(rsp); + return ret; + } + buf = "Bad Request\n"; + rsp = MHD_create_response_from_buffer(strlen(buf), (void *)buf, MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_response(connection, MHD_HTTP_BAD_REQUEST, rsp); + MHD_destroy_response(rsp); + return ret; +} + +static int ogs_metrics_context_mhd_server_start(ogs_metrics_context_t *ctx) +{ +#define MAX_NUM_OF_MHD_OPTION_ITEM 8 + struct MHD_OptionItem mhd_ops[MAX_NUM_OF_MHD_OPTION_ITEM]; + const union MHD_DaemonInfo *mhd_info = NULL; + int index = 0; + char buf[OGS_ADDRSTRLEN]; + ogs_sockaddr_t *addr = ctx->node.addr; + char *hostname = NULL; + + ogs_assert(addr); + +#if MHD_VERSION >= 0x00095300 + unsigned int mhd_flags = MHD_USE_ERROR_LOG; +#else + unsigned int mhd_flags = MHD_USE_DEBUG; +#endif + +#if MHD_VERSION >= 0x00095300 + mhd_flags |= MHD_ALLOW_SUSPEND_RESUME; +#elif MHD_VERSION >= 0x00093400 + mhd_flags |= MHD_USE_SUSPEND_RESUME; +#else + mhd_flags |= MHD_USE_PIPE_FOR_SHUTDOWN; +#endif + + if (addr->ogs_sa_family == AF_INET6) + mhd_flags |= MHD_USE_IPv6; + + mhd_ops[index].option = MHD_OPTION_NOTIFY_CONNECTION; + mhd_ops[index].value = (intptr_t)&mhd_server_notify_connection; + mhd_ops[index].ptr_value = NULL; + index++; + + mhd_ops[index].option = MHD_OPTION_SOCK_ADDR; + mhd_ops[index].value = 0; + mhd_ops[index].ptr_value = (void *)&addr->sa; + index++; + + mhd_ops[index].option = MHD_OPTION_END; + mhd_ops[index].value = 0; + mhd_ops[index].ptr_value = NULL; + index++; + + if (ctx->mhd_server) { + ogs_error("Prometheus HTTP server is already opened!"); + return OGS_ERROR; + } + + ctx->mhd_server = MHD_start_daemon( + mhd_flags, + 0, + NULL, NULL, + mhd_server_access_handler, ctx, + MHD_OPTION_ARRAY, mhd_ops, + MHD_OPTION_END); + if (!ctx->mhd_server) { + ogs_error("Cannot start Prometheus HTTP server"); + return OGS_ERROR; + } + + /* Setup poll for server listening socket */ + mhd_info = MHD_get_daemon_info(ctx->mhd_server, MHD_DAEMON_INFO_LISTEN_FD); + ogs_assert(mhd_info); + + ctx->node.poll = ogs_pollset_add(ogs_app()->pollset, + OGS_POLLIN, mhd_info->listen_fd, mhd_server_run, ctx->mhd_server); + ogs_assert(ctx->node.poll); + + hostname = ogs_gethostname(addr); + if (hostname) + ogs_info("Prometheus mhd_server() [%s]:%d", hostname, OGS_PORT(addr)); + else + ogs_info("Prometheus mhd_server() [%s]:%d", OGS_ADDR(addr, buf), OGS_PORT(addr)); + return OGS_OK; +} +void ogs_metrics_context_open(ogs_metrics_context_t *ctx) +{ + ogs_assert(ogs_metrics_context_mhd_server_start(ctx) == OGS_OK); +} + +static int ogs_metrics_context_mhd_server_stop(ogs_metrics_context_t *ctx) +{ + ogs_assert(ctx); + + if (ctx->node.poll) + ogs_pollset_remove(ctx->node.poll); + + if (ctx->mhd_server) { + MHD_stop_daemon(ctx->mhd_server); + ctx->mhd_server = NULL; + } + return OGS_OK; +} +void ogs_metrics_context_close(ogs_metrics_context_t *ctx) +{ + ogs_assert(ogs_metrics_context_mhd_server_stop(ctx) == OGS_OK); +} + +ogs_metrics_spec_t *ogs_metrics_spec_new( + ogs_metrics_context_t *ctx, ogs_metrics_metric_type_t type, + const char *name, const char *description, + int initial_val, unsigned int num_labels, const char ** labels) +{ + ogs_metrics_spec_t *spec; + unsigned int i; + + ogs_assert(name); + ogs_assert(description); + ogs_assert(num_labels <= MAX_LABELS); + + ogs_pool_alloc(&metrics_spec_pool, &spec); + ogs_assert(spec); + memset(spec, 0, sizeof *spec); + spec->ctx = ctx; + ogs_list_init(&spec->inst_list); + spec->type = type; + spec->name = ogs_strdup(name); + spec->description = ogs_strdup(description); + spec->initial_val = initial_val; + spec->num_labels = num_labels; + for (i = 0; i < num_labels; i++) { + ogs_assert(labels[i]); + spec->labels[i] = ogs_strdup(labels[i]); + } + + switch (type) { + case OGS_METRICS_METRIC_TYPE_COUNTER: + spec->prom = prom_counter_new(spec->name, spec->description, + spec->num_labels, (const char **)spec->labels); + break; + case OGS_METRICS_METRIC_TYPE_GAUGE: + spec->prom = prom_gauge_new(spec->name, spec->description, + spec->num_labels, (const char **)spec->labels); + break; + default: + ogs_assert_if_reached(); + break; + } + prom_collector_registry_must_register_metric(spec->prom); + + ogs_list_add(&ctx->spec_list, &spec->entry); + return spec; +} + +void ogs_metrics_spec_free(ogs_metrics_spec_t *spec) +{ + ogs_metrics_inst_t *inst = NULL, *next = NULL; + unsigned int i; + + ogs_list_remove(&spec->ctx->spec_list, spec); + + ogs_list_for_each_entry_safe(&spec->inst_list, next, inst, entry) { + ogs_metrics_inst_free(inst); + } + + ogs_free(spec->name); + ogs_free(spec->description); + for (i = 0; i < spec->num_labels; i++) + ogs_free(spec->labels[i]); + + ogs_pool_free(&metrics_spec_pool, spec); +} + + +ogs_metrics_inst_t *ogs_metrics_inst_new( + ogs_metrics_spec_t *spec, + unsigned int num_labels, const char **label_values) +{ + ogs_metrics_inst_t *inst; + unsigned int i; + + ogs_assert(spec); + ogs_assert(num_labels == spec->num_labels); + + inst = ogs_calloc(1, sizeof(ogs_metrics_inst_t)); + ogs_assert(inst); + inst->spec = spec; + inst->num_labels = num_labels; + for (i = 0; i < num_labels; i++) { + ogs_assert(label_values[i]); + inst->label_values[i] = ogs_strdup(label_values[i]); + } + ogs_list_add(&spec->inst_list, &inst->entry); + ogs_metrics_inst_reset(inst); + return inst; +} + +void ogs_metrics_inst_free(ogs_metrics_inst_t *inst) +{ + unsigned int i; + + ogs_list_remove(&inst->spec->inst_list, inst); + + for (i = 0; i < inst->num_labels; i++) + ogs_free(inst->label_values[i]); + + ogs_free(inst); +} + +void ogs_metrics_inst_set(ogs_metrics_inst_t *inst, int val) +{ + switch (inst->spec->type) { + case OGS_METRICS_METRIC_TYPE_GAUGE: + prom_gauge_set(inst->spec->prom, (double)val, (const char **)inst->label_values); + break; + default: + ogs_assert_if_reached(); + break; + } +} + +void ogs_metrics_inst_reset(ogs_metrics_inst_t *inst) +{ + switch (inst->spec->type) { + case OGS_METRICS_METRIC_TYPE_COUNTER: + prom_counter_add(inst->spec->prom, 0.0, (const char **)inst->label_values); + break; + case OGS_METRICS_METRIC_TYPE_GAUGE: + prom_gauge_set(inst->spec->prom, (double)inst->spec->initial_val, (const char **)inst->label_values); + break; + default: + /* Other types have no way to reset */ + break; + } +} + +void ogs_metrics_inst_add(ogs_metrics_inst_t *inst, int val) +{ + switch (inst->spec->type) { + case OGS_METRICS_METRIC_TYPE_COUNTER: + ogs_assert(val >= 0); + prom_counter_add(inst->spec->prom, (double)val, (const char **)inst->label_values); + break; + case OGS_METRICS_METRIC_TYPE_GAUGE: + if (val >= 0) + prom_gauge_add(inst->spec->prom, (double)val, (const char **)inst->label_values); + else + prom_gauge_sub(inst->spec->prom, (double)-1.0*(double)val, (const char **)inst->label_values); + break; + default: + ogs_assert_if_reached(); + break; + } +} diff --git a/lib/metrics/void/context.c b/lib/metrics/void/context.c new file mode 100644 index 000000000..ac7e41dd0 --- /dev/null +++ b/lib/metrics/void/context.c @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH + * + * This file is part of Open5GS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "ogs-metrics.h" + +typedef struct ogs_metrics_context_s { + int unused; +} ogs_metrics_context_t; +typedef struct ogs_metrics_spec_s { + int unused; +} ogs_metrics_spec_t; +typedef struct ogs_metrics_inst_s { + int unused; +} ogs_metrics_inst_t; + +static ogs_metrics_context_t self; +static int context_initialized = 0; + +void ogs_metrics_context_init(void) +{ + ogs_assert(context_initialized == 0); + ogs_log_install_domain(&__ogs_metrics_domain, "metrics", ogs_core()->log.level); + context_initialized = 1; +} + +void ogs_metrics_context_final(void) +{ + ogs_assert(context_initialized == 1); + context_initialized = 0; +} + +ogs_metrics_context_t *ogs_metrics_self(void) +{ + return &self; +} + +int ogs_metrics_context_parse_config(void) +{ + return OGS_OK; +} + +void ogs_metrics_context_open(ogs_metrics_context_t *ctx) +{ +} + +void ogs_metrics_context_close(ogs_metrics_context_t *ctx) +{ +} + +ogs_metrics_spec_t *ogs_metrics_spec_new( + ogs_metrics_context_t *ctx, ogs_metrics_metric_type_t type, + const char *name, const char *description, + int initial_val, unsigned int num_labels, const char ** labels) +{ + return (ogs_metrics_spec_t *)&self; +} + +void ogs_metrics_spec_free(ogs_metrics_spec_t *spec) +{ +} + +ogs_metrics_inst_t *ogs_metrics_inst_new( + ogs_metrics_spec_t *spec, + unsigned int num_labels, const char **label_values) +{ + return (ogs_metrics_inst_t *)&self; +} + +void ogs_metrics_inst_free(ogs_metrics_inst_t *inst) +{ +} + +void ogs_metrics_inst_set(ogs_metrics_inst_t *inst, int val) +{ +} + +void ogs_metrics_inst_reset(ogs_metrics_inst_t *inst) +{ +} + +void ogs_metrics_inst_add(ogs_metrics_inst_t *inst, int val) +{ +} diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 000000000..9350a6b8f --- /dev/null +++ b/meson_options.txt @@ -0,0 +1 @@ +option('metrics_impl', type : 'combo', choices : ['void', 'prometheus'], value : 'void', description : 'libogsmetrics implementation') diff --git a/src/smf/context.c b/src/smf/context.c index c267debbe..6a7372e90 100644 --- a/src/smf/context.c +++ b/src/smf/context.c @@ -821,12 +821,10 @@ int smf_context_parse_config(void) return OGS_OK; } -smf_ue_t *smf_ue_add_by_supi(char *supi) +static smf_ue_t *smf_ue_add(void) { smf_ue_t *smf_ue = NULL; - ogs_assert(supi); - ogs_pool_alloc(&smf_ue_pool, &smf_ue); if (!smf_ue) { ogs_error("Maximum number of smf_ue[%lld] reached", @@ -837,45 +835,45 @@ smf_ue_t *smf_ue_add_by_supi(char *supi) ogs_list_init(&smf_ue->sess_list); + ogs_list_add(&self.smf_ue_list, smf_ue); + + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_UES_ACTIVE); + ogs_info("[Added] Number of SMF-UEs is now %d", + ogs_list_count(&self.smf_ue_list)); + return smf_ue; +} + +smf_ue_t *smf_ue_add_by_supi(char *supi) +{ + smf_ue_t *smf_ue; + + ogs_assert(supi); + + if ((smf_ue = smf_ue_add()) == NULL) + return NULL; + smf_ue->supi = ogs_strdup(supi); ogs_assert(smf_ue->supi); ogs_hash_set(self.supi_hash, smf_ue->supi, strlen(smf_ue->supi), smf_ue); - ogs_list_add(&self.smf_ue_list, smf_ue); - - ogs_info("[Added] Number of SMF-UEs is now %d", - ogs_list_count(&self.smf_ue_list)); - return smf_ue; } smf_ue_t *smf_ue_add_by_imsi(uint8_t *imsi, int imsi_len) { - smf_ue_t *smf_ue = NULL; + smf_ue_t *smf_ue; ogs_assert(imsi); ogs_assert(imsi_len); - ogs_pool_alloc(&smf_ue_pool, &smf_ue); - if (!smf_ue) { - ogs_error("Maximum number of smf_ue[%lld] reached", - (long long)ogs_app()->max.ue); - return NULL; - } - memset(smf_ue, 0, sizeof *smf_ue); - - ogs_list_init(&smf_ue->sess_list); + if ((smf_ue = smf_ue_add()) == NULL) + return NULL;; smf_ue->imsi_len = imsi_len; memcpy(smf_ue->imsi, imsi, smf_ue->imsi_len); ogs_buffer_to_bcd(smf_ue->imsi, smf_ue->imsi_len, smf_ue->imsi_bcd); ogs_hash_set(self.imsi_hash, smf_ue->imsi, smf_ue->imsi_len, smf_ue); - ogs_list_add(&self.smf_ue_list, smf_ue); - - ogs_info("[Added] Number of SMF-UEs is now %d", - ogs_list_count(&self.smf_ue_list)); - return smf_ue; } @@ -898,6 +896,7 @@ void smf_ue_remove(smf_ue_t *smf_ue) ogs_pool_free(&smf_ue_pool, smf_ue); + smf_metrics_inst_global_dec(SMF_METR_GLOB_GAUGE_UES_ACTIVE); ogs_info("[Removed] Number of SMF-UEs is now %d", ogs_list_count(&self.smf_ue_list)); } @@ -1146,6 +1145,7 @@ smf_sess_t *smf_sess_add_by_gtp1_message(ogs_gtp1_message_t *message) sess = smf_sess_add_by_apn(smf_ue, apn, req->rat_type.u8); sess->gtp.version = 1; + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_GTP1_PDPCTXS_ACTIVE); return sess; } @@ -1210,6 +1210,7 @@ smf_sess_t *smf_sess_add_by_gtp2_message(ogs_gtp2_message_t *message) sess = smf_sess_add_by_apn(smf_ue, apn, req->rat_type.u8); sess->gtp.version = 2; + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_GTP2_SESSIONS_ACTIVE); return sess; } @@ -1620,9 +1621,16 @@ void smf_sess_remove(smf_sess_t *sess) smf_qfi_pool_final(sess); smf_pf_precedence_pool_final(sess); - ogs_pool_free(&smf_sess_pool, sess); - + switch (sess->gtp.version) { + case 1: + smf_metrics_inst_global_dec(SMF_METR_GLOB_GAUGE_GTP1_PDPCTXS_ACTIVE); + break; + case 2: + smf_metrics_inst_global_dec(SMF_METR_GLOB_GAUGE_GTP2_SESSIONS_ACTIVE); + break; + } stats_remove_smf_session(); + ogs_pool_free(&smf_sess_pool, sess); } void smf_sess_remove_all(smf_ue_t *smf_ue) @@ -2203,6 +2211,7 @@ smf_bearer_t *smf_bearer_add(smf_sess_t *sess) ogs_list_add(&sess->bearer_list, bearer); + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_BEARERS_ACTIVE); return bearer; } @@ -2244,6 +2253,7 @@ int smf_bearer_remove(smf_bearer_t *bearer) ogs_pool_free(&smf_bearer_pool, bearer); + smf_metrics_inst_global_dec(SMF_METR_GLOB_GAUGE_BEARERS_ACTIVE); return OGS_OK; } @@ -2864,12 +2874,14 @@ void smf_pf_precedence_pool_final(smf_sess_t *sess) static void stats_add_smf_session(void) { + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_SESSIONS_ACTIVE); num_of_smf_sess = num_of_smf_sess + 1; ogs_info("[Added] Number of SMF-Sessions is now %d", num_of_smf_sess); } static void stats_remove_smf_session(void) { + smf_metrics_inst_global_dec(SMF_METR_GLOB_GAUGE_SESSIONS_ACTIVE); num_of_smf_sess = num_of_smf_sess - 1; ogs_info("[Removed] Number of SMF-Sessions is now %d", num_of_smf_sess); } diff --git a/src/smf/context.h b/src/smf/context.h index eee32a941..c2b765d50 100644 --- a/src/smf/context.h +++ b/src/smf/context.h @@ -36,6 +36,7 @@ #include "timer.h" #include "smf-sm.h" +#include "metrics.h" #if HAVE_NET_IF_H #include diff --git a/src/smf/event.h b/src/smf/event.h index 36c700c7b..03ac91fa1 100644 --- a/src/smf/event.h +++ b/src/smf/event.h @@ -37,6 +37,7 @@ typedef struct ogs_diam_gy_message_s ogs_diam_gy_message_t; typedef struct ogs_diam_s6b_message_s ogs_diam_s6b_message_t; typedef struct smf_sess_s smf_sess_t; typedef struct smf_upf_s smf_upf_t; +typedef struct smf_gtp_node_s smf_gtp_node_t; typedef struct ogs_sbi_request_s ogs_sbi_request_t; typedef struct ogs_sbi_response_s ogs_sbi_response_t; typedef struct ogs_sbi_message_s ogs_sbi_message_t; @@ -77,7 +78,7 @@ typedef struct smf_event_s { ogs_pkbuf_t *pkbuf; int timer_id; - ogs_gtp_node_t *gnode; + smf_gtp_node_t *gnode; ogs_gtp_xact_t *gtp_xact; ogs_pfcp_node_t *pfcp_node; diff --git a/src/smf/gtp-path.c b/src/smf/gtp-path.c index 102454927..12abdf5df 100644 --- a/src/smf/gtp-path.c +++ b/src/smf/gtp-path.c @@ -41,6 +41,8 @@ #include "s5c-build.h" #include "gn-build.h" +static OGS_POOL(smf_gtp_node_pool, smf_gtp_node_t); + static bool check_if_router_solicit(ogs_pkbuf_t *pkbuf); static void send_router_advertisement(smf_sess_t *sess, uint8_t *ip6_dst); @@ -91,9 +93,11 @@ static void _gtpv1v2_c_recv_cb(short when, ogs_socket_t fd, void *data) gnode = ogs_gtp_node_add_by_addr(&smf_self()->sgw_s5c_list, &from); ogs_assert(gnode); gnode->sock = data; + smf_gtp_node_new(gnode); + smf_metrics_inst_global_inc(SMF_METR_GLOB_GAUGE_GTP_PEERS_ACTIVE); } ogs_assert(e); - e->gnode = gnode; + e->gnode = gnode->data_ptr; /* smf_gtp_node_t */ e->pkbuf = pkbuf; rv = ogs_queue_push(ogs_app()->queue, e); @@ -294,6 +298,8 @@ int smf_gtp_open(void) ogs_gtp_self()->link_local_addr = ogs_link_local_addr_by_sa(ogs_gtp_self()->gtpu_addr6); + ogs_pool_init(&smf_gtp_node_pool, ogs_app()->pool.gtp_node); + return OGS_OK; } @@ -306,6 +312,37 @@ void smf_gtp_close(void) ogs_socknode_remove_all(&ogs_gtp_self()->gtpc_list6); ogs_socknode_remove_all(&ogs_gtp_self()->gtpu_list); + + ogs_pool_final(&smf_gtp_node_pool); +} + +smf_gtp_node_t *smf_gtp_node_new(ogs_gtp_node_t *gnode) +{ + smf_gtp_node_t *smf_gnode = NULL; + char addr[OGS_ADDRSTRLEN]; + + ogs_pool_alloc(&smf_gtp_node_pool, &smf_gnode); + ogs_expect_or_return_val(smf_gnode, NULL); + memset(smf_gnode, 0, sizeof(smf_gtp_node_t)); + + addr[0] = '\0'; + ogs_assert(gnode->sa_list); + ogs_inet_ntop(&gnode->sa_list[0].sa, addr, sizeof(addr)); + ogs_assert(smf_metrics_init_inst_gtp_node(smf_gnode->metrics, addr) + == OGS_OK); + + smf_gnode->gnode = gnode; + gnode->data_ptr = smf_gnode; /* Set backpointer */ + return smf_gnode; +} + +void smf_gtp_node_free(smf_gtp_node_t *smf_gnode) +{ + ogs_assert(smf_gnode); + if (smf_gnode->gnode) + smf_gnode->gnode->data_ptr = NULL; /*Drop backpointer */ + smf_metrics_free_inst_gtp_node(smf_gnode->metrics); + ogs_pool_free(&smf_gtp_node_pool, smf_gnode); } int smf_gtp1_send_create_pdp_context_response( diff --git a/src/smf/gtp-path.h b/src/smf/gtp-path.h index 4b3874fc2..78c298c09 100644 --- a/src/smf/gtp-path.h +++ b/src/smf/gtp-path.h @@ -29,6 +29,15 @@ extern "C" { int smf_gtp_open(void); void smf_gtp_close(void); +typedef struct smf_gtp_node_s { + ogs_gtp_node_t *gnode; + ogs_metrics_inst_t *metrics[_SMF_METR_GTP_NODE_MAX]; +} smf_gtp_node_t; + +smf_gtp_node_t *smf_gtp_node_new(ogs_gtp_node_t *gnode); +void smf_gtp_node_free(smf_gtp_node_t *smf_gnode); + + int smf_gtp1_send_create_pdp_context_response( smf_sess_t *sess, ogs_gtp_xact_t *xact); int smf_gtp1_send_delete_pdp_context_response( diff --git a/src/smf/init.c b/src/smf/init.c index 170258466..0280f4bb3 100644 --- a/src/smf/init.c +++ b/src/smf/init.c @@ -22,6 +22,7 @@ #include "gtp-path.h" #include "pfcp-path.h" #include "sbi-path.h" +#include "metrics.h" static ogs_thread_t *thread; static void smf_main(void *data); @@ -32,6 +33,7 @@ int smf_initialize() { int rv; + ogs_metrics_context_init(); ogs_gtp_context_init(ogs_app()->pool.nf * OGS_MAX_NUM_OF_GTPU_RESOURCE); ogs_pfcp_context_init(); @@ -54,6 +56,9 @@ int smf_initialize() rv = ogs_sbi_context_parse_config("smf", "nrf"); if (rv != OGS_OK) return rv; + rv = ogs_metrics_context_parse_config(); + if (rv != OGS_OK) return rv; + rv = smf_context_parse_config(); if (rv != OGS_OK) return rv; @@ -64,6 +69,9 @@ int smf_initialize() rv = ogs_pfcp_ue_pool_generate(); if (rv != OGS_OK) return rv; + rv = smf_metrics_open(); + if (rv != 0) return OGS_ERROR; + rv = smf_fd_init(); if (rv != 0) return OGS_ERROR; @@ -117,6 +125,7 @@ void smf_terminate(void) smf_gtp_close(); smf_pfcp_close(); smf_sbi_close(); + smf_metrics_close(); smf_fd_final(); @@ -125,6 +134,7 @@ void smf_terminate(void) ogs_pfcp_context_final(); ogs_sbi_context_final(); ogs_gtp_context_final(); + ogs_metrics_context_final(); ogs_pfcp_xact_final(); ogs_gtp_xact_final(); diff --git a/src/smf/meson.build b/src/smf/meson.build index 1bd0b7d08..dc06cac43 100644 --- a/src/smf/meson.build +++ b/src/smf/meson.build @@ -67,6 +67,7 @@ libsmf_sources = files(''' ngap-build.h ngap-handler.h ngap-path.h + metrics.h init.c event.c @@ -107,11 +108,13 @@ libsmf_sources = files(''' ngap-build.c ngap-handler.c ngap-path.c + metrics.c '''.split()) libsmf = static_library('smf', sources : libsmf_sources, dependencies : [libapp_dep, + libmetrics_dep, libsbi_dep, libngap_dep, libnas_5gs_dep, @@ -125,6 +128,7 @@ libsmf = static_library('smf', libsmf_dep = declare_dependency( link_with : libsmf, dependencies : [libapp_dep, + libmetrics_dep, libsbi_dep, libngap_dep, libnas_5gs_dep, diff --git a/src/smf/metrics.c b/src/smf/metrics.c new file mode 100644 index 000000000..103b63b74 --- /dev/null +++ b/src/smf/metrics.c @@ -0,0 +1,196 @@ +#include "ogs-app.h" +#include "context.h" + +#include "metrics.h" + +typedef struct smf_metrics_spec_def_s { + unsigned int type; + const char *name; + const char *description; + int initial_val; + unsigned int num_labels; + const char **labels; +} smf_metrics_spec_def_t; + +/* Helper generic functions: */ +static int smf_metrics_init_inst(ogs_metrics_inst_t **inst, ogs_metrics_spec_t **specs, + unsigned int len, unsigned int num_labels, const char **labels) +{ + unsigned int i; + for (i = 0; i < len; i++) + inst[i] = ogs_metrics_inst_new(specs[i], num_labels, labels); + return OGS_OK; +} + +static int smf_metrics_free_inst(ogs_metrics_inst_t **inst, + unsigned int len) +{ + unsigned int i; + for (i = 0; i < len; i++) + ogs_metrics_inst_free(inst[i]); + memset(inst, 0, sizeof(inst[0]) * len); + return OGS_OK; +} + +static int smf_metrics_init_spec(ogs_metrics_context_t *ctx, + ogs_metrics_spec_t **dst, smf_metrics_spec_def_t *src, unsigned int len) +{ + unsigned int i; + for (i = 0; i < len; i++) { + dst[i] = ogs_metrics_spec_new(ctx, src[i].type, + src[i].name, src[i].description, + src[i].initial_val, src[i].num_labels, src[i].labels); + } + return OGS_OK; +} + +/* GLOBAL */ +ogs_metrics_spec_t *smf_metrics_spec_global[_SMF_METR_GLOB_MAX]; +ogs_metrics_inst_t *smf_metrics_inst_global[_SMF_METR_GLOB_MAX]; +smf_metrics_spec_def_t smf_metrics_spec_def_global[_SMF_METR_GLOB_MAX] = { +/* Global Counters: */ +[SMF_METR_GLOB_CTR_GN_RX_PARSE_FAILED] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "gn_rx_parse_failed", + .description = "Received GTPv1C messages discarded due to parsing failure", +}, +[SMF_METR_GLOB_CTR_GN_RX_CREATEPDPCTXREQ] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "gn_rx_createpdpcontextreq", + .description = "Received GTPv1C CreatePDPContextRequest messages", +}, +[SMF_METR_GLOB_CTR_GN_RX_DELETEPDPCTXREQ] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "gn_rx_deletepdpcontextreq", + .description = "Received GTPv1C DeletePDPContextRequest messages", +}, +[SMF_METR_GLOB_CTR_S5C_RX_PARSE_FAILED] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "s5c_rx_parse_failed", + .description = "Received GTPv2C messages discarded due to parsing failure", +}, +[SMF_METR_GLOB_CTR_S5C_RX_CREATESESSIONREQ] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "s5c_rx_createsession", + .description = "Received GTPv2C CreateSessionRequest messages", +}, +[SMF_METR_GLOB_CTR_S5C_RX_DELETESESSIONREQ] = { + .type = OGS_METRICS_METRIC_TYPE_COUNTER, + .name = "s5c_rx_deletesession", + .description = "Received GTPv2C DeleteSessionRequest messages", +}, +/* Global Gauges: */ +[SMF_METR_GLOB_GAUGE_UES_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "ues_active", + .description = "Active User Equipments", +}, +[SMF_METR_GLOB_GAUGE_SESSIONS_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "sessions_active", + .description = "Active Sessions", +}, +[SMF_METR_GLOB_GAUGE_BEARERS_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "bearers_active", + .description = "Active Bearers", +}, +[SMF_METR_GLOB_GAUGE_GTP1_PDPCTXS_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "gtp1_pdpctxs_active", + .description = "Active GTPv1 PDP Contexts (GGSN)", +}, +[SMF_METR_GLOB_GAUGE_GTP2_SESSIONS_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "gtp2_sessions_active", + .description = "Active GTPv2 Sessions (PGW)", +}, +[SMF_METR_GLOB_GAUGE_GTP_PEERS_ACTIVE] = { + .type = OGS_METRICS_METRIC_TYPE_GAUGE, + .name = "gtp_peers_active", + .description = "Active GTP peers", +}, +}; +static int smf_metrics_init_inst_global(void) +{ + return smf_metrics_init_inst(smf_metrics_inst_global, smf_metrics_spec_global, + _SMF_METR_GLOB_MAX, 0, NULL); +} +static int smf_metrics_free_inst_global(void) +{ + return smf_metrics_free_inst(smf_metrics_inst_global, _SMF_METR_GLOB_MAX); +} + +/* GTP NODE: */ +const char *labels_gtp_node[] = { + "addr" +}; +#define SMF_METR_GTP_NODE_CTR_ENTRY(_id, _name, _desc) \ + [_id] = { \ + .type = OGS_METRICS_METRIC_TYPE_COUNTER, \ + .name = _name, \ + .description = _desc, \ + .num_labels = OGS_ARRAY_SIZE(labels_gtp_node), \ + .labels = labels_gtp_node, \ + }, +ogs_metrics_spec_t *smf_metrics_spec_gtp_node[_SMF_METR_GTP_NODE_MAX]; +smf_metrics_spec_def_t smf_metrics_spec_def_gtp_node[_SMF_METR_GTP_NODE_MAX] = { +/* Global Counters: */ +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_GN_RX_PARSE_FAILED, + "gtp_node_gn_rx_parse_failed", + "Received GTPv1C messages discarded due to parsing failure") +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_GN_RX_CREATEPDPCTXREQ, + "gtp_node_gn_rx_createpdpcontextreq", + "Received GTPv1C CreatePDPContextRequest messages") +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_GN_RX_DELETEPDPCTXREQ, + "gtp_node_gn_rx_deletepdpcontextreq", + "Received GTPv1C DeletePDPContextRequest messages") +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_S5C_RX_PARSE_FAILED, + "gtp_node_s5c_rx_parse_failed", + "Received GTPv2C messages discarded due to parsing failure") +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_S5C_RX_CREATESESSIONREQ, + "gtp_node_s5c_rx_createsession", + "Received GTPv2C CreateSessionRequest messages") +SMF_METR_GTP_NODE_CTR_ENTRY( + SMF_METR_GTP_NODE_CTR_S5C_RX_DELETESESSIONREQ, + "gtp_node_s5c_rx_deletesession", + "Received GTPv2C DeleteSessionRequest messages") +}; +int smf_metrics_init_inst_gtp_node(ogs_metrics_inst_t **inst, const char *addr) +{ + return smf_metrics_init_inst(inst, + smf_metrics_spec_gtp_node, _SMF_METR_GTP_NODE_MAX, + smf_metrics_spec_def_gtp_node->num_labels, (const char *[]){ addr }); +} +int smf_metrics_free_inst_gtp_node(ogs_metrics_inst_t **inst) +{ + return smf_metrics_free_inst(inst, _SMF_METR_GTP_NODE_MAX); +} + +int smf_metrics_open(void) +{ + ogs_metrics_context_t *ctx = ogs_metrics_self(); + ogs_metrics_context_open(ctx); + + smf_metrics_init_spec(ctx, smf_metrics_spec_global, smf_metrics_spec_def_global, + _SMF_METR_GLOB_MAX); + + smf_metrics_init_spec(ctx, smf_metrics_spec_gtp_node, smf_metrics_spec_def_gtp_node, + _SMF_METR_GTP_NODE_MAX); + + smf_metrics_init_inst_global(); + return 0; +} + +int smf_metrics_close(void) +{ + ogs_metrics_context_t *ctx = ogs_metrics_self(); + smf_metrics_free_inst_global(); + ogs_metrics_context_close(ctx); + return OGS_OK; +} diff --git a/src/smf/metrics.h b/src/smf/metrics.h new file mode 100644 index 000000000..e5e6bfa5f --- /dev/null +++ b/src/smf/metrics.h @@ -0,0 +1,70 @@ +#ifndef SMF_METRICS_H +#define SMF_METRICS_H + +#include "ogs-metrics.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* GLOBAL */ +typedef enum smf_metric_type_global_s { + SMF_METR_GLOB_CTR_GN_RX_PARSE_FAILED = 0, + SMF_METR_GLOB_CTR_GN_RX_CREATEPDPCTXREQ, + SMF_METR_GLOB_CTR_GN_RX_DELETEPDPCTXREQ, + SMF_METR_GLOB_CTR_S5C_RX_PARSE_FAILED, + SMF_METR_GLOB_CTR_S5C_RX_CREATESESSIONREQ, + SMF_METR_GLOB_CTR_S5C_RX_DELETESESSIONREQ, + SMF_METR_GLOB_GAUGE_UES_ACTIVE, + SMF_METR_GLOB_GAUGE_SESSIONS_ACTIVE, + SMF_METR_GLOB_GAUGE_BEARERS_ACTIVE, + SMF_METR_GLOB_GAUGE_GTP1_PDPCTXS_ACTIVE, + SMF_METR_GLOB_GAUGE_GTP2_SESSIONS_ACTIVE, + SMF_METR_GLOB_GAUGE_GTP_PEERS_ACTIVE, + _SMF_METR_GLOB_MAX, +} smf_metric_type_global_t; +extern ogs_metrics_inst_t *smf_metrics_inst_global[_SMF_METR_GLOB_MAX]; + +static inline void smf_metrics_inst_global_set(smf_metric_type_global_t t, int val) +{ ogs_metrics_inst_set(smf_metrics_inst_global[t], val); } +static inline void smf_metrics_inst_global_add(smf_metric_type_global_t t, int val) +{ ogs_metrics_inst_add(smf_metrics_inst_global[t], val); } +static inline void smf_metrics_inst_global_inc(smf_metric_type_global_t t) +{ ogs_metrics_inst_inc(smf_metrics_inst_global[t]); } +static inline void smf_metrics_inst_global_dec(smf_metric_type_global_t t) +{ ogs_metrics_inst_dec(smf_metrics_inst_global[t]); } + +/* GTP NODE */ +typedef enum smf_metric_type_gtp_node_s { + SMF_METR_GTP_NODE_CTR_GN_RX_PARSE_FAILED = 0, + SMF_METR_GTP_NODE_CTR_GN_RX_CREATEPDPCTXREQ, + SMF_METR_GTP_NODE_CTR_GN_RX_DELETEPDPCTXREQ, + SMF_METR_GTP_NODE_CTR_S5C_RX_PARSE_FAILED, + SMF_METR_GTP_NODE_CTR_S5C_RX_CREATESESSIONREQ, + SMF_METR_GTP_NODE_CTR_S5C_RX_DELETESESSIONREQ, + _SMF_METR_GTP_NODE_MAX, +} smf_metric_type_gtp_node_t; +int smf_metrics_init_inst_gtp_node(ogs_metrics_inst_t **inst, const char *addr); +int smf_metrics_free_inst_gtp_node(ogs_metrics_inst_t **inst); + +static inline void smf_metrics_inst_gtp_node_set( + ogs_metrics_inst_t **inst, smf_metric_type_gtp_node_t t, int val) +{ ogs_metrics_inst_set(inst[t], val); } +static inline void smf_metrics_inst_gtp_node_add( + ogs_metrics_inst_t **inst, smf_metric_type_gtp_node_t t, int val) +{ ogs_metrics_inst_add(inst[t], val); } +static inline void smf_metrics_inst_gtp_node_inc( + ogs_metrics_inst_t **inst, smf_metric_type_gtp_node_t t) +{ ogs_metrics_inst_inc(inst[t]); } +static inline void smf_metrics_inst_gtp_node_dec( + ogs_metrics_inst_t **inst, smf_metric_type_gtp_node_t t) +{ ogs_metrics_inst_dec(inst[t]); } + +int smf_metrics_open(void); +int smf_metrics_close(void); + +#ifdef __cplusplus +} +#endif + +#endif /* SMF_METRICS_H */ diff --git a/src/smf/smf-sm.c b/src/smf/smf-sm.c index 9a9a618af..ee644b778 100644 --- a/src/smf/smf-sm.c +++ b/src/smf/smf-sm.c @@ -54,8 +54,8 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) ogs_pkbuf_t *recvbuf = NULL; smf_sess_t *sess = NULL; smf_ue_t *smf_ue = NULL; + smf_gtp_node_t *smf_gnode = NULL; - ogs_gtp_node_t *gnode = NULL; ogs_gtp_xact_t *gtp_xact = NULL; ogs_gtp2_message_t gtp2_message; ogs_gtp1_message_t gtp1_message; @@ -96,17 +96,19 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) recvbuf = e->pkbuf; ogs_assert(recvbuf); + smf_gnode = e->gnode; + ogs_assert(smf_gnode); + if (ogs_gtp2_parse_msg(>p2_message, recvbuf) != OGS_OK) { ogs_error("ogs_gtp2_parse_msg() failed"); + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_S5C_RX_PARSE_FAILED); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_S5C_RX_PARSE_FAILED); ogs_pkbuf_free(recvbuf); break; } e->gtp2_message = >p2_message; - gnode = e->gnode; - ogs_assert(gnode); - - rv = ogs_gtp_xact_receive(gnode, >p2_message.h, >p_xact); + rv = ogs_gtp_xact_receive(smf_gnode->gnode, >p2_message.h, >p_xact); if (rv != OGS_OK) { ogs_pkbuf_free(recvbuf); break; @@ -125,11 +127,13 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) smf_s5c_handle_echo_response(gtp_xact, >p2_message.echo_response); break; case OGS_GTP2_CREATE_SESSION_REQUEST_TYPE: + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_S5C_RX_CREATESESSIONREQ); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_S5C_RX_CREATESESSIONREQ); if (gtp2_message.h.teid == 0) { ogs_expect(!sess); sess = smf_sess_add_by_gtp2_message(>p2_message); if (sess) - OGS_SETUP_GTP_NODE(sess, gnode); + OGS_SETUP_GTP_NODE(sess, smf_gnode->gnode); } if (!sess) { ogs_gtp2_send_error_message(gtp_xact, 0, @@ -141,6 +145,8 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) ogs_fsm_dispatch(&sess->sm, e); break; case OGS_GTP2_DELETE_SESSION_REQUEST_TYPE: + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_S5C_RX_DELETESESSIONREQ); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_S5C_RX_DELETESESSIONREQ); if (!sess) { ogs_gtp2_send_error_message(gtp_xact, 0, OGS_GTP2_DELETE_SESSION_RESPONSE_TYPE, @@ -186,8 +192,13 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) recvbuf = e->pkbuf; ogs_assert(recvbuf); + smf_gnode = e->gnode; + ogs_assert(smf_gnode); + if (ogs_gtp1_parse_msg(>p1_message, recvbuf) != OGS_OK) { ogs_error("ogs_gtp2_parse_msg() failed"); + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_GN_RX_PARSE_FAILED); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_GN_RX_PARSE_FAILED); ogs_pkbuf_free(recvbuf); break; } @@ -197,10 +208,7 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) sess = smf_sess_find_by_teid(gtp1_message.h.teid); } - gnode = e->gnode; - ogs_assert(gnode); - - rv = ogs_gtp1_xact_receive(gnode, >p1_message.h, >p_xact); + rv = ogs_gtp1_xact_receive(smf_gnode->gnode, >p1_message.h, >p_xact); if (rv != OGS_OK) { ogs_pkbuf_free(recvbuf); break; @@ -215,11 +223,13 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) smf_gn_handle_echo_response(gtp_xact, >p1_message.echo_response); break; case OGS_GTP1_CREATE_PDP_CONTEXT_REQUEST_TYPE: + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_GN_RX_CREATEPDPCTXREQ); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_GN_RX_CREATEPDPCTXREQ); if (gtp1_message.h.teid == 0) { ogs_expect(!sess); sess = smf_sess_add_by_gtp1_message(>p1_message); if (sess) - OGS_SETUP_GTP_NODE(sess, gnode); + OGS_SETUP_GTP_NODE(sess, smf_gnode->gnode); } if (!sess) { ogs_gtp1_send_error_message(gtp_xact, 0, @@ -231,6 +241,8 @@ void smf_state_operational(ogs_fsm_t *s, smf_event_t *e) ogs_fsm_dispatch(&sess->sm, e); break; case OGS_GTP1_DELETE_PDP_CONTEXT_REQUEST_TYPE: + smf_metrics_inst_global_inc(SMF_METR_GLOB_CTR_GN_RX_DELETEPDPCTXREQ); + smf_metrics_inst_gtp_node_inc(smf_gnode->metrics, SMF_METR_GTP_NODE_CTR_GN_RX_DELETEPDPCTXREQ); if (!sess) { ogs_gtp1_send_error_message(gtp_xact, 0, OGS_GTP1_DELETE_PDP_CONTEXT_RESPONSE_TYPE, diff --git a/subprojects/prometheus-client-c.wrap b/subprojects/prometheus-client-c.wrap new file mode 100644 index 000000000..6627c0e45 --- /dev/null +++ b/subprojects/prometheus-client-c.wrap @@ -0,0 +1,7 @@ +[wrap-git] +directory = prometheus-client-c +url = https://github.com/open5gs/prometheus-client-c.git +revision = open5gs + +[provide] +dependency_names = libprom