Implement custom prometheus http handler (#1591)

Implement a custom prometheus http handler in order to:

1. Support listening on a specified address as opposed to any
2. Remove the requirement on the unmaintained promhttp library

This feature comes with one limitation: if an IPv4 address is used, the
server will not listen on the IPv6-mapped address, even if IPv6 is
available. That is, dual-stacking does not work.

Solves: #1475

---------

Co-authored-by: Pavel Punsky <[email protected]>

.github/workflows/actions/ubuntu-build-deps/action.yml

@@ -57,5 +57,4 @@ runs:
       shell: bash
       run: |
         wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libprom-dev-0.1.3-Linux.deb
-        wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libpromhttp-dev-0.1.3-Linux.deb
-        ${{env.AS_ROOT}} apt install -y ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb
+        ${{env.AS_ROOT}} apt install -y ./libprom-dev-0.1.3-Linux.deb

.github/workflows/amazon_linux2023_tests.yml

@@ -42,4 +42,4 @@ jobs:
 
     - name: Integration Test
       working-directory: examples
-      run: ./run_tests.sh && ./run_tests_conf.sh
+      run: ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

.github/workflows/amazon_linux2_tests.yml

@@ -43,4 +43,4 @@ jobs:
 
     - name: Integration Test
       working-directory: examples
-      run: ./run_tests.sh && ./run_tests_conf.sh
+      run: ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

.github/workflows/cmake.yaml

@@ -3,39 +3,36 @@ name: CMake
 on:
   push:
   pull_request:
-    types: [ opened, reopened, synchronize ]
+    types: [opened, reopened, synchronize]
 
 env:
   BUILD_TYPE: Release
 
 jobs:
   build:
-
     runs-on: ubuntu-20.04
 
     steps:
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-            wget \
-            libevent-dev \
-            libssl-dev \
-            libpq-dev libmariadb-dev libsqlite3-dev \
-            libhiredis-dev \
-            libmongoc-dev \
-            libmicrohttpd-dev
-    - uses: actions/checkout@v4
-    - name: Prometheus support
-      run: |
-        wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libprom-dev-0.1.3-Linux.deb && \
-            wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libpromhttp-dev-0.1.3-Linux.deb && \
-            sudo apt install ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb && \
-            rm ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb
-    - name: Configure CMake
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
-    - name: Build
-      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
-    - name: apps tests
-      run: cd examples && ./run_tests.sh && ./run_tests_conf.sh
-      
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+              wget \
+              libevent-dev \
+              libssl-dev \
+              libpq-dev libmariadb-dev libsqlite3-dev \
+              libhiredis-dev \
+              libmongoc-dev \
+              libmicrohttpd-dev
+      - uses: actions/checkout@v4
+      - name: Prometheus support
+        run: |
+          wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libprom-dev-0.1.3-Linux.deb && \
+              sudo apt install ./libprom-dev-0.1.3-Linux.deb && \
+              rm ./libprom-dev-0.1.3-Linux.deb
+      - name: Configure CMake
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      - name: Build
+        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+      - name: apps tests
+        run: cd examples && ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

.github/workflows/compiler-sanitizers.yml

@@ -44,4 +44,4 @@ jobs:
     - name: Test
       run: |
         cd examples
-        ./run_tests.sh && ./run_tests_conf.sh
+        ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

.github/workflows/macos.yml

@@ -56,4 +56,4 @@ jobs:
 
     - name: Integration Test
       working-directory: examples
-      run: ./run_tests.sh && ./run_tests_conf.sh
+      run: ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

.github/workflows/mingw.yml

@@ -82,15 +82,6 @@ jobs:
             -DCMAKE_INSTALL_PREFIX=${{env.INSTALL_DIR}}
         cmake --build . --config ${{matrix.BUILD_TYPE}}
         cmake --build . --config ${{matrix.BUILD_TYPE}} --target install
-        cd ${{env.SOURCE_DIR}}/prometheus-client-c/promhttp
-        mkdir build
-        cd build
-        cmake .. -G"MinGW Makefiles" ^
-            -DBUILD_SHARED_LIBS=${{matrix.BUILD_SHARED_LIBS}} ^
-            -DCMAKE_BUILD_TYPE=${{matrix.BUILD_TYPE}} ^
-            -DCMAKE_INSTALL_PREFIX=${{env.INSTALL_DIR}}
-        cmake --build . --config ${{matrix.BUILD_TYPE}}
-        cmake --build . --config ${{matrix.BUILD_TYPE}} --target install
         
     - name: build coturn
       working-directory: ${{github.workspace}}

.github/workflows/ubuntu_tests.yml

@@ -33,4 +33,4 @@ jobs:
     - name: make check
       run: make check
     - name: apps tests
-      run: cd examples && ./run_tests.sh && ./run_tests_conf.sh
+      run: cd examples && ./run_tests.sh && ./run_tests_conf.sh && ./run_tests_prom.sh

cmake/FindPrometheus.cmake

@@ -7,7 +7,7 @@
 #
 # Try to find prometheus
 # Once done, this will define:
-#  Prometheus_FOUND        - Prometheus (or all requested components of prom, promhttp, microhttpd) was found.
+#  Prometheus_FOUND        - Prometheus (or all requested components of prom, microhttpd) was found.
 #  Prometheus_INCLUDE_DIRS - Libevent include directories
 #  Prometheus_LIBRARIES    - libraries needed to use Prometheus
 #
@@ -16,9 +16,8 @@ include(FindPackageHandleStandardArgs)
 
 find_package(PkgConfig)
 pkg_check_modules(PC_prom QUIET prom)
-pkg_check_modules(PC_promhttp QUIET promhttp)
 pkg_check_modules(PC_microhttd QUIET microhttpd)
-    
+
 find_path(microhttpd_include_dir
     NAMES microhttpd.h
     HINTS ${Prometheus_DIR} ${Prometheus_ROOT} ${PC_microhttd_INCLUDE_DIRS} /usr
@@ -47,28 +46,12 @@ find_library(
     PATHS $ENV{Prometheus_DIR} $ENV{Prometheus_ROOT}
     PATH_SUFFIXES lib ${CMAKE_INSTALL_LIBDIR})
 
-find_path(promhttp_INCLUDE_DIR
-    NAMES promhttp.h
-    HINTS ${Prometheus_DIR} ${Prometheus_ROOT} ${PC_promhttp_INCLUDE_DIRS} /usr
-    PATHS $ENV{Prometheus_DIR} $ENV{Prometheus_ROOT}
-    PATH_SUFFIXES include
-    )
-
-find_library(
-    promhttp_libs
-    NAMES promhttp
-    HINTS ${Prometheus_DIR} ${Prometheus_ROOT} ${PC_promhttp_LIBRARY_DIRS}
-    PATHS $ENV{Prometheus_DIR} $ENV{Prometheus_ROOT}
-    PATH_SUFFIXES lib ${CMAKE_INSTALL_LIBDIR})
-
 find_package_handle_standard_args(Prometheus
     REQUIRED_VARS prom_libs prom_INCLUDE_DIR
-        promhttp_libs promhttp_INCLUDE_DIR
         microhttpd_include_dir microhttpd_libs
         )
 
 set(Prometheus_INCLUDE_DIRS
     ${prom_INCLUDE_DIR}
-    ${promhttp_INCLUDE_DIR}
     ${microhttpd_include_dir})
-set(Prometheus_LIBRARIES ${prom_libs} ${promhttp_libs} ${microhttpd_libs})
+set(Prometheus_LIBRARIES ${prom_libs} ${microhttpd_libs})

configure

@@ -848,30 +848,18 @@ if [ -z "${TURN_NO_PROMETHEUS}" ] ; then
     ER=$?
     if ! [ ${ER} -eq 0 ] ; then
         ${ECHO_CMD} "Prometheus lib found."
-        testlib promhttp
+        testlib microhttpd
         ER=$?
         if ! [ ${ER} -eq 0 ] ; then
-            ${ECHO_CMD} "Prometheus http lib found."
-            testlib microhttpd
-            ER=$?
-            if ! [ ${ER} -eq 0 ] ; then
-                ${ECHO_CMD} "Microhttpd lib found."
-                # Adjustments for Debian
-                # See: https://github.com/coturn/coturn/pull/754#issuecomment-824693226
-                if [ -f "/etc/debian_version" ] ; then
-                    OSLIBS="${OSLIBS} -latomic"
-                fi
-            else
-                ${ECHO_CMD}
-                ${ECHO_CMD} "Warning: microhttpd development libraries are not installed properly in required location."
-                ${ECHO_CMD} "Prometheus support will be disabled."
-                ${ECHO_CMD} "See the docs/Prometheus.md file."
-                ${ECHO_CMD}
-                OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS"
+            ${ECHO_CMD} "Microhttpd lib found."
+            # Adjustments for Debian
+            # See: https://github.com/coturn/coturn/pull/754#issuecomment-824693226
+            if [ -f "/etc/debian_version" ] ; then
+                OSLIBS="${OSLIBS} -latomic"
             fi
         else
             ${ECHO_CMD}
-            ${ECHO_CMD} "Warning: Libpromhttp development libraries are not installed properly in required location."
+            ${ECHO_CMD} "Warning: microhttpd development libraries are not installed properly in required location."
             ${ECHO_CMD} "Prometheus support will be disabled."
             ${ECHO_CMD} "See the docs/Prometheus.md file."
             ${ECHO_CMD}

docker/coturn/alpine/Dockerfile

@@ -46,27 +46,13 @@ RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \
                  .. \
  && make
 
-# Build libpromhttp.so from sources.
-RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \
- # Fix compiler warning: -Werror=incompatible-pointer-types
- && sed -i 's/\&promhttp_handler/(MHD_AccessHandlerCallback)\&promhttp_handler/' \
-           /build/promhttp/src/promhttp.c \
- && TEST=0 cmake -G "Unix Makefiles" \
-                 -DCMAKE_INSTALL_PREFIX=/usr \
-                 -DCMAKE_SKIP_BUILD_RPATH=TRUE \
-                 -DCMAKE_C_FLAGS="-g -O3" \
-                 .. \
- && make VERBOSE=1
-
 # Install prometheus-client-c.
 RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \
  && mkdir -p $LIBS_DIR/ \
  && cp -rf /build/prom/build/libprom.so \
-           /build/promhttp/build/libpromhttp.so \
        $LIBS_DIR/ \
  && mkdir -p /out/usr/include/ \
  && cp -rf /build/prom/include/* \
-           /build/promhttp/include/* \
        /out/usr/include/ \
  # Preserve license file.
  && mkdir -p /out/usr/share/licenses/prometheus-client-c/ \

docker/coturn/debian/Dockerfile

@@ -46,27 +46,13 @@ RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \
                  .. \
  && make
 
-# Build libpromhttp.so from sources.
-RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \
- # Fix compiler warning: -Werror=incompatible-pointer-types
- && sed -i 's/\&promhttp_handler/(MHD_AccessHandlerCallback)\&promhttp_handler/' \
-           /build/promhttp/src/promhttp.c \
- && TEST=0 cmake -G "Unix Makefiles" \
-                 -DCMAKE_INSTALL_PREFIX=/usr \
-                 -DCMAKE_SKIP_BUILD_RPATH=TRUE \
-                 -DCMAKE_C_FLAGS="-g -O3" \
-                 .. \
- && make VERBOSE=1
-
 # Install prometheus-client-c.
 RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \
  && mkdir -p $LIBS_DIR/ \
  && cp -rf /build/prom/build/libprom.so \
-           /build/promhttp/build/libpromhttp.so \
        $LIBS_DIR/ \
  && mkdir -p /out/usr/include/ \
  && cp -rf /build/prom/include/* \
-           /build/promhttp/include/* \
        /out/usr/include/ \
  # Preserve license file.
  && mkdir -p /out/usr/share/licenses/prometheus-client-c/ \

examples/run_tests_prom.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+function assert_prom_no_response() {
+  wget --quiet --output-document=/dev/null --tries=1 "$1"
+  status="$?"
+  if [ "$status" -eq 0 ]; then
+    echo FAIL
+    exit 1
+  else
+    echo OK
+  fi
+}
+
+function assert_prom_response() {
+  # Match something that looks like the expected body
+  wget --quiet --output-document=- --tries=1 "$1" | grep 'TYPE\|HELP\|counter\|gauge' >/dev/null
+  status="$?"
+  if [ "$status" -eq 0 ]; then
+    echo OK
+  else
+    echo FAIL
+    exit "$status"
+  fi
+}
+
+echo "Running without prometheus"
+../bin/turnserver &
+turnserver_pid="$!"
+sleep 2
+assert_prom_no_response "http://localhost:9641/metrics"
+kill "$turnserver_pid"
+
+echo "Running turnserver with prometheus, using defaults"
+../bin/turnserver --prometheus &
+turnserver_pid="$!"
+sleep 2
+assert_prom_response "http://localhost:9641/metrics"
+kill "$turnserver_pid"
+
+echo "Running turnserver with prometheus, using custom address"
+../bin/turnserver --prometheus --prometheus-address="127.0.0.1" &
+turnserver_pid="$!"
+sleep 2
+assert_prom_response "http://127.0.0.1:9641/metrics"
+kill "$turnserver_pid"
+
+echo "Running turnserver with prometheus, using custom port"
+../bin/turnserver --prometheus --prometheus-port="8080" &
+turnserver_pid="$!"
+sleep 2
+assert_prom_response "http://localhost:8080/metrics"
+kill "$turnserver_pid"
+
+echo "Running turnserver with prometheus, using custom address and port"
+../bin/turnserver --prometheus --prometheus-address="127.0.0.1" --prometheus-port="8080" &
+turnserver_pid="$!"
+sleep 2
+assert_prom_response "http://127.0.0.1:8080/metrics"
+kill "$turnserver_pid"

src/apps/relay/CMakeLists.txt

@@ -119,7 +119,7 @@ if(Prometheus_FOUND)
     list(APPEND turnserver_include_dirs ${Prometheus_INCLUDE_DIRS})
 else()
     message(AUTHOR_WARNING "Could not find prometheus. Please install "
-        "prom, promhttp, microhttpd, and set Prometheus_ROOT. "
+        "prom, microhttpd, and set Prometheus_ROOT. "
         "See docs/Prometheus.md")
     list(APPEND turnserver_DEFINED TURN_NO_PROMETHEUS)
 endif()

src/apps/relay/mainrelay.c

@@ -212,6 +212,7 @@ turn_params_t turn_params = {
     0,                                  /* user_quota */
     0,                                  /* prometheus disabled by default */
     DEFAULT_PROM_SERVER_PORT,           /* prometheus port */
+    "",                                 /* prometheus address */
     0, /* prometheus username labelling disabled by default when prometheus is enabled */
 
     ///////////// Users DB //////////////
@@ -1138,6 +1139,7 @@ static char Usage[] =
     "enabled it will listen on port 9641 under the path /metrics\n"
     "						also the path / on this port can be used as a health check\n"
     " --prometheus-port		<port>		Prometheus metrics port (Default: 9641).\n"
+    " --prometheus-address		<address>		Prometheus listening address (Default: any).\n"
     " --prometheus-username-labels			When metrics are enabled, add labels with client usernames.\n"
 #endif
     " --use-auth-secret				TURN REST API flag.\n"
@@ -1436,6 +1438,7 @@ enum EXTRA_OPTS {
   PERMISSION_LIFETIME_OPT,
   PROMETHEUS_OPT,
   PROMETHEUS_PORT_OPT,
+  PROMETHEUS_ADDRESS_OPT,
   PROMETHEUS_ENABLE_USERNAMES_OPT,
   AUTH_SECRET_OPT,
   NO_AUTH_PINGS_OPT,
@@ -1556,6 +1559,7 @@ static const struct myoption long_options[] = {
 #if !defined(TURN_NO_PROMETHEUS)
     {"prometheus", optional_argument, NULL, PROMETHEUS_OPT},
     {"prometheus-port", optional_argument, NULL, PROMETHEUS_PORT_OPT},
+    {"prometheus-address", optional_argument, NULL, PROMETHEUS_ADDRESS_OPT},
     {"prometheus-username-labels", optional_argument, NULL, PROMETHEUS_ENABLE_USERNAMES_OPT},
 #endif
     {"use-auth-secret", optional_argument, NULL, AUTH_SECRET_OPT},
@@ -2219,6 +2223,9 @@ static void set_option(int c, char *value) {
   case PROMETHEUS_PORT_OPT:
     turn_params.prometheus_port = atoi(value);
     break;
+  case PROMETHEUS_ADDRESS_OPT:
+    STRCPY(turn_params.prometheus_address, value);
+    break;
   case PROMETHEUS_ENABLE_USERNAMES_OPT:
     turn_params.prometheus_username_labels = 1;
     break;

src/apps/relay/mainrelay.h

@@ -312,6 +312,7 @@ typedef struct _turn_params_ {
   vint user_quota;
   int prometheus;
   int prometheus_port;
+  char prometheus_address[INET6_ADDRSTRLEN];
   int prometheus_username_labels;
 
   /////// Users DB ///////////

src/apps/relay/prom_server.c

@@ -35,6 +35,42 @@ prom_counter_t *turn_total_traffic_peer_sentb;
 
 prom_gauge_t *turn_total_allocations;
 
+#if MHD_VERSION >= 0x00097002
+#define MHD_RESULT enum MHD_Result
+#else
+#define MHD_RESULT int
+#endif
+
+MHD_RESULT promhttp_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method,
+                            const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) {
+  MHD_RESULT ret;
+
+  char *body = "not found";
+  enum MHD_ResponseMemoryMode mode = MHD_RESPMEM_PERSISTENT;
+  unsigned int status = MHD_HTTP_NOT_FOUND;
+
+  if (strcmp(method, "GET") != 0) {
+    status = MHD_HTTP_METHOD_NOT_ALLOWED;
+    body = "method not allowed";
+  } else if (strcmp(url, "/metrics") == 0) {
+    body = prom_collector_registry_bridge(PROM_COLLECTOR_REGISTRY_DEFAULT);
+    mode = MHD_RESPMEM_MUST_FREE;
+    status = MHD_HTTP_OK;
+  }
+
+  struct MHD_Response *response = MHD_create_response_from_buffer(strlen(body), body, mode);
+  if (response == NULL) {
+    if (mode == MHD_RESPMEM_MUST_FREE) {
+      free(body);
+    }
+    ret = MHD_NO;
+  } else {
+    ret = MHD_queue_response(connection, status, response);
+    MHD_destroy_response(response);
+  }
+  return ret;
+}
+
 void start_prometheus_server(void) {
   if (turn_params.prometheus == 0) {
     TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus collector disabled, not started\n");
@@ -103,13 +139,8 @@ void start_prometheus_server(void) {
   turn_total_allocations = prom_collector_registry_must_register_metric(
       prom_gauge_new("turn_total_allocations", "Represents current allocations number", 1, typeLabel));
 
-  promhttp_set_active_collector_registry(NULL);
-
   // some flags appeared first in microhttpd v0.9.53
   unsigned int flags = 0;
-  if (MHD_is_feature_supported(MHD_FEATURE_IPv6) && is_ipv6_enabled()) {
-    flags |= MHD_USE_DUAL_STACK;
-  }
 #if MHD_VERSION >= 0x00095300
   flags |= MHD_USE_ERROR_LOG;
 #endif
@@ -127,7 +158,36 @@ void start_prometheus_server(void) {
     TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prometheus exporter server will start using SELECT. "
                                           "The exporter might be unreachable on highly used servers\n");
   }
-  struct MHD_Daemon *daemon = promhttp_start_daemon(flags, turn_params.prometheus_port, NULL, NULL);
+
+  ioa_addr server_addr;
+  addr_set_any(&server_addr);
+  if (turn_params.prometheus_address[0]) {
+    if (make_ioa_addr((const uint8_t *)turn_params.prometheus_address, turn_params.prometheus_port, &server_addr) < 0) {
+      TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "could not parse prometheus collector's server address\n");
+      return;
+    }
+
+    if (is_ipv6_enabled() && server_addr.ss.sa_family == AF_INET6) {
+      flags |= MHD_USE_IPv6;
+    }
+  } else {
+    if (MHD_is_feature_supported(MHD_FEATURE_IPv6) && is_ipv6_enabled()) {
+      flags |= MHD_USE_DUAL_STACK;
+      server_addr.ss.sa_family = AF_INET6;
+      server_addr.s6.sin6_port = htons((uint16_t)turn_params.prometheus_port);
+    } else {
+      server_addr.ss.sa_family = AF_INET;
+      server_addr.s4.sin_port = htons((uint16_t)turn_params.prometheus_port);
+    }
+  }
+
+  uint8_t addr[MAX_IOA_ADDR_STRING];
+  addr_to_string(&server_addr, addr);
+  TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus exporter server will listen on %s\n", addr);
+
+  struct MHD_Daemon *daemon =
+      MHD_start_daemon(flags, 0, NULL, NULL, &promhttp_handler, NULL, MHD_OPTION_LISTENING_ADDRESS_REUSE, 1,
+                       MHD_OPTION_SOCK_ADDR, &server_addr, MHD_OPTION_END);
   if (daemon == NULL) {
     TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "could not start prometheus collector\n");
     return;

src/apps/relay/prom_server.h

@@ -18,7 +18,6 @@ extern "C" {
 #endif
 #include <microhttpd.h>
 #include <prom.h>
-#include <promhttp.h>
 #ifdef __cplusplus
 }
 #endif /* __clplusplus */