From de64969899050adf05d9b6303e0b913290af7d8d Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Fri, 10 Feb 2023 17:22:11 +0700 Subject: [PATCH] Incorrect pjlib-test on GitHub CI (#3374) * Fix pjlib-test not returning error on error * Add --ci-mode in pjlib-test and pjlib-test-ci Makefile target * Better sleep_duration_test(): more lenient under --ci-mode and stricter under normal mode --- .github/workflows/ci-linux.yml | 4 +-- .github/workflows/ci-mac.yml | 4 +-- .github/workflows/ci-win.yml | 4 +-- Makefile | 3 +++ pjlib/src/pjlib-test/main.c | 41 ++++++++++++++++++------------- pjlib/src/pjlib-test/sleep.c | 45 +++++++++++++++++++++------------- pjlib/src/pjlib-test/test.c | 6 ++++- 7 files changed, 66 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 38533c82b..15fd00ce4 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -32,7 +32,7 @@ jobs: with: python-version: 2.7 - name: unit tests - run: make pjlib-test pjlib-util-test pjmedia-test pjsua-test + run: make pjlib-test-ci pjlib-util-test pjmedia-test pjsua-test ubuntu-default-full-bundle-2: # full bundle 2: running pjnath test @@ -122,7 +122,7 @@ jobs: with: python-version: 2.7 - name: unit tests - run: make pjlib-test pjlib-util-test pjmedia-test pjsua-test + run: make pjlib-test-ci pjlib-util-test pjmedia-test pjsua-test ubuntu-video-openh264-2: # video 2: running pjnath test diff --git a/.github/workflows/ci-mac.yml b/.github/workflows/ci-mac.yml index cce0509e5..71651b488 100644 --- a/.github/workflows/ci-mac.yml +++ b/.github/workflows/ci-mac.yml @@ -34,7 +34,7 @@ jobs: with: python-version: 2.7 - name: unit tests - run: make pjlib-test pjmedia-test pjlib-util-test pjsua-test + run: make pjlib-test-ci pjmedia-test pjlib-util-test pjsua-test mac-default-full-bundle-2: # full bundle 2: running pjnath test @@ -126,7 +126,7 @@ jobs: with: python-version: 2.7 - name: unit tests - run: make pjlib-test pjmedia-test pjlib-util-test pjsua-test + run: make pjlib-test-ci pjmedia-test pjlib-util-test pjsua-test mac-video-openh264-2: # video 2: running pjnath test diff --git a/.github/workflows/ci-win.yml b/.github/workflows/ci-win.yml index 47607b0d4..e9774b09b 100644 --- a/.github/workflows/ci-win.yml +++ b/.github/workflows/ci-win.yml @@ -74,7 +74,7 @@ jobs: run: | $env:OPENSSL_DIR = Get-Content .\openssl_dir.txt $env:PATH+=";$env:OPENSSL_DIR\bin" - cd pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe + cd pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe --ci-mode cd ../../pjlib-util/bin; ./pjlib-util-test-i386-Win32-vc14-Release.exe cd ../../pjmedia/bin/; ./pjmedia-test-i386-Win32-vc14-Release.exe shell: powershell @@ -267,7 +267,7 @@ jobs: $env:SDL_DIR = Get-Content .\sdl_dir.txt $env:PATH+=";$env:OPENSSL_DIR\bin;$env:SDL_DIR\lib\x86;" cd tests/pjsua; python runall.py - cd ../../pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe + cd ../../pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe --ci-mode cd ../../pjlib-util/bin; ./pjlib-util-test-i386-Win32-vc14-Release.exe cd ../../pjmedia/bin/; ./pjmedia-test-i386-Win32-vc14-Release.exe shell: powershell diff --git a/Makefile b/Makefile index 44cacdb4e..3fded7e2e 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,9 @@ selftest: pjlib-test pjlib-util-test pjnath-test pjmedia-test pjsip-test pjsua-t pjlib-test: pjlib/bin/pjlib-test-$(TARGET_NAME) cd pjlib/build && ../bin/pjlib-test-$(TARGET_NAME) +pjlib-test-ci: pjlib/bin/pjlib-test-$(TARGET_NAME) + cd pjlib/build && ../bin/pjlib-test-$(TARGET_NAME) --ci-mode + pjlib-util-test: pjlib-util/bin/pjlib-util-test-$(TARGET_NAME) cd pjlib-util/build && ../bin/pjlib-util-test-$(TARGET_NAME) diff --git a/pjlib/src/pjlib-test/main.c b/pjlib/src/pjlib-test/main.c index 7d0a5ef96..43c29b00d 100644 --- a/pjlib/src/pjlib-test/main.c +++ b/pjlib/src/pjlib-test/main.c @@ -26,6 +26,7 @@ extern int param_echo_sock_type; extern const char *param_echo_server; extern int param_echo_port; +extern pj_bool_t param_ci_mode; //#if defined(PJ_WIN32) && PJ_WIN32!=0 @@ -55,19 +56,19 @@ static void init_signals() #elif (PJ_LINUX || PJ_DARWINOS) && defined(PJ_HAS_EXECINFO_H) && PJ_HAS_EXECINFO_H != 0 -#include +#include #include -#include -#include -#include +#include +#include +#include static void print_stack(int sig) { - void *array[16]; - size_t size; - - size = backtrace(array, 16); - fprintf(stderr, "Error: signal %d:\n", sig); - backtrace_symbols_fd(array, size, STDERR_FILENO); + void *array[16]; + size_t size; + + size = backtrace(array, 16); + fprintf(stderr, "Error: signal %d:\n", sig); + backtrace_symbols_fd(array, size, STDERR_FILENO); exit(1); } @@ -83,14 +84,14 @@ static void init_signals(void) int main(int argc, char *argv[]) { - int rc; + int iarg=1, rc; int interractive = 0; int no_trap = 0; boost(); - while (argc > 1) { - char *arg = argv[--argc]; + while (iarg < argc) { + char *arg = argv[iarg++]; if (*arg=='-' && *(arg+1)=='i') { interractive = 1; @@ -98,24 +99,30 @@ int main(int argc, char *argv[]) } else if (*arg=='-' && *(arg+1)=='n') { no_trap = 1; } else if (*arg=='-' && *(arg+1)=='p') { - pj_str_t port = pj_str(argv[--argc]); + pj_str_t port = pj_str(argv[iarg++]); param_echo_port = pj_strtoul(&port); } else if (*arg=='-' && *(arg+1)=='s') { - param_echo_server = argv[--argc]; + param_echo_server = argv[iarg++]; } else if (*arg=='-' && *(arg+1)=='t') { - pj_str_t type = pj_str(argv[--argc]); + pj_str_t type = pj_str(argv[iarg++]); if (pj_stricmp2(&type, "tcp")==0) param_echo_sock_type = pj_SOCK_STREAM(); else if (pj_stricmp2(&type, "udp")==0) param_echo_sock_type = pj_SOCK_DGRAM(); else { - PJ_LOG(3,("", "error: unknown socket type %s", type.ptr)); + printf("Error: unknown socket type %s\n", type.ptr); return 1; } + } else if (strcmp(arg, "--ci-mode")==0) { + param_ci_mode = PJ_TRUE; + + } else { + printf("Error in argument \"%s\"\n", arg); + return 1; } } diff --git a/pjlib/src/pjlib-test/sleep.c b/pjlib/src/pjlib-test/sleep.c index e42a11946..020823970 100644 --- a/pjlib/src/pjlib-test/sleep.c +++ b/pjlib/src/pjlib-test/sleep.c @@ -52,6 +52,8 @@ #define THIS_FILE "sleep_test" +extern pj_bool_t param_ci_mode; + static int simple_sleep_test(void) { enum { COUNT = 10 }; @@ -90,17 +92,19 @@ static int simple_sleep_test(void) static int sleep_duration_test(void) { - enum { MIS = 20}; + const int MAX_SLIP = param_ci_mode? 200 : 20; unsigned duration[] = { 2000, 1000, 500, 200, 100 }; unsigned i; + unsigned avg_diff, max_diff; pj_status_t rc; PJ_LOG(3,(THIS_FILE, "..running sleep duration test")); /* Test pj_thread_sleep() and pj_gettimeofday() */ - for (i=0; i duration[i] * (100+MIS)/100) - { + diff = PJ_ABS(msec - duration[i]); + avg_diff = ((avg_diff * i) + diff) / (i+1); + max_diff = diff>max_diff ? diff : max_diff; + if (diff > MAX_SLIP) { PJ_LOG(3,(THIS_FILE, "...error: slept for %d ms instead of %d ms " - "(outside %d%% err window)", - msec, duration[i], MIS)); - return -30; + "(outside %d msec tolerance)", + msec, duration[i], MAX_SLIP)); } } - + PJ_LOG(3,(THIS_FILE, "...avg/max slippage: %d/%d ms", + avg_diff, max_diff)); + if (max_diff > MAX_SLIP) + return -30; /* Test pj_thread_sleep() and pj_get_timestamp() and friends */ - for (i=0; i duration[i] * (100+MIS)/100) - { + diff = PJ_ABS(msec - duration[i]); + avg_diff = ((avg_diff * i) + diff) / (i+1); + max_diff = diff>max_diff ? diff : max_diff; + if (diff > MAX_SLIP) { PJ_LOG(3,(THIS_FILE, "...error: slept for %d ms instead of %d ms " - "(outside %d%% err window)", - msec, duration[i], MIS)); + "(outside %d msec tolerance)", + msec, duration[i], MAX_SLIP)); PJ_TIME_VAL_SUB(t2, t1); PJ_LOG(3,(THIS_FILE, "...info: gettimeofday() reported duration is " "%d msec", PJ_TIME_VAL_MSEC(t2))); - - return -76; } } + PJ_LOG(3,(THIS_FILE, "...avg/max slippage: %d/%d ms", + avg_diff, max_diff)); + if (max_diff > MAX_SLIP) + return -76; /* All done. */ return 0; diff --git a/pjlib/src/pjlib-test/test.c b/pjlib/src/pjlib-test/test.c index 953449539..96f6edaf5 100644 --- a/pjlib/src/pjlib-test/test.c +++ b/pjlib/src/pjlib-test/test.c @@ -49,6 +49,7 @@ const char *param_echo_server = ECHO_SERVER_ADDRESS; int param_echo_port = ECHO_SERVER_START_PORT; int param_log_decor = PJ_LOG_HAS_NEWLINE | PJ_LOG_HAS_TIME | PJ_LOG_HAS_MICRO_SEC | PJ_LOG_HAS_INDENT; +pj_bool_t param_ci_mode = PJ_FALSE; /* GH CI mode: more lenient tests */ int null_func() { @@ -76,6 +77,9 @@ int test_inner(void) //pj_dump_config(); pj_caching_pool_init( &caching_pool, NULL, 0 ); + if (param_ci_mode) + PJ_LOG(3,("test", "Using ci-mode")); + #if INCLUDE_ERRNO_TEST DO_TEST( errno_test() ); #endif @@ -221,7 +225,7 @@ on_return: pj_shutdown(); - return 0; + return rc; } #include