Incorrect pjlib-test on GitHub CI (#3374)

* Fix pjlib-test not returning error on error

* Add --ci-mode in pjlib-test and pjlib-test-ci Makefile target

* Better sleep_duration_test(): more lenient under --ci-mode and stricter under normal mode

.github/workflows/ci-linux.yml

@@ -32,7 +32,7 @@ jobs:
       with:
         python-version: 2.7
     - name: unit tests
-      run: make pjlib-test pjlib-util-test pjmedia-test pjsua-test
+      run: make pjlib-test-ci pjlib-util-test pjmedia-test pjsua-test
 
   ubuntu-default-full-bundle-2:
   # full bundle 2: running pjnath test
@@ -122,7 +122,7 @@ jobs:
       with:
         python-version: 2.7
     - name: unit tests
-      run: make pjlib-test pjlib-util-test pjmedia-test pjsua-test
+      run: make pjlib-test-ci pjlib-util-test pjmedia-test pjsua-test
 
   ubuntu-video-openh264-2:
   # video 2: running pjnath test

.github/workflows/ci-mac.yml

@@ -34,7 +34,7 @@ jobs:
       with:
         python-version: 2.7
     - name: unit tests
-      run: make pjlib-test pjmedia-test pjlib-util-test pjsua-test
+      run: make pjlib-test-ci pjmedia-test pjlib-util-test pjsua-test
 
   mac-default-full-bundle-2:
   # full bundle 2: running pjnath test
@@ -126,7 +126,7 @@ jobs:
       with:
         python-version: 2.7
     - name: unit tests
-      run: make pjlib-test pjmedia-test pjlib-util-test pjsua-test
+      run: make pjlib-test-ci pjmedia-test pjlib-util-test pjsua-test
 
   mac-video-openh264-2:
   # video 2: running pjnath test

.github/workflows/ci-win.yml

@@ -74,7 +74,7 @@ jobs:
       run: |
         $env:OPENSSL_DIR = Get-Content .\openssl_dir.txt
         $env:PATH+=";$env:OPENSSL_DIR\bin"
-        cd pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe
+        cd pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe --ci-mode
         cd ../../pjlib-util/bin; ./pjlib-util-test-i386-Win32-vc14-Release.exe
         cd ../../pjmedia/bin/; ./pjmedia-test-i386-Win32-vc14-Release.exe
       shell: powershell
@@ -267,7 +267,7 @@ jobs:
         $env:SDL_DIR = Get-Content .\sdl_dir.txt
         $env:PATH+=";$env:OPENSSL_DIR\bin;$env:SDL_DIR\lib\x86;"
         cd tests/pjsua; python runall.py
-        cd ../../pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe
+        cd ../../pjlib/bin; ./pjlib-test-i386-Win32-vc14-Release.exe --ci-mode
         cd ../../pjlib-util/bin; ./pjlib-util-test-i386-Win32-vc14-Release.exe
         cd ../../pjmedia/bin/; ./pjmedia-test-i386-Win32-vc14-Release.exe
       shell: powershell

Makefile

@@ -106,6 +106,9 @@ selftest: pjlib-test pjlib-util-test pjnath-test pjmedia-test pjsip-test pjsua-t
 pjlib-test: pjlib/bin/pjlib-test-$(TARGET_NAME)
 	cd pjlib/build && ../bin/pjlib-test-$(TARGET_NAME)
 
+pjlib-test-ci: pjlib/bin/pjlib-test-$(TARGET_NAME)
+	cd pjlib/build && ../bin/pjlib-test-$(TARGET_NAME) --ci-mode
+
 pjlib-util-test: pjlib-util/bin/pjlib-util-test-$(TARGET_NAME)
 	cd pjlib-util/build && ../bin/pjlib-util-test-$(TARGET_NAME)
 

pjlib/src/pjlib-test/main.c

@@ -26,6 +26,7 @@
 extern int param_echo_sock_type;
 extern const char *param_echo_server;
 extern int param_echo_port;
+extern pj_bool_t param_ci_mode;
 
 
 //#if defined(PJ_WIN32) && PJ_WIN32!=0
@@ -55,19 +56,19 @@ static void init_signals()
 
 #elif (PJ_LINUX || PJ_DARWINOS) && defined(PJ_HAS_EXECINFO_H) && PJ_HAS_EXECINFO_H != 0
 
-#include <execinfo.h>
+#include <execinfo.h>
 #include <signal.h>
-#include <stdio.h>
+#include <stdio.h>
-#include <stdlib.h>
+#include <stdlib.h>
-#include <unistd.h>
+#include <unistd.h>
 static void print_stack(int sig)
 {
-    void *array[16];
+    void *array[16];
-    size_t size;
+    size_t size;
-
+
-    size = backtrace(array, 16);
+    size = backtrace(array, 16);
-    fprintf(stderr, "Error: signal %d:\n", sig);
+    fprintf(stderr, "Error: signal %d:\n", sig);
-    backtrace_symbols_fd(array, size, STDERR_FILENO);
+    backtrace_symbols_fd(array, size, STDERR_FILENO);
     exit(1);
 }
 
@@ -83,14 +84,14 @@ static void init_signals(void)
 
 int main(int argc, char *argv[])
 {
-    int rc;
+    int iarg=1, rc;
     int interractive = 0;
     int no_trap = 0;
 
     boost();
 
-    while (argc > 1) {
+    while (iarg < argc) {
-        char *arg = argv[--argc];
+        char *arg = argv[iarg++];
 
         if (*arg=='-' && *(arg+1)=='i') {
             interractive = 1;
@@ -98,24 +99,30 @@ int main(int argc, char *argv[])
         } else if (*arg=='-' && *(arg+1)=='n') {
             no_trap = 1;
         } else if (*arg=='-' && *(arg+1)=='p') {
-            pj_str_t port = pj_str(argv[--argc]);
+            pj_str_t port = pj_str(argv[iarg++]);
 
             param_echo_port = pj_strtoul(&port);
 
         } else if (*arg=='-' && *(arg+1)=='s') {
-            param_echo_server = argv[--argc];
+            param_echo_server = argv[iarg++];
 
         } else if (*arg=='-' && *(arg+1)=='t') {
-            pj_str_t type = pj_str(argv[--argc]);
+            pj_str_t type = pj_str(argv[iarg++]);
             
             if (pj_stricmp2(&type, "tcp")==0)
                 param_echo_sock_type = pj_SOCK_STREAM();
             else if (pj_stricmp2(&type, "udp")==0)
                 param_echo_sock_type = pj_SOCK_DGRAM();
             else {
-                PJ_LOG(3,("", "error: unknown socket type %s", type.ptr));
+                printf("Error: unknown socket type %s\n", type.ptr);
                 return 1;
             }
+        } else if (strcmp(arg, "--ci-mode")==0) {
+            param_ci_mode = PJ_TRUE;
+
+        } else {
+            printf("Error in argument \"%s\"\n", arg);
+            return 1;
         }
     }
 

pjlib/src/pjlib-test/sleep.c

@@ -52,6 +52,8 @@
 
 #define THIS_FILE   "sleep_test"
 
+extern pj_bool_t param_ci_mode;
+
 static int simple_sleep_test(void)
 {
     enum { COUNT = 10 };
@@ -90,17 +92,19 @@ static int simple_sleep_test(void)
 
 static int sleep_duration_test(void)
 {
-    enum { MIS = 20};
+    const int MAX_SLIP = param_ci_mode? 200 : 20;
     unsigned duration[] = { 2000, 1000, 500, 200, 100 };
     unsigned i;
+    unsigned avg_diff, max_diff;
     pj_status_t rc;
 
     PJ_LOG(3,(THIS_FILE, "..running sleep duration test"));
 
     /* Test pj_thread_sleep() and pj_gettimeofday() */
-    for (i=0; i<PJ_ARRAY_SIZE(duration); ++i) {
+    for (i=0, avg_diff=0, max_diff=0; i<PJ_ARRAY_SIZE(duration); ++i) {
         pj_time_val start, stop;
         pj_uint32_t msec;
+        unsigned diff;
 
         /* Mark start of test. */
         rc = pj_gettimeofday(&start);
@@ -126,23 +130,27 @@ static int sleep_duration_test(void)
         msec = PJ_TIME_VAL_MSEC(stop);
 
         /* Check if it's within range. */
-        if (msec < duration[i] * (100-MIS)/100 ||
+        diff = PJ_ABS(msec - duration[i]);
-            msec > duration[i] * (100+MIS)/100)
+        avg_diff = ((avg_diff * i) + diff) / (i+1);
-        {
+        max_diff = diff>max_diff ? diff : max_diff;
+        if (diff > MAX_SLIP) {
             PJ_LOG(3,(THIS_FILE, 
                       "...error: slept for %d ms instead of %d ms "
-                      "(outside %d%% err window)",
+                      "(outside %d msec tolerance)",
-                      msec, duration[i], MIS));
+                      msec, duration[i], MAX_SLIP));
-            return -30;
         }
     }
-
+    PJ_LOG(3,(THIS_FILE, "...avg/max slippage: %d/%d ms", 
+              avg_diff, max_diff));
+    if (max_diff > MAX_SLIP)
+        return -30;
 
     /* Test pj_thread_sleep() and pj_get_timestamp() and friends */
-    for (i=0; i<PJ_ARRAY_SIZE(duration); ++i) {
+    for (i=0, avg_diff=0, max_diff=0; i<PJ_ARRAY_SIZE(duration); ++i) {
         pj_time_val t1, t2;
         pj_timestamp start, stop;
         pj_uint32_t msec;
+        unsigned diff;
 
         pj_thread_sleep(0);
 
@@ -179,22 +187,25 @@ static int sleep_duration_test(void)
         msec = pj_elapsed_msec(&start, &stop);
 
         /* Check if it's within range. */
-        if (msec < duration[i] * (100-MIS)/100 ||
+        diff = PJ_ABS(msec - duration[i]);
-            msec > duration[i] * (100+MIS)/100)
+        avg_diff = ((avg_diff * i) + diff) / (i+1);
-        {
+        max_diff = diff>max_diff ? diff : max_diff;
+        if (diff > MAX_SLIP) {
             PJ_LOG(3,(THIS_FILE, 
                       "...error: slept for %d ms instead of %d ms "
-                      "(outside %d%% err window)",
+                      "(outside %d msec tolerance)",
-                      msec, duration[i], MIS));
+                      msec, duration[i], MAX_SLIP));
             PJ_TIME_VAL_SUB(t2, t1);
             PJ_LOG(3,(THIS_FILE, 
                       "...info: gettimeofday() reported duration is "
                       "%d msec",
                       PJ_TIME_VAL_MSEC(t2)));
-
-            return -76;
         }
     }
+    PJ_LOG(3,(THIS_FILE, "...avg/max slippage: %d/%d ms", 
+              avg_diff, max_diff));
+    if (max_diff > MAX_SLIP)
+        return -76;
 
     /* All done. */
     return 0;

pjlib/src/pjlib-test/test.c

@@ -49,6 +49,7 @@ const char *param_echo_server = ECHO_SERVER_ADDRESS;
 int param_echo_port = ECHO_SERVER_START_PORT;
 int param_log_decor = PJ_LOG_HAS_NEWLINE | PJ_LOG_HAS_TIME |
                       PJ_LOG_HAS_MICRO_SEC | PJ_LOG_HAS_INDENT;
+pj_bool_t param_ci_mode = PJ_FALSE;  /* GH CI mode: more lenient tests */
 
 int null_func()
 {
@@ -76,6 +77,9 @@ int test_inner(void)
     //pj_dump_config();
     pj_caching_pool_init( &caching_pool, NULL, 0 );
 
+    if (param_ci_mode)
+        PJ_LOG(3,("test", "Using ci-mode"));
+
 #if INCLUDE_ERRNO_TEST
     DO_TEST( errno_test() );
 #endif
@@ -221,7 +225,7 @@ on_return:
 
     pj_shutdown();
 
-    return 0;
+    return rc;
 }
 
 #include <pj/sock.h>