sysmocom
/
pjproject
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ticket #543: Updated PESQ to start/stop manually stream/record instead of using auto-rec/play/hangup, and also updated thresholds 

git-svn-id: https://svn.pjsip.org/repos/pjproject/trunk@2063 74dad513-b988-da41-8d7b-12977e46ad98
This commit is contained in:
Nanang Izzuddin 2008-06-26 18:52:16 +00:00
parent 5242a423c9
commit 9eac547221
17 changed files with 100 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
pjsip-apps/src/test-pjsua/mod_pesq.py
View File

@ -15,6 +15,7 @@ import imp
import sys
import re
import subprocess
import wave
import inc_const as const
from inc_cfg import *
@ -25,7 +26,7 @@ cfg_file = imp.load_source("cfg_file", sys.argv[2])
# PESQ configs
# PESQ_THRESHOLD specifies the minimum acceptable PESQ MOS value, so test can be declared successful
PESQ = "tools/pesq.exe"
PESQ_THRESHOLD = 3.0
PESQ_DEFAULT_THRESHOLD = 3.4
# UserData
class mod_pesq_user_data:
@ -39,9 +40,6 @@ class mod_pesq_user_data:
# Test body function
def test_func(t, user_data):
if len(t.process) == 0:
return
ua1 = t.process[0]
ua2 = t.process[1]
@ -64,27 +62,41 @@ def test_func(t, user_data):
# Get conference clock rate of UA2 for PESQ sample rate option
user_data.pesq_sample_rate_opt = "+" + clock_rate + "000"
# Get WAV input length, in seconds
fin = wave.open(user_data.input_filename, "r")
if fin == None:
raise TestError("Failed opening input WAV file")
inwavlen = fin.getnframes() // fin.getframerate()
if (fin.getnframes() % fin.getframerate()) > 0:
inwavlen = inwavlen + 1
fin.close()
# UA1 making call
ua1.send("m")
ua1.send(t.inst_params[1].uri)
ua1.expect(const.STATE_CALLING)
# UA2 wait until call established
ua2.expect(const.STATE_CONFIRMED)
# Disconnect mic -> rec file to avoid echo recorded when using sound device
ua2.send("cd 0 1")
# Disconnect mic -> rec file, to avoid echo recorded when using sound device
# Disconnect stream -> spk, make it silent
# Connect stream -> rec file, start recording
ua2.send("cd 0 1\ncd 4 0\ncc 4 1")
# Auto answer, auto play, auto hangup
# Just wait for call disconnected
# Assumed WAV input is no more than 30 secs
while 1:
line = ua2.proc.stdout.readline()
if line == "":
raise TestError(ua2.name + ": Premature EOF")
# Disconnect mic -> stream, make stream purely sending from file
# Disconnect stream -> spk, make it silent
# Connect file -> stream, start sending
ua1.send("cd 0 4\ncd 4 0\ncc 1 4")
time.sleep(inwavlen)
# Disconnect files from bridge
ua2.send("cd 4 1")
ua2.expect(const.MEDIA_DISCONN_PORT_SUCCESS)
ua1.send("cd 1 4")
ua1.expect(const.MEDIA_DISCONN_PORT_SUCCESS)
# Search for disconnected text
if re.search(const.STATE_DISCONNECTED, line) != None:
break
# Post body function
def post_func(t, user_data):
@ -101,9 +113,15 @@ def post_func(t, user_data):
if (mo_pesq_out == None):
raise TestError("Failed to fetch PESQ result")
# Evaluate the similarity value
# Get threshold
if (cfg_file.pesq_threshold != None) | (cfg_file.pesq_threshold > -0.5 ):
threshold = cfg_file.pesq_threshold
else:
threshold = PESQ_DEFAULT_THRESHOLD
# Evaluate the PESQ MOS value
pesq_res = mo_pesq_out.group(1)
if (float(pesq_res) >= PESQ_THRESHOLD):
if (float(pesq_res) >= threshold):
endpt.trace("Success, PESQ result = " + pesq_res)
else:
endpt.trace("Failed, PESQ result = " + pesq_res)

6
pjsip-apps/src/test-pjsua/scripts-pesq/100_defaults.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ defaults pjsua settings",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --play-file wavs/input.16.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-audio --max-calls=1 --rec-file wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --play-file wavs/input.16.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --rec-file wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200")
]
)
pesq_threshold = None

5
pjsip-apps/src/test-pjsua/scripts-pesq/101_defaults.py
View File

@ -6,8 +6,8 @@ from inc_cfg import *
test_param = TestParam(
"PESQ defaults pjsua settings (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --rec-file wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --play-file wavs/input.16.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --rec-file wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200")
]
)
@ -15,3 +15,4 @@ test_param = TestParam(
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = None

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g711a.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec PCMA",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
pesq_threshold = 3.5

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g711u.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec PCMU",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
pesq_threshold = 3.5

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g722.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec G722",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file wavs/tmp.16.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file wavs/tmp.16.wav --auto-answer 200")
]
)
pesq_threshold = 3.7

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_gsm.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec GSM",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
pesq_threshold = 3.0

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_ilbc.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec iLBC",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
pesq_threshold = 3.0

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_speex_16000.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec Speex WB",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file wavs/tmp.16.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file wavs/tmp.16.wav --auto-answer 200")
]
)
pesq_threshold = 3.7

6
pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_speex_8000.py
View File

@ -11,7 +11,9 @@ if (HAS_SND_DEV == 0):
test_param = TestParam(
"PESQ codec Speex NB",
[
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav"),
InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
pesq_threshold = 3.0

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g711a.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec PCMA (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.5

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g711u.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec PCMU (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.5

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g722.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec G722 (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file wavs/tmp.16.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file wavs/tmp.16.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.7

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_gsm.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec GSM (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.0

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_ilbc.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec iLBC (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.0

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_speex_16000.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec Speex WB (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file wavs/tmp.16.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file wavs/tmp.16.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.7

6
pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_speex_8000.py
View File

@ -6,10 +6,12 @@ from inc_cfg import *
test_param = TestParam(
"PESQ codec Speex NB (RX side uses snd dev)",
[
InstanceParam("UA1", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200 --auto-rec")
InstanceParam("UA1", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
InstanceParam("UA2", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file wavs/tmp.8.wav --auto-answer 200")
]
)
if (HAS_SND_DEV == 0):
test_param.skip = True
pesq_threshold = 3.0