diff --git a/configs/samples/ast_debug_tools.conf.sample b/configs/samples/ast_debug_tools.conf.sample index 90e976f1bc..f26626b20f 100644 --- a/configs/samples/ast_debug_tools.conf.sample +++ b/configs/samples/ast_debug_tools.conf.sample @@ -38,3 +38,20 @@ COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt)) # # Readable Local time DATEFORMAT='date +%FT%H-%M-%S%z' + +# A list of log files and/or log file search patterns using the +# same syntax as COREDUMPS. +# +LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \ + /var/log/asterisk/debug* /var/log/asterisk/security*) + +# ast_loggrabber converts POSIX timestamps to readable format +# using this Python strftime format string. If not specified +# or an empty string, no format covnersion is done. +LOG_DATEFORMAT="%m/%d-%H:%M:%S.%f" + +# The timezone to use when converting POSIX timestamps to +# readable format. It can be specified in "/" +# format or in abbreviation format such as "CST6CDT". If not +# specified, the "local" timezone is used. +# LOG_TIMEZONE= diff --git a/contrib/Makefile b/contrib/Makefile index a5775cb82c..a667243113 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -20,10 +20,12 @@ clean: include $(ASTTOPDIR)/Makefile.rules install: - $(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts"; \ - $(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"; \ + $(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts" + $(INSTALL) -m 755 scripts/ast_loggrabber "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber" $(INSTALL) -m 755 scripts/ast_coredumper "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper" + $(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py" uninstall: - rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py" - rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper" + -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber" + -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper" + -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py" diff --git a/contrib/scripts/ast_coredumper b/contrib/scripts/ast_coredumper index c82732be9b..81e94e9458 100755 --- a/contrib/scripts/ast_coredumper +++ b/contrib/scripts/ast_coredumper @@ -15,6 +15,7 @@ SYNOPSIS $prog [ --help ] [ --running | --RUNNING ] [ --latest ] [ --tarball-coredumps ] [ --delete-coredumps-after ] [ --tarball-results ] [ --delete-results-after ] + [ --tarball-uniqueid="" ] [ --no-default-search ] [ --append-coredumps ] [ | ... ] @@ -81,6 +82,11 @@ DESCRIPTION to use this option unless you have also specified --tarball-results. + --tarball-uniqueid="" + Normally DATEFORMAT is used to make the tarballs unique + but you can use your own unique id in the tarball names + such as the Jira issue id. + --no-default-search Ignore COREDUMPS from the config files and process only coredumps listed on the command line (if any) and/or @@ -111,6 +117,8 @@ DESCRIPTION /tmp/core[-._]\$(hostname)!(*.txt) NOTES + You must be root to use $prog. + The script relies on not only bash, but also recent GNU date and gdb with python support. *BSD operating systems may require installation of the 'coreutils' and 'devel/gdb' packagess and minor @@ -171,6 +179,11 @@ EOF exit 1 } +if [ $EUID -ne 0 ] ; then + echo "You must be root to use $prog." + exit 1 +fi + running=false RUNNING=false latest=false @@ -245,6 +258,9 @@ for a in "$@" ; do --append-coredumps) append_coredumps=true ;; + --tarball-uniqueid=*) + tarball_uniqueid=${a#*=} + ;; --help|-*) print_help ;; @@ -294,7 +310,7 @@ if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then fi # Timestamp to use for output files -df=$(${DATEFORMAT}) +df=${tarball_uniqueid:-$(${DATEFORMAT})} if $running || $RUNNING ; then # We need to go through some gyrations to find the pid of the running @@ -321,12 +337,9 @@ if $running || $RUNNING ; then read -p "WARNING: Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved. Do you wish to continue? (y/N) " answer fi if [[ "$answer" =~ ^[Yy] ]] ; then - cf="/tmp/core.asterisk.running.$df" - # We want a consistent coredump so stop the process - # and continue it after the dump is complete. - # kill -STOP $pid + cf="/tmp/core-asterisk-running-$df" + echo "Dumping running asterisk process to $cf" ${GDB} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1 - # kill -CONT $pid COREDUMPS+=("$cf") else echo "Skipping dump of running process" @@ -343,17 +356,17 @@ fi # and save them to /tmp/.gdbinit ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:` -tail -n +${ss} $0 >/tmp/.gdbinit +tail -n +${ss} $0 >/tmp/.ast_coredumper.gdbinit # Now iterate over the coredumps and dump the debugging info for i in ${!COREDUMPS[@]} ; do cf=${COREDUMPS[$i]} echo "Processing $cf" - ${GDB} -n --batch -q --ex "source /tmp/.gdbinit" $(which asterisk) "$cf" 2>/dev/null | ( + ${GDB} -n --batch -q --ex "source /tmp/.ast_coredumper.gdbinit" $(which asterisk) "$cf" 2>/dev/null | ( of=/dev/null while IFS= read line ; do if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then - of=$cf.${BASH_REMATCH[1]} + of=${cf}-${BASH_REMATCH[1]} of=${of//:/-} rm -f "$of" echo "Creating $of" @@ -364,7 +377,7 @@ for i in ${!COREDUMPS[@]} ; do done if $tarball_coredumps ; then - tf=/tmp/asterisk.$df.coredumps.tar + tf=/tmp/asterisk-$df.coredumps.tar echo "Creating $tf.gz" for i in ${!COREDUMPS[@]} ; do tar -uvf $tf "${COREDUMPS[@]}" 2>/dev/null @@ -379,17 +392,17 @@ if $delete_coredumps_after ; then fi if $tarball_results ; then - tf=/tmp/asterisk.$df.results.tar + tf=/tmp/asterisk-$df-results.tar echo "Creating $tf.gz" for i in ${!COREDUMPS[@]} ; do - tar -uvf $tf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt 2>/dev/null + tar -uvf $tf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt 2>/dev/null done gzip $tf fi if $delete_results_after ; then for i in ${!COREDUMPS[@]} ; do - rm -rf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt + rm -rf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt done fi diff --git a/contrib/scripts/ast_loggrabber b/contrib/scripts/ast_loggrabber new file mode 100755 index 0000000000..2036d54baf --- /dev/null +++ b/contrib/scripts/ast_loggrabber @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Turn on extended globbing +shopt -s extglob +# Bail on any error +set -e + +prog=$(basename $0) + +print_help() { +cat < | ... ] + +DESCRIPTION + + Gathers log files, optionally converts POSIX timestamps + to readable format. and creates a tarball. + + Options: + + --help + Print this help. + + --dateformat="" + A Python strftime format string to be used when converting + POSIX timestamps in log files to readable format. If not + specified as an argument or in the config file, no conversion + is done. + + --timezone="" + The timezone to use when converting POSIX timestamps to + readable format. It can be specified in "/" + format or in abbreviation format such as "CST6CDT". If not + specified as an argument or in the config file, the "local" + timezone is used. + + --append-logfiles + Append any log files specified on the command line to the + config file specified ones instead of overriding them. + + --tarball-uniqueid="" + Normally DATEFORMAT is used to make the tarballs unique + but you can use your own unique id in the tarball names + such as a Jira issue id. + + | + A list of log files or log file search patterns. Unless + --append-logfiles was specified, these entries will override + those specified in the config files. + + If no files are specified on the command line the, value of + LOGFILES from ast_debug_tools.conf will be used. Failing + that, the following patterns will be used: + /var/log/asterisk/messages* + /var/log/asterisk/queue* + /var/log/asterisk/debug* + /var/log/asterisk/security* + +NOTES + Any files output will have ':' characters changed to '-'. This is + to facilitate uploading those files to Jira which doesn't like the + colons. + +FILES + /etc/asterisk/ast_debug_tools.conf + ~/ast_debug_tools.conf + ./ast_debug_tools.conf + + # Readable Local time for the tarball names + DATEFORMAT='date +%FT%H-%M-%S%z' + + # A list of log files and/or log file search patterns using the + # same syntax as COREDUMPS. + # + LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \\ + /var/log/asterisk/debug* /var/log/asterisk/security*) + + # $prog converts POSIX timestamps to readable format + # using this Python strftime format string. If not specified + # or an empty string, no format covnersion is done. + LOG_DATEFORMAT="%m/%d %H:%M:%S.%f" + + # The timezone to use when converting POSIX timestamps to + # readable format. It can be specified in "/" + # format or in abbreviation format such as "CST6CDT". If not + # specified, the "local" timezone is used. + # LOG_TIMEZONE= + +EOF + exit 1 +} + +append_logfiles=false + +declare -a LOGFILES +declare -a ARGS_LOGFILES + +# Read config files from least important to most important +[ -f /etc/asterisk/ast_debug_tools.conf ] && source /etc/asterisk/ast_debug_tools.conf +[ -f ~/ast_debug_tools.conf ] && source ~/ast_debug_tools.conf +[ -f ./ast_debug_tools.conf ] && source ./ast_debug_tools.conf + +if [ ${#LOGFILES[@]} -eq 0 ] ; then + LOGFILES+=(/var/log/asterisk/messages* /var/log/asterisk/queue* \ + /var/log/asterisk/debug* /var/log/asterisk/security*) +fi + +DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'} + +# Use "$@" (with the quotes) so spaces in patterns or +# file names are preserved. +# Later on when we have to iterate over LOGFILES, we always +# use the indexes rather than trying to expand the values of LOGFILES +# just in case. + +for a in "$@" ; do + case "$a" in + --dateformat=*) + LOG_DATEFORMAT=${a#*=} + ;; + --timezone=*) + LOG_TIMEZONE=${a#*=} + ;; + --append-logfiles) + append_logfiles=true + ;; + --tarball-uniqueid=*) + tarball_uniqueid=${a#*=} + ;; + --help|-*) + print_help + ;; + *) + ARGS_LOGFILES+=("$a") + # If any files are specified on the command line, ignore those + # specified in the config files unless append-logfiles was specified. + if ! $append_logfiles ; then + LOGFILES=() + fi + esac +done + +# append logfiles/patterns specified as command line arguments to LOGFILES. +for i in ${!ARGS_LOGFILES[@]} ; do + LOGFILES+=("${ARGS_LOGFILES[$i]}") +done + +# At this point, all glob entries that match files should be expanded. +# Any entries that don't exist are probably globs that didn't match anything +# and need to be pruned. + +for i in ${!LOGFILES[@]} ; do + if [ ! -f "${LOGFILES[$i]}" ] ; then + unset LOGFILES[$i] + continue + fi +done + +# Sort and weed out any dups +IFS=$'\x0a' +readarray -t LOGFILES < <(echo -n "${LOGFILES[*]}" | sort -u ) +unset IFS + +if [ "${#LOGFILES[@]}" -eq 0 ] ; then + echo "No log files found" + print_help +fi + +# Timestamp to use for output files +df=${tarball_uniqueid:-$(${DATEFORMAT})} + +# Extract the Python timestamp conver script from the end of this +# script and save it to /tmp/.ast_tsconvert.py + +ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:` +tail -n +${ss} $0 >/tmp/.ast_tsconvert.py + +tmpdir=$(mktemp -d) +if [ -z "$tmpdir" ] ; then + echo "${prog}: Unable to create temporary directory." + exit 1 +fi +trap "rm -rf $tmpdir" EXIT +tardir=asterisk-${df}.logfiles + +# Now iterate over the logfiles +for i in ${!LOGFILES[@]} ; do + lf=${LOGFILES[$i]} + destdir="$tmpdir/$tardir/$(dirname $lf)" + destfile="$tmpdir/$tardir/$lf" + mkdir -p "$destdir" 2>/dev/null || : + if [ -n "$LOG_DATEFORMAT" ] ; then + echo "Converting $lf" + cat "$lf" | python /tmp/.ast_tsconvert.py --format="$LOG_DATEFORMAT" --timezone="$LOG_TIMEZONE" > "${destfile}" + else + echo "Copying $lf" + cp "$lf" "${destfile}" + fi +done + +echo "Creating /tmp/$tardir.tar.gz" +tar -czvf /tmp/$tardir.tar.gz -C $tmpdir $tardir 2>/dev/null + +exit + +# Be careful editng the inline scripts. +# They're space-indented. + +# We need the python bit because lock_infos isn't +# a valid symbol in asterisk unless DEBUG_THREADS was +# used during the compile. Also, interrupt and continue +# are only valid for a running program. + +#@@@SCRIPTSTART@@@ +import argparse +import datetime as dt +import dateutil.tz as tz +import re +import sys +import time + +parser = argparse.ArgumentParser(description="Make POSIX timestamps readable") +parser.add_argument('--format', action='store', required=True) +parser.add_argument('--timezone', action='store', required=False) +args=parser.parse_args() + +# We only convert timestamps that are at the beginning of a line +# or are preceeded by a whilespace character or a '[' +rets = re.compile(r'(^|(?<=\s|\[))\d+(\.\d+)?', flags=re.M) +if args.timezone and len(args.timezone) > 0: + tzf = tz.tzfile('/usr/share/zoneinfo/' + args.timezone) +else: + tzf = tz.tzfile('/etc/localtime') + +now = time.time() +a_year_ago = now - (86400.0 * 365) + +def convert(match): + ts = float(match.group(0)) + if ts <= now and ts > a_year_ago and len(args.format) > 0: + return dt.datetime.fromtimestamp(ts, tzf).strftime(args.format) + else: + return match.group(0) + +while 1: + line = sys.stdin.readline() + if not line: + break + print(rets.sub(convert, line))