Merge "debug_utilities: Create ast_loggrabber"
This commit is contained in:
zuul
Gerrit Code Review
commit
68968bc226
|
|
@ -38,3 +38,20 @@ COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
|
|||
#
|
||||
# Readable Local time
|
||||
DATEFORMAT='date +%FT%H-%M-%S%z'
|
||||
|
||||
# A list of log files and/or log file search patterns using the
|
||||
# same syntax as COREDUMPS.
|
||||
#
|
||||
LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \
|
||||
/var/log/asterisk/debug* /var/log/asterisk/security*)
|
||||
|
||||
# ast_loggrabber converts POSIX timestamps to readable format
|
||||
# using this Python strftime format string. If not specified
|
||||
# or an empty string, no format covnersion is done.
|
||||
LOG_DATEFORMAT="%m/%d-%H:%M:%S.%f"
|
||||
|
||||
# The timezone to use when converting POSIX timestamps to
|
||||
# readable format. It can be specified in "<continent>/<city>"
|
||||
# format or in abbreviation format such as "CST6CDT". If not
|
||||
# specified, the "local" timezone is used.
|
||||
# LOG_TIMEZONE=
|
||||
|
|
|
|||
|
|
@ -20,10 +20,12 @@ clean:
|
|||
include $(ASTTOPDIR)/Makefile.rules
|
||||
|
||||
install:
|
||||
$(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts"; \
|
||||
$(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"; \
|
||||
$(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts"
|
||||
$(INSTALL) -m 755 scripts/ast_loggrabber "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber"
|
||||
$(INSTALL) -m 755 scripts/ast_coredumper "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
|
||||
$(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
|
||||
|
||||
uninstall:
|
||||
rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
|
||||
rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
|
||||
-rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber"
|
||||
-rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
|
||||
-rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ SYNOPSIS
|
|||
$prog [ --help ] [ --running | --RUNNING ] [ --latest ]
|
||||
[ --tarball-coredumps ] [ --delete-coredumps-after ]
|
||||
[ --tarball-results ] [ --delete-results-after ]
|
||||
[ --tarball-uniqueid="<uniqueid>" ]
|
||||
[ --no-default-search ] [ --append-coredumps ]
|
||||
[ <coredump> | <pattern> ... ]
|
||||
|
||||
|
|
@ -81,6 +82,11 @@ DESCRIPTION
|
|||
to use this option unless you have also specified
|
||||
--tarball-results.
|
||||
|
||||
--tarball-uniqueid="<uniqueid>"
|
||||
Normally DATEFORMAT is used to make the tarballs unique
|
||||
but you can use your own unique id in the tarball names
|
||||
such as the Jira issue id.
|
||||
|
||||
--no-default-search
|
||||
Ignore COREDUMPS from the config files and process only
|
||||
coredumps listed on the command line (if any) and/or
|
||||
|
|
@ -111,6 +117,8 @@ DESCRIPTION
|
|||
/tmp/core[-._]\$(hostname)!(*.txt)
|
||||
|
||||
NOTES
|
||||
You must be root to use $prog.
|
||||
|
||||
The script relies on not only bash, but also recent GNU date and
|
||||
gdb with python support. *BSD operating systems may require
|
||||
installation of the 'coreutils' and 'devel/gdb' packagess and minor
|
||||
|
|
@ -171,6 +179,11 @@ EOF
|
|||
exit 1
|
||||
}
|
||||
|
||||
if [ $EUID -ne 0 ] ; then
|
||||
echo "You must be root to use $prog."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
running=false
|
||||
RUNNING=false
|
||||
latest=false
|
||||
|
|
@ -245,6 +258,9 @@ for a in "$@" ; do
|
|||
--append-coredumps)
|
||||
append_coredumps=true
|
||||
;;
|
||||
--tarball-uniqueid=*)
|
||||
tarball_uniqueid=${a#*=}
|
||||
;;
|
||||
--help|-*)
|
||||
print_help
|
||||
;;
|
||||
|
|
@ -294,7 +310,7 @@ if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
|
|||
fi
|
||||
|
||||
# Timestamp to use for output files
|
||||
df=$(${DATEFORMAT})
|
||||
df=${tarball_uniqueid:-$(${DATEFORMAT})}
|
||||
|
||||
if $running || $RUNNING ; then
|
||||
# We need to go through some gyrations to find the pid of the running
|
||||
|
|
@ -321,12 +337,9 @@ if $running || $RUNNING ; then
|
|||
read -p "WARNING: Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved. Do you wish to continue? (y/N) " answer
|
||||
fi
|
||||
if [[ "$answer" =~ ^[Yy] ]] ; then
|
||||
cf="/tmp/core.asterisk.running.$df"
|
||||
# We want a consistent coredump so stop the process
|
||||
# and continue it after the dump is complete.
|
||||
# kill -STOP $pid
|
||||
cf="/tmp/core-asterisk-running-$df"
|
||||
echo "Dumping running asterisk process to $cf"
|
||||
${GDB} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
|
||||
# kill -CONT $pid
|
||||
COREDUMPS+=("$cf")
|
||||
else
|
||||
echo "Skipping dump of running process"
|
||||
|
|
@ -343,17 +356,17 @@ fi
|
|||
# and save them to /tmp/.gdbinit
|
||||
|
||||
ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
|
||||
tail -n +${ss} $0 >/tmp/.gdbinit
|
||||
tail -n +${ss} $0 >/tmp/.ast_coredumper.gdbinit
|
||||
|
||||
# Now iterate over the coredumps and dump the debugging info
|
||||
for i in ${!COREDUMPS[@]} ; do
|
||||
cf=${COREDUMPS[$i]}
|
||||
echo "Processing $cf"
|
||||
${GDB} -n --batch -q --ex "source /tmp/.gdbinit" $(which asterisk) "$cf" 2>/dev/null | (
|
||||
${GDB} -n --batch -q --ex "source /tmp/.ast_coredumper.gdbinit" $(which asterisk) "$cf" 2>/dev/null | (
|
||||
of=/dev/null
|
||||
while IFS= read line ; do
|
||||
if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
|
||||
of=$cf.${BASH_REMATCH[1]}
|
||||
of=${cf}-${BASH_REMATCH[1]}
|
||||
of=${of//:/-}
|
||||
rm -f "$of"
|
||||
echo "Creating $of"
|
||||
|
|
@ -364,7 +377,7 @@ for i in ${!COREDUMPS[@]} ; do
|
|||
done
|
||||
|
||||
if $tarball_coredumps ; then
|
||||
tf=/tmp/asterisk.$df.coredumps.tar
|
||||
tf=/tmp/asterisk-$df.coredumps.tar
|
||||
echo "Creating $tf.gz"
|
||||
for i in ${!COREDUMPS[@]} ; do
|
||||
tar -uvf $tf "${COREDUMPS[@]}" 2>/dev/null
|
||||
|
|
@ -379,17 +392,17 @@ if $delete_coredumps_after ; then
|
|||
fi
|
||||
|
||||
if $tarball_results ; then
|
||||
tf=/tmp/asterisk.$df.results.tar
|
||||
tf=/tmp/asterisk-$df-results.tar
|
||||
echo "Creating $tf.gz"
|
||||
for i in ${!COREDUMPS[@]} ; do
|
||||
tar -uvf $tf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt 2>/dev/null
|
||||
tar -uvf $tf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt 2>/dev/null
|
||||
done
|
||||
gzip $tf
|
||||
fi
|
||||
|
||||
if $delete_results_after ; then
|
||||
for i in ${!COREDUMPS[@]} ; do
|
||||
rm -rf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt
|
||||
rm -rf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt
|
||||
done
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,255 @@
|
|||
#!/usr/bin/env bash
|
||||
# Turn on extended globbing
|
||||
shopt -s extglob
|
||||
# Bail on any error
|
||||
set -e
|
||||
|
||||
prog=$(basename $0)
|
||||
|
||||
print_help() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$prog - Gather asterisk log files
|
||||
|
||||
SYNOPSIS
|
||||
$prog [ --help ] [ --dateformat="<dateformat>" ]
|
||||
[ --timezone="<timezone>" ] [ --append-logfiles ]
|
||||
[ --tarball-uniqueid="<uniqueid>" ]
|
||||
[ <logfiles> | <pattern> ... ]
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
Gathers log files, optionally converts POSIX timestamps
|
||||
to readable format. and creates a tarball.
|
||||
|
||||
Options:
|
||||
|
||||
--help
|
||||
Print this help.
|
||||
|
||||
--dateformat="<dateformat>"
|
||||
A Python strftime format string to be used when converting
|
||||
POSIX timestamps in log files to readable format. If not
|
||||
specified as an argument or in the config file, no conversion
|
||||
is done.
|
||||
|
||||
--timezone="<timezone>"
|
||||
The timezone to use when converting POSIX timestamps to
|
||||
readable format. It can be specified in "<continent>/<city>"
|
||||
format or in abbreviation format such as "CST6CDT". If not
|
||||
specified as an argument or in the config file, the "local"
|
||||
timezone is used.
|
||||
|
||||
--append-logfiles
|
||||
Append any log files specified on the command line to the
|
||||
config file specified ones instead of overriding them.
|
||||
|
||||
--tarball-uniqueid="<uniqueid>"
|
||||
Normally DATEFORMAT is used to make the tarballs unique
|
||||
but you can use your own unique id in the tarball names
|
||||
such as a Jira issue id.
|
||||
|
||||
<logfiles> | <pattern>
|
||||
A list of log files or log file search patterns. Unless
|
||||
--append-logfiles was specified, these entries will override
|
||||
those specified in the config files.
|
||||
|
||||
If no files are specified on the command line the, value of
|
||||
LOGFILES from ast_debug_tools.conf will be used. Failing
|
||||
that, the following patterns will be used:
|
||||
/var/log/asterisk/messages*
|
||||
/var/log/asterisk/queue*
|
||||
/var/log/asterisk/debug*
|
||||
/var/log/asterisk/security*
|
||||
|
||||
NOTES
|
||||
Any files output will have ':' characters changed to '-'. This is
|
||||
to facilitate uploading those files to Jira which doesn't like the
|
||||
colons.
|
||||
|
||||
FILES
|
||||
/etc/asterisk/ast_debug_tools.conf
|
||||
~/ast_debug_tools.conf
|
||||
./ast_debug_tools.conf
|
||||
|
||||
# Readable Local time for the tarball names
|
||||
DATEFORMAT='date +%FT%H-%M-%S%z'
|
||||
|
||||
# A list of log files and/or log file search patterns using the
|
||||
# same syntax as COREDUMPS.
|
||||
#
|
||||
LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \\
|
||||
/var/log/asterisk/debug* /var/log/asterisk/security*)
|
||||
|
||||
# $prog converts POSIX timestamps to readable format
|
||||
# using this Python strftime format string. If not specified
|
||||
# or an empty string, no format covnersion is done.
|
||||
LOG_DATEFORMAT="%m/%d %H:%M:%S.%f"
|
||||
|
||||
# The timezone to use when converting POSIX timestamps to
|
||||
# readable format. It can be specified in "<continent>/<city>"
|
||||
# format or in abbreviation format such as "CST6CDT". If not
|
||||
# specified, the "local" timezone is used.
|
||||
# LOG_TIMEZONE=
|
||||
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
append_logfiles=false
|
||||
|
||||
declare -a LOGFILES
|
||||
declare -a ARGS_LOGFILES
|
||||
|
||||
# Read config files from least important to most important
|
||||
[ -f /etc/asterisk/ast_debug_tools.conf ] && source /etc/asterisk/ast_debug_tools.conf
|
||||
[ -f ~/ast_debug_tools.conf ] && source ~/ast_debug_tools.conf
|
||||
[ -f ./ast_debug_tools.conf ] && source ./ast_debug_tools.conf
|
||||
|
||||
if [ ${#LOGFILES[@]} -eq 0 ] ; then
|
||||
LOGFILES+=(/var/log/asterisk/messages* /var/log/asterisk/queue* \
|
||||
/var/log/asterisk/debug* /var/log/asterisk/security*)
|
||||
fi
|
||||
|
||||
DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
|
||||
|
||||
# Use "$@" (with the quotes) so spaces in patterns or
|
||||
# file names are preserved.
|
||||
# Later on when we have to iterate over LOGFILES, we always
|
||||
# use the indexes rather than trying to expand the values of LOGFILES
|
||||
# just in case.
|
||||
|
||||
for a in "$@" ; do
|
||||
case "$a" in
|
||||
--dateformat=*)
|
||||
LOG_DATEFORMAT=${a#*=}
|
||||
;;
|
||||
--timezone=*)
|
||||
LOG_TIMEZONE=${a#*=}
|
||||
;;
|
||||
--append-logfiles)
|
||||
append_logfiles=true
|
||||
;;
|
||||
--tarball-uniqueid=*)
|
||||
tarball_uniqueid=${a#*=}
|
||||
;;
|
||||
--help|-*)
|
||||
print_help
|
||||
;;
|
||||
*)
|
||||
ARGS_LOGFILES+=("$a")
|
||||
# If any files are specified on the command line, ignore those
|
||||
# specified in the config files unless append-logfiles was specified.
|
||||
if ! $append_logfiles ; then
|
||||
LOGFILES=()
|
||||
fi
|
||||
esac
|
||||
done
|
||||
|
||||
# append logfiles/patterns specified as command line arguments to LOGFILES.
|
||||
for i in ${!ARGS_LOGFILES[@]} ; do
|
||||
LOGFILES+=("${ARGS_LOGFILES[$i]}")
|
||||
done
|
||||
|
||||
# At this point, all glob entries that match files should be expanded.
|
||||
# Any entries that don't exist are probably globs that didn't match anything
|
||||
# and need to be pruned.
|
||||
|
||||
for i in ${!LOGFILES[@]} ; do
|
||||
if [ ! -f "${LOGFILES[$i]}" ] ; then
|
||||
unset LOGFILES[$i]
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
# Sort and weed out any dups
|
||||
IFS=$'\x0a'
|
||||
readarray -t LOGFILES < <(echo -n "${LOGFILES[*]}" | sort -u )
|
||||
unset IFS
|
||||
|
||||
if [ "${#LOGFILES[@]}" -eq 0 ] ; then
|
||||
echo "No log files found"
|
||||
print_help
|
||||
fi
|
||||
|
||||
# Timestamp to use for output files
|
||||
df=${tarball_uniqueid:-$(${DATEFORMAT})}
|
||||
|
||||
# Extract the Python timestamp conver script from the end of this
|
||||
# script and save it to /tmp/.ast_tsconvert.py
|
||||
|
||||
ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
|
||||
tail -n +${ss} $0 >/tmp/.ast_tsconvert.py
|
||||
|
||||
tmpdir=$(mktemp -d)
|
||||
if [ -z "$tmpdir" ] ; then
|
||||
echo "${prog}: Unable to create temporary directory."
|
||||
exit 1
|
||||
fi
|
||||
trap "rm -rf $tmpdir" EXIT
|
||||
tardir=asterisk-${df}.logfiles
|
||||
|
||||
# Now iterate over the logfiles
|
||||
for i in ${!LOGFILES[@]} ; do
|
||||
lf=${LOGFILES[$i]}
|
||||
destdir="$tmpdir/$tardir/$(dirname $lf)"
|
||||
destfile="$tmpdir/$tardir/$lf"
|
||||
mkdir -p "$destdir" 2>/dev/null || :
|
||||
if [ -n "$LOG_DATEFORMAT" ] ; then
|
||||
echo "Converting $lf"
|
||||
cat "$lf" | python /tmp/.ast_tsconvert.py --format="$LOG_DATEFORMAT" --timezone="$LOG_TIMEZONE" > "${destfile}"
|
||||
else
|
||||
echo "Copying $lf"
|
||||
cp "$lf" "${destfile}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Creating /tmp/$tardir.tar.gz"
|
||||
tar -czvf /tmp/$tardir.tar.gz -C $tmpdir $tardir 2>/dev/null
|
||||
|
||||
exit
|
||||
|
||||
# Be careful editng the inline scripts.
|
||||
# They're space-indented.
|
||||
|
||||
# We need the python bit because lock_infos isn't
|
||||
# a valid symbol in asterisk unless DEBUG_THREADS was
|
||||
# used during the compile. Also, interrupt and continue
|
||||
# are only valid for a running program.
|
||||
|
||||
#@@@SCRIPTSTART@@@
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import dateutil.tz as tz
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
parser = argparse.ArgumentParser(description="Make POSIX timestamps readable")
|
||||
parser.add_argument('--format', action='store', required=True)
|
||||
parser.add_argument('--timezone', action='store', required=False)
|
||||
args=parser.parse_args()
|
||||
|
||||
# We only convert timestamps that are at the beginning of a line
|
||||
# or are preceeded by a whilespace character or a '['
|
||||
rets = re.compile(r'(^|(?<=\s|\[))\d+(\.\d+)?', flags=re.M)
|
||||
if args.timezone and len(args.timezone) > 0:
|
||||
tzf = tz.tzfile('/usr/share/zoneinfo/' + args.timezone)
|
||||
else:
|
||||
tzf = tz.tzfile('/etc/localtime')
|
||||
|
||||
now = time.time()
|
||||
a_year_ago = now - (86400.0 * 365)
|
||||
|
||||
def convert(match):
|
||||
ts = float(match.group(0))
|
||||
if ts <= now and ts > a_year_ago and len(args.format) > 0:
|
||||
return dt.datetime.fromtimestamp(ts, tzf).strftime(args.format)
|
||||
else:
|
||||
return match.group(0)
|
||||
|
||||
while 1:
|
||||
line = sys.stdin.readline()
|
||||
if not line:
|
||||
break
|
||||
print(rets.sub(convert, line))
|
||||
Loading…
Reference in New Issue