Blame tests/run-debuginfod-find.sh.debuginfod-timeout-progress

Packit 86a02d
#!/bin/bash
Packit 86a02d
#
Packit 86a02d
# Copyright (C) 2019 Red Hat, Inc.
Packit 86a02d
# This file is part of elfutils.
Packit 86a02d
#
Packit 86a02d
# This file is free software; you can redistribute it and/or modify
Packit 86a02d
# it under the terms of the GNU General Public License as published by
Packit 86a02d
# the Free Software Foundation; either version 3 of the License, or
Packit 86a02d
# (at your option) any later version.
Packit 86a02d
#
Packit 86a02d
# elfutils is distributed in the hope that it will be useful, but
Packit 86a02d
# WITHOUT ANY WARRANTY; without even the implied warranty of
Packit 86a02d
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Packit 86a02d
# GNU General Public License for more details.
Packit 86a02d
#
Packit 86a02d
# You should have received a copy of the GNU General Public License
Packit 86a02d
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
Packit 86a02d
Packit 86a02d
. $srcdir/test-subr.sh  # includes set -e
Packit 86a02d
Packit 86a02d
DB=${PWD}/.debuginfod_tmp.sqlite
Packit 86a02d
tempfiles $DB
Packit 86a02d
export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache
Packit 86a02d
Packit 86a02d
PID1=0
Packit 86a02d
PID2=0
Packit 86a02d
Packit 86a02d
cleanup()
Packit 86a02d
{
Packit 86a02d
  if [ $PID1 -ne 0 ]; then kill $PID1; wait $PID1; fi
Packit 86a02d
  if [ $PID2 -ne 0 ]; then kill $PID2; wait $PID2; fi
Packit 86a02d
Packit 86a02d
  rm -rf F R L ${PWD}/.client_cache*
Packit 86a02d
  exit_cleanup
Packit 86a02d
}
Packit 86a02d
Packit 86a02d
# clean up trash if we were aborted early
Packit 86a02d
trap cleanup 0 1 2 3 5 9 15
Packit 86a02d
Packit 86a02d
# find an unused port number
Packit 86a02d
while true; do
Packit 86a02d
    PORT1=`expr '(' $RANDOM % 1000 ')' + 9000`
Packit 86a02d
    ss -atn | fgrep ":$PORT1" || break
Packit 86a02d
done    
Packit 86a02d
Packit 86a02d
# We want to run debuginfod in the background.  We also want to start
Packit 86a02d
# it with the same check/installcheck-sensitive LD_LIBRARY_PATH stuff
Packit 86a02d
# that the testrun alias sets.  But: we if we just use
Packit 86a02d
#    testrun .../debuginfod
Packit 86a02d
# it runs in a subshell, with different pid, so not helpful.
Packit 86a02d
#
Packit 86a02d
# So we gather the LD_LIBRARY_PATH with this cunning trick:
Packit 86a02d
ldpath=`testrun sh -c 'echo $LD_LIBRARY_PATH'`
Packit 86a02d
Packit 86a02d
mkdir F R L
Packit 86a02d
# not tempfiles F R L - they are directories which we clean up manually
Packit 86a02d
ln -s ${abs_builddir}/dwfllines L/foo   # any program not used elsewhere in this test
Packit 86a02d
Packit 86a02d
wait_ready()
Packit 86a02d
{
Packit 86a02d
  port=$1;
Packit 86a02d
  what=$2;
Packit 86a02d
  value=$3;
Packit 86a02d
  timeout=20;
Packit 86a02d
Packit 86a02d
  echo "Wait $timeout seconds on $port for metric $what to change to $value"
Packit 86a02d
  while [ $timeout -gt 0 ]; do
Packit 86a02d
    mvalue="$(curl -s http://127.0.0.1:$port/metrics \
Packit 86a02d
              | grep "$what" | awk '{print $NF}')"
Packit 86a02d
    if [ -z "$mvalue" ]; then mvalue=0; fi
Packit 86a02d
      echo "metric $what: $mvalue"
Packit 86a02d
      if [ "$mvalue" -eq "$value" ]; then
Packit 86a02d
        break;
Packit 86a02d
    fi
Packit 86a02d
    sleep 0.5;
Packit 86a02d
    ((timeout--));
Packit 86a02d
  done;
Packit 86a02d
Packit 86a02d
  if [ $timeout -eq 0 ]; then
Packit 86a02d
    echo "metric $what never changed to $value on port $port"
Packit 86a02d
    exit 1;
Packit 86a02d
  fi
Packit 86a02d
}
Packit 86a02d
Packit 86a02d
env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod -F -R -d $DB -p $PORT1 -t0 -g0 R F L &
Packit 86a02d
PID1=$!
Packit 86a02d
# Server must become ready
Packit 86a02d
wait_ready $PORT1 'ready' 1
Packit 86a02d
export DEBUGINFOD_URLS=http://127.0.0.1:$PORT1/   # or without trailing /
Packit 86a02d
Packit 86a02d
# Be patient when run on a busy machine things might take a bit.
Packit 86a02d
export DEBUGINFOD_TIMEOUT=10
Packit 86a02d
Packit 86a02d
# We use -t0 and -g0 here to turn off time-based scanning & grooming.
Packit 86a02d
# For testing purposes, we just sic SIGUSR1 / SIGUSR2 at the process.
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Compile a simple program, strip its debuginfo and save the build-id.
Packit 86a02d
# Also move the debuginfo into another directory so that elfutils
Packit 86a02d
# cannot find it without debuginfod.
Packit 86a02d
echo "int main() { return 0; }" > ${PWD}/prog.c
Packit 86a02d
tempfiles prog.c
Packit 86a02d
gcc -g -o prog ${PWD}/prog.c
Packit 86a02d
 ${abs_top_builddir}/src/strip -g -f prog.debug ${PWD}/prog
Packit 86a02d
BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
Packit 86a02d
          -a prog | grep 'Build ID' | cut -d ' ' -f 7`
Packit 86a02d
Packit 86a02d
mv prog F
Packit 86a02d
mv prog.debug F
Packit 86a02d
kill -USR1 $PID1
Packit 86a02d
# Wait till both files are in the index.
Packit 86a02d
wait_ready $PORT1 'thread_work_total{file="F"}' 2
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Test whether elfutils, via the debuginfod client library dlopen hooks,
Packit 86a02d
# is able to fetch debuginfo from the local debuginfod.
Packit 86a02d
testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Test whether debuginfod-find is able to fetch those files.
Packit 86a02d
rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID`
Packit 86a02d
cmp $filename F/prog.debug
Packit 86a02d
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID`
Packit 86a02d
cmp $filename F/prog
Packit 86a02d
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID ${PWD}/prog.c`
Packit 86a02d
cmp $filename  ${PWD}/prog.c
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Add artifacts to the search paths and test whether debuginfod finds them while already running.
Packit 86a02d
Packit 86a02d
# Build another, non-stripped binary
Packit 86a02d
echo "int main() { return 0; }" > ${PWD}/prog2.c
Packit 86a02d
tempfiles prog2.c
Packit 86a02d
gcc -g -o prog2 ${PWD}/prog2.c
Packit 86a02d
BUILDID2=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
Packit 86a02d
          -a prog2 | grep 'Build ID' | cut -d ' ' -f 7`
Packit 86a02d
Packit 86a02d
mv prog2 F
Packit 86a02d
kill -USR1 $PID1
Packit 86a02d
# Now there should be 3 files in the index
Packit 86a02d
wait_ready $PORT1 'thread_work_total{file="F"}' 3
Packit 86a02d
Packit 86a02d
# Rerun same tests for the prog2 binary
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find -v debuginfo $BUILDID2 2>vlog`
Packit 86a02d
cmp $filename F/prog2
Packit 86a02d
cat vlog
Packit 86a02d
grep -q Progress vlog
Packit 86a02d
tempfiles vlog
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2`
Packit 86a02d
cmp $filename F/prog2
Packit 86a02d
filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID2 ${PWD}/prog2.c`
Packit 86a02d
cmp $filename ${PWD}/prog2.c
Packit 86a02d
Packit 86a02d
cp -rp ${abs_srcdir}/debuginfod-rpms R
Packit 86a02d
kill -USR1 $PID1
Packit 86a02d
# All rpms need to be in the index
Packit 86a02d
rpms=$(find R -name \*rpm | wc -l)
Packit 86a02d
wait_ready $PORT1 'scanned_total{source="rpm"}' $rpms
Packit 86a02d
Packit 86a02d
kill -USR1 $PID1  # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs
Packit 86a02d
# Expect all source files found in the rpms (they are all called hello.c :)
Packit 86a02d
# We will need to extract all rpms (in their own directory) and could all
Packit 86a02d
# sources referenced in the .debug files.
Packit 86a02d
mkdir extracted
Packit 86a02d
cd extracted
Packit 86a02d
subdir=0;
Packit 86a02d
newrpms=$(find ../R -name \*\.rpm)
Packit 86a02d
for i in $newrpms; do
Packit 86a02d
    subdir=$[$subdir+1];
Packit 86a02d
    mkdir $subdir;
Packit 86a02d
    cd $subdir;
Packit 86a02d
    ls -lah ../$i
Packit 86a02d
    rpm2cpio ../$i | cpio -id;
Packit 86a02d
    cd ..;
Packit 86a02d
done
Packit 86a02d
sourcefiles=$(find -name \*\\.debug \
Packit 86a02d
	      | env LD_LIBRARY_PATH=$ldpath xargs \
Packit 86a02d
		${abs_top_builddir}/src/readelf --debug-dump=decodedline \
Packit 86a02d
	      | grep mtime: | wc --lines)
Packit 86a02d
cd ..
Packit 86a02d
rm -rf extracted
Packit 86a02d
Packit 86a02d
wait_ready $PORT1 'found_sourcerefs_total{source="rpm"}' $sourcefiles
Packit 86a02d
Packit 86a02d
# Run a bank of queries against the debuginfod-rpms test cases
Packit 86a02d
Packit 86a02d
rpm_test() {
Packit 86a02d
    __BUILDID=$1
Packit 86a02d
    __SOURCEPATH=$2
Packit 86a02d
    __SOURCESHA1=$3
Packit 86a02d
    
Packit 86a02d
    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $__BUILDID`
Packit 86a02d
    buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
Packit 86a02d
             -a $filename | grep 'Build ID' | cut -d ' ' -f 7`
Packit 86a02d
    test $__BUILDID = $buildid
Packit 86a02d
Packit 86a02d
    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $__BUILDID`
Packit 86a02d
    buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
Packit 86a02d
             -a $filename | grep 'Build ID' | cut -d ' ' -f 7`
Packit 86a02d
    test $__BUILDID = $buildid
Packit 86a02d
    
Packit 86a02d
    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH`
Packit 86a02d
    hash=`cat $filename | sha1sum | awk '{print $1}'`
Packit 86a02d
    test $__SOURCESHA1 = $hash
Packit 86a02d
}
Packit 86a02d
Packit 86a02d
Packit 86a02d
# common source file sha1
Packit 86a02d
SHA=f4a1a8062be998ae93b8f1cd744a398c6de6dbb1
Packit 86a02d
# fedora30
Packit 86a02d
rpm_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
Packit 86a02d
rpm_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
Packit 86a02d
# rhel7
Packit 86a02d
rpm_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA
Packit 86a02d
rpm_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA
Packit 86a02d
# rhel6
Packit 86a02d
rpm_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA
Packit 86a02d
rpm_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA
Packit 86a02d
Packit 86a02d
RPM_BUILDID=d44d42cbd7d915bc938c81333a21e355a6022fb7 # in rhel6/ subdir, for a later test
Packit 86a02d
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Drop some of the artifacts, run a groom cycle; confirm that
Packit 86a02d
# debuginfod has forgotten them, but remembers others
Packit 86a02d
Packit 86a02d
rm -r R/debuginfod-rpms/rhel6/*
Packit 86a02d
kill -USR2 $PID1  # groom cycle
Packit 86a02d
# Expect 3 rpms to be deleted by the groom
Packit 86a02d
wait_ready $PORT1 'groom{statistic="file d/e"}' 3
Packit 86a02d
Packit 86a02d
rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests
Packit 86a02d
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $RPM_BUILDID && false || true
Packit 86a02d
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Federation mode
Packit 86a02d
Packit 86a02d
# find another unused port
Packit 86a02d
while true; do
Packit 86a02d
    PORT2=`expr '(' $RANDOM % 1000 ')' + 9000`
Packit 86a02d
    ss -atn | fgrep ":$PORT2" || break
Packit 86a02d
done
Packit 86a02d
Packit 86a02d
export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache2
Packit 86a02d
mkdir -p $DEBUGINFOD_CACHE_PATH
Packit 86a02d
# NB: inherits the DEBUGINFOD_URLS to the first server
Packit 86a02d
# NB: run in -L symlink-following mode for the L subdir
Packit 86a02d
env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod -F -d ${DB}_2 -p $PORT2 -L L &
Packit 86a02d
PID2=$!
Packit 86a02d
tempfiles ${DB}_2
Packit 86a02d
wait_ready $PORT2 'ready' 1
Packit 86a02d
Packit 86a02d
# have clients contact the new server
Packit 86a02d
export DEBUGINFOD_URLS=http://127.0.0.1:$PORT2
Packit 86a02d
rm -rf $DEBUGINFOD_CACHE_PATH
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID
Packit 86a02d
Packit 86a02d
# confirm that first server can't resolve symlinked info in L/ but second can
Packit 86a02d
BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
Packit 86a02d
         -a L/foo | grep 'Build ID' | cut -d ' ' -f 7`
Packit 86a02d
file L/foo
Packit 86a02d
file -L L/foo
Packit 86a02d
export DEBUGINFOD_URLS=http://127.0.0.1:$PORT1
Packit 86a02d
rm -rf $DEBUGINFOD_CACHE_PATH
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID && false || true
Packit 86a02d
export DEBUGINFOD_URLS=http://127.0.0.1:$PORT2
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID
Packit 86a02d
Packit 86a02d
Packit 86a02d
# test parallel queries in client
Packit 86a02d
export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache3
Packit 86a02d
mkdir -p $DEBUGINFOD_CACHE_PATH
Packit 86a02d
export DEBUGINFOD_URLS="BAD http://127.0.0.1:$PORT1 127.0.0.1:$PORT1 http://127.0.0.1:$PORT2 DNE"
Packit 86a02d
Packit 86a02d
testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Fetch some metrics, if curl program is installed
Packit 86a02d
if type curl 2>/dev/null; then
Packit 86a02d
    curl http://127.0.0.1:$PORT1/badapi
Packit 86a02d
    curl http://127.0.0.1:$PORT1/metrics
Packit 86a02d
    curl http://127.0.0.1:$PORT2/metrics
Packit 86a02d
    curl http://127.0.0.1:$PORT1/metrics | grep -q 'http_responses_total.*result.*error'
Packit 86a02d
    curl http://127.0.0.1:$PORT2/metrics | grep -q 'http_responses_total.*result.*upstream'
Packit 86a02d
fi
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Run the tests again without the servers running. The target file should
Packit 86a02d
# be found in the cache.
Packit 86a02d
Packit 86a02d
kill -INT $PID1 $PID2
Packit 86a02d
wait $PID1 $PID2
Packit 86a02d
PID1=0
Packit 86a02d
PID2=0
Packit 86a02d
tempfiles .debuginfod_*
Packit 86a02d
Packit 86a02d
testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1
Packit 86a02d
Packit 86a02d
########################################################################
Packit 86a02d
Packit 86a02d
# Trigger a cache clean and run the tests again. The clients should be unable to
Packit 86a02d
# find the target.
Packit 86a02d
echo 0 > $DEBUGINFOD_CACHE_PATH/cache_clean_interval_s
Packit 86a02d
echo 0 > $DEBUGINFOD_CACHE_PATH/max_unused_age_s
Packit 86a02d
Packit 86a02d
testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1
Packit 86a02d
Packit 86a02d
testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID2 && false || true
Packit 86a02d
Packit 86a02d
exit 0