[libvirt] [PATCH 4/4] Introduce new OOM testing support
Daniel P. Berrange
berrange at redhat.com
Wed Sep 25 17:23:38 UTC 2013
From: "Daniel P. Berrange" <berrange at redhat.com>
The previous OOM testing support would re-run the entire "main"
method each iteration, failing a different malloc each time.
When a test suite has 'n' allocations, the number of repeats
requires is (n * (n + 1) ) / 2. This gets very large, very
quickly.
This new OOM testing support instead integrates at the
virtTestRun level, so each individual test case gets repeated,
instead of the entire test suite. This means the values of
'n' are orders of magnitude smaller.
The simple usage is
$ VIR_TEST_OOM=1 ./qemuxml2argvtest
...
29) QEMU XML-2-ARGV clock-utc ... OK
Test OOM for nalloc=36 .................................... OK
30) QEMU XML-2-ARGV clock-localtime ... OK
Test OOM for nalloc=36 .................................... OK
31) QEMU XML-2-ARGV clock-france ... OK
Test OOM for nalloc=38 ...................................... OK
...
the second lines reports how many mallocs have to be failed, and thus
how many repeats of the test wil be run.
If it crashes, then running under valgrind will often show the problem
$ VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest
When debugging problems it is also helpful to select an individual
test case
$ VIR_TEST_RANGE=30 VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest
When things get really tricky, it is possible to request that just
specific allocs are failed. eg to fail allocs 5 -> 12, use
$ VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-12 ../run valgrind ./qemuxml2argvtest
In the worse case, you might want to know the stack trace of the
alloc which was failed then VIR_TEST_OOM_TRACE can be set. If it
is set to 1 then it will only print if it things a mistake happened.
This is often not reliable, so setting it to 2 will make it print
the stack trace for every alloc that is failed.
$ VIR_TEST_OOM_TRACE=2 VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-5 ../run valgrind ./qemuxml2argvtest
30) QEMU XML-2-ARGV clock-localtime ... OK
Test OOM for nalloc=36 !virAllocN
/home/berrange/src/virt/libvirt/src/util/viralloc.c:180
virHashCreateFull
/home/berrange/src/virt/libvirt/src/util/virhash.c:144
virDomainDefParseXML
/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11745
virDomainDefParseNode
/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12646
virDomainDefParse
/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12590
testCompareXMLToArgvFiles
/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:106
virtTestRun
/home/berrange/src/virt/libvirt/tests/testutils.c:250
mymain
/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:418 (discriminator 2)
virtTestMain
/home/berrange/src/virt/libvirt/tests/testutils.c:750
??
??:0
_start
??:?
FAILED
Signed-off-by: Daniel P. Berrange <berrange at redhat.com>
---
tests/cputest.c | 3 +-
tests/qemuargv2xmltest.c | 14 ++--
tests/qemuxml2argvtest.c | 12 ++-
tests/qemuxmlnstest.c | 18 +++--
tests/testutils.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++-
tests/testutils.h | 2 +
6 files changed, 216 insertions(+), 22 deletions(-)
diff --git a/tests/cputest.c b/tests/cputest.c
index 8e3640b..17cb6af 100644
--- a/tests/cputest.c
+++ b/tests/cputest.c
@@ -476,7 +476,8 @@ cpuTestRun(const char *name, const struct data *data)
if (virTestGetDebug()) {
char *log;
if ((log = virtTestLogContentAndReset()) &&
- strlen(log) > 0)
+ log != NULL &&
+ strlen(log) > 0)
fprintf(stderr, "\n%s\n", log);
VIR_FREE(log);
}
diff --git a/tests/qemuargv2xmltest.c b/tests/qemuargv2xmltest.c
index 6dd8bb0..92fc89a 100644
--- a/tests/qemuargv2xmltest.c
+++ b/tests/qemuargv2xmltest.c
@@ -42,7 +42,6 @@ static int testCompareXMLToArgvFiles(const char *xml,
char *cmd = NULL;
int ret = -1;
virDomainDefPtr vmdef = NULL;
- char *log;
if (virtTestLoadFile(cmdfile, &cmd) < 0)
goto fail;
@@ -53,13 +52,16 @@ static int testCompareXMLToArgvFiles(const char *xml,
cmd, NULL, NULL, NULL)))
goto fail;
- if ((log = virtTestLogContentAndReset()) == NULL)
- goto fail;
- if ((*log != '\0') != expect_warning) {
+ if (!virtTestOOMActive()) {
+ char *log;
+ if ((log = virtTestLogContentAndReset()) == NULL)
+ goto fail;
+ if ((*log != '\0') != expect_warning) {
+ VIR_FREE(log);
+ goto fail;
+ }
VIR_FREE(log);
- goto fail;
}
- VIR_FREE(log);
if (!(actualxml = virDomainDefFormat(vmdef, 0)))
goto fail;
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 5658792..4feae7c 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -106,7 +106,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
if (!(vmdef = virDomainDefParseFile(xml, driver.caps, driver.xmlopt,
QEMU_EXPECTED_VIRT_TYPES,
VIR_DOMAIN_XML_INACTIVE))) {
- if (flags & FLAG_EXPECT_PARSE_ERROR)
+ if (!virtTestOOMActive() &&
+ (flags & FLAG_EXPECT_PARSE_ERROR))
goto ok;
goto out;
}
@@ -159,7 +160,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
migrateFrom, migrateFd, NULL,
VIR_NETDEV_VPORT_PROFILE_OP_NO_OP,
&testCallbacks))) {
- if (flags & FLAG_EXPECT_FAILURE) {
+ if (!virtTestOOMActive() &&
+ (flags & FLAG_EXPECT_FAILURE)) {
ret = 0;
if (virTestGetDebug() > 1)
fprintf(stderr, "Got expected error: %s\n",
@@ -173,7 +175,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
goto out;
}
- if (!!virGetLastError() != !!(flags & FLAG_EXPECT_ERROR)) {
+ if (!virtTestOOMActive() &&
+ (!!virGetLastError() != !!(flags & FLAG_EXPECT_ERROR))) {
if (virTestGetDebug() && (log = virtTestLogContentAndReset()))
fprintf(stderr, "\n%s", log);
goto out;
@@ -194,7 +197,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
}
ok:
- if (flags & FLAG_EXPECT_ERROR) {
+ if (!virtTestOOMActive() &&
+ (flags & FLAG_EXPECT_ERROR)) {
/* need to suppress the errors */
virResetLastError();
}
diff --git a/tests/qemuxmlnstest.c b/tests/qemuxmlnstest.c
index 2cc15d1..38c98bd 100644
--- a/tests/qemuxmlnstest.c
+++ b/tests/qemuxmlnstest.c
@@ -117,15 +117,17 @@ static int testCompareXMLToArgvFiles(const char *xml,
&testCallbacks)))
goto fail;
- if (!!virGetLastError() != expectError) {
- if (virTestGetDebug() && (log = virtTestLogContentAndReset()))
- fprintf(stderr, "\n%s", log);
- goto fail;
- }
+ if (!virtTestOOMActive()) {
+ if (!!virGetLastError() != expectError) {
+ if (virTestGetDebug() && (log = virtTestLogContentAndReset()))
+ fprintf(stderr, "\n%s", log);
+ goto fail;
+ }
- if (expectError) {
- /* need to suppress the errors */
- virResetLastError();
+ if (expectError) {
+ /* need to suppress the errors */
+ virResetLastError();
+ }
}
if (!(actualargv = virCommandToString(cmd)))
diff --git a/tests/testutils.c b/tests/testutils.c
index 7d020db..56728e0 100644
--- a/tests/testutils.c
+++ b/tests/testutils.c
@@ -47,6 +47,9 @@
#include "virprocess.h"
#include "virstring.h"
+#include <dlfcn.h>
+#include <execinfo.h>
+
#ifdef HAVE_PATHS_H
# include <paths.h>
#endif
@@ -64,6 +67,18 @@ static unsigned int testDebug = -1;
static unsigned int testVerbose = -1;
static unsigned int testExpensive = -1;
+#ifdef TEST_OOM
+static unsigned int testOOM = 0;
+static unsigned int testOOMStart = -1;
+static unsigned int testOOMEnd = -1;
+static unsigned int testOOMTrace = 0;
+# ifdef TEST_OOM_TRACE
+void *testAllocStack[30];
+int ntestAllocStack;
+# endif
+#endif
+static bool testOOMActive = false;
+
static size_t testCounter = 0;
static size_t testStart = 0;
static size_t testEnd = 0;
@@ -71,6 +86,20 @@ static size_t testEnd = 0;
char *progname;
char *abs_srcdir;
+
+bool virtTestOOMActive(void)
+{
+ return testOOMActive;
+}
+
+#ifdef TEST_OOM_TRACE
+static void virTestAllocHook(int nalloc ATTRIBUTE_UNUSED,
+ void *opaque ATTRIBUTE_UNUSED)
+{
+ ntestAllocStack = backtrace(testAllocStack, ARRAY_CARDINALITY(testAllocStack));
+}
+#endif
+
double
virtTestCountAverage(double *items, int nitems)
{
@@ -122,6 +151,35 @@ void virtTestResult(const char *name, int ret, const char *msg, ...)
va_end(vargs);
}
+#ifdef TEST_OOM_TRACE
+static void
+virTestShowTrace(void)
+{
+ size_t j;
+ for (j = 2; j < ntestAllocStack; j++) {
+ Dl_info info;
+ char *cmd;
+
+ dladdr(testAllocStack[j], &info);
+ if (info.dli_fname &&
+ strstr(info.dli_fname, ".so")) {
+ if (virAsprintf(&cmd, "addr2line -f -e %s %p",
+ info.dli_fname,
+ ((void*)((unsigned long long)testAllocStack[j]
+ - (unsigned long long)info.dli_fbase))) < 0)
+ continue;
+ } else {
+ if (virAsprintf(&cmd, "addr2line -f -e %s %p",
+ (char*)(info.dli_fname ? info.dli_fname : "<unknown>"),
+ testAllocStack[j]) < 0)
+ continue;
+ }
+ ignore_value(system(cmd));
+ VIR_FREE(cmd);
+ }
+}
+#endif
+
/*
* Runs test
*
@@ -168,7 +226,7 @@ virtTestRun(const char *title,
!((testCounter-1) % 40)) {
fprintf(stderr, " %-3zu\n", (testCounter-1));
fprintf(stderr, " ");
- }
+ }
if (ret == 0)
fprintf(stderr, ".");
else if (ret == EXIT_AM_SKIP)
@@ -177,6 +235,77 @@ virtTestRun(const char *title,
fprintf(stderr, "!");
}
+#ifdef TEST_OOM
+ if (testOOM && ret != EXIT_AM_SKIP) {
+ int nalloc;
+ int oomret;
+ int start, end;
+ size_t i;
+ virResetLastError();
+ virAllocTestInit();
+# ifdef TEST_OOM_TRACE
+ virAllocTestHook(virTestAllocHook, NULL);
+# endif
+ oomret = body(data);
+ nalloc = virAllocTestCount();
+ fprintf(stderr, " Test OOM for nalloc=%d ", nalloc);
+ if (testOOMStart == -1 ||
+ testOOMEnd == -1) {
+ start = 0;
+ end = nalloc;
+ } else {
+ start = testOOMStart;
+ end = testOOMEnd + 1;
+ }
+ testOOMActive = true;
+ for (i = start; i < end; i++) {
+ bool missingFail = false;
+# ifdef TEST_OOM_TRACE
+ memset(testAllocStack, 0, ARRAY_CARDINALITY(testAllocStack));
+ ntestAllocStack = 0;
+# endif
+ virAllocTestOOM(i+1, 1);
+ oomret = body(data);
+
+ /* fprintf() disabled because XML parsing APIs don't allow
+ * distinguish between element / attribute not present
+ * in the XML (which is non-fatal), vs OOM / malformed
+ * which should be fatal. Thus error reporting for
+ * optionally present XML is mostly broken.
+ */
+ if (oomret == 0) {
+ missingFail = true;
+# if 0
+ fprintf(stderr, " alloc %zu failed but no err status\n", i + 1);
+# endif
+ } else {
+ virErrorPtr lerr = virGetLastError();
+ if (!lerr) {
+# if 0
+ fprintf(stderr, " alloc %zu failed but no error report\n", i + 1);
+# endif
+ missingFail = true;
+ }
+ }
+ if ((missingFail && testOOMTrace) || (testOOMTrace > 1)) {
+ fprintf(stderr, "%s", "!");
+# ifdef TEST_OOM_TRACE
+ virTestShowTrace();
+# endif
+ ret = -1;
+ } else {
+ fprintf(stderr, "%s", ".");
+ }
+ }
+ testOOMActive = false;
+ if (ret == 0)
+ fprintf(stderr, " OK\n");
+ else
+ fprintf(stderr, " FAILED\n");
+ virAllocTestInit();
+ }
+#endif /* TEST_OOM */
+
return ret;
}
@@ -205,7 +334,7 @@ virtTestLoadFile(const char *file, char **buf)
tmplen = buflen = st.st_size + 1;
if (VIR_ALLOC_N(*buf, buflen) < 0) {
- fprintf(stderr, "%s: larger than available memory (> %d)\n", file, buflen);
+ //fprintf(stderr, "%s: larger than available memory (> %d)\n", file, buflen);
VIR_FORCE_FCLOSE(fp);
return -1;
}
@@ -481,7 +610,8 @@ virtTestLogOutput(virLogSource source ATTRIBUTE_UNUSED,
{
struct virtTestLogData *log = data;
virCheckFlags(VIR_LOG_STACK_TRACE,);
- virBufferAsprintf(&log->buf, "%s: %s", timestamp, str);
+ if (!testOOMActive)
+ virBufferAsprintf(&log->buf, "%s: %s", timestamp, str);
}
static void
@@ -550,6 +680,9 @@ int virtTestMain(int argc,
int ret;
bool abs_srcdir_cleanup = false;
char *testRange = NULL;
+#ifdef TEST_OOM
+ char *oomstr;
+#endif
abs_srcdir = getenv("abs_srcdir");
if (!abs_srcdir) {
@@ -610,6 +743,56 @@ int virtTestMain(int argc,
}
}
+#ifdef TEST_OOM
+ if ((oomstr = getenv("VIR_TEST_OOM")) != NULL) {
+ char *next;
+ if (testDebug == -1)
+ testDebug = 1;
+ testOOM = 1;
+ if (oomstr[0] != '\0' &&
+ oomstr[1] == ':') {
+ if (virStrToLong_ui(oomstr + 2, &next, 10, &testOOMStart) < 0) {
+ fprintf(stderr, "Cannot parse range %s\n", oomstr);
+ return EXIT_FAILURE;
+ }
+ if (*next != '-') {
+ fprintf(stderr, "Cannot parse range %s\n", oomstr);
+ return EXIT_FAILURE;
+ }
+ if (virStrToLong_ui(next+1, NULL, 10, &testOOMEnd) < 0) {
+ fprintf(stderr, "Cannot parse range %s\n", oomstr);
+ return EXIT_FAILURE;
+ }
+ } else {
+ testOOMStart = -1;
+ testOOMEnd = -1;
+ }
+ }
+
+# ifdef TEST_OOM_TRACE
+ if ((oomstr = getenv("VIR_TEST_OOM_TRACE")) != NULL) {
+ if (virStrToLong_ui(oomstr, NULL, 10, &testOOMTrace) < 0) {
+ fprintf(stderr, "Cannot parse oom trace %s\n", oomstr);
+ return EXIT_FAILURE;
+ }
+ }
+# else
+ if (getenv("VIR_TEST_OOM_TRACE")) {
+ fprintf(stderr, "%s", "OOM test tracing not enabled in this build\n");
+ return EXIT_FAILURE;
+ }
+# endif
+#else /* TEST_OOM */
+ if (getenv("VIR_TEST_OOM")) {
+ fprintf(stderr, "%s", "OOM testing not enabled in this build\n");
+ return EXIT_FAILURE;
+ }
+ if (getenv("VIR_TEST_OOM_TRACE")) {
+ fprintf(stderr, "%s", "OOM test tracing not enabled in this build\n");
+ return EXIT_FAILURE;
+ }
+#endif /* TEST_OOM */
+
ret = (func)();
if (abs_srcdir_cleanup)
diff --git a/tests/testutils.h b/tests/testutils.h
index 96edeb9..62caacd 100644
--- a/tests/testutils.h
+++ b/tests/testutils.h
@@ -40,6 +40,8 @@
extern char *progname;
extern char *abs_srcdir;
+bool virtTestOOMActive(void);
+
double virtTestCountAverage(double *items,
int nitems);
--
1.8.3.1
More information about the libvir-list
mailing list