[sos-devel] [PATCH v3] sosreport: Check for rpm database corruption during initialization

[Aruna Balakrishnaiah]

sosreport runs an rpm query to get the package list. If rpmdb is corrupted
sosreport hangs for ever hence check for rpmdb consistency before running
the rpm query

Signed-off-by: Aruna Balakrishnaiah <aruna at linux.vnet.ibm.com>
---
Changes from v2:
	Introduce timeout in shell_out wrapper

Changes from v1:
	Addressed issues of the maintainer
	Introduce timeout instead of relying on yum.

When rpmdb is corrupted rpm commands hangs forever,
yum check will result in:

yum check
error: rpmdb: BDB0113 Thread/process 43828/70366497037824 failed: BDB1507 Thread died in Berkeley DB library
error: db5 error(-30973) from dbenv->failchk: BDB0087 DB_RUNRECOVERY: Fatal error, run database recovery
error: cannot open Packages index using db5 -  (-30973)
error: cannot open Packages database in /var/lib/rpm
CRITICAL:yum.main:

Error: rpmdb open failed 

 sos/policies/__init__.py |    5 +++--
 sos/policies/redhat.py   |   10 +++++++++-
 sos/utilities.py         |    4 ++--
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py
index 9fcbd55..f048e43 100644
--- a/sos/policies/__init__.py
+++ b/sos/policies/__init__.py
@@ -57,8 +57,9 @@ class PackageManager(object):
 
     query_command = None
 
-    def __init__(self, query_command=None):
+    def __init__(self, query_command=None, timeout=180):
         self.packages = {}
+        self.timeout = timeout
         if query_command:
             self.query_command = query_command
 
@@ -92,7 +93,7 @@ class PackageManager(object):
                           version': 'major.minor.version'}}
         """
         if self.query_command:
-            pkg_list = shell_out(self.query_command).splitlines()
+            pkg_list = shell_out(self.query_command, self.timeout).splitlines()
             for pkg in pkg_list:
                 if '|' not in pkg:
                     continue
diff --git a/sos/policies/redhat.py b/sos/policies/redhat.py
index 2219246..e5a3f3f 100644
--- a/sos/policies/redhat.py
+++ b/sos/policies/redhat.py
@@ -21,6 +21,7 @@ import sys
 from sos.plugins import RedHatPlugin
 from sos.policies import LinuxPolicy, PackageManager
 from sos import _sos as _
+from distutils.log import error
 
 sys.path.insert(0, "/usr/share/rhn/")
 try:
@@ -46,8 +47,15 @@ class RedHatPolicy(LinuxPolicy):
             'rpm -qa --queryformat "%{NAME}|%{VERSION}\\n"')
         self.valid_subclasses = [RedHatPlugin]
 
+        pkgs = self.package_manager.all_pkgs()
+
+        # If rpm query timed out after timeout duration exit
+        if not pkgs:
+            error("Could not obtain installed package list")
+            sys.exit(1)
+
         # handle PATH for UsrMove
-        if self.package_manager.all_pkgs()['filesystem']['version'][0] == '3':
+        if pkgs['filesystem']['version'][0] == '3':
             self.PATH = "/usr/sbin:/usr/bin:/root/bin"
         else:
             self.PATH = "/sbin:/bin:/usr/sbin:/usr/bin:/root/bin"
diff --git a/sos/utilities.py b/sos/utilities.py
index 7e8cd7e..f3eea57 100644
--- a/sos/utilities.py
+++ b/sos/utilities.py
@@ -178,11 +178,11 @@ def import_module(module_fqname, superclasses=None):
     return modules
 
 
-def shell_out(cmd, runat=None):
+def shell_out(cmd, timeout=180, runat=None):
     """Shell out to an external command and return the output or the empty
     string in case of error.
     """
-    return sos_get_command_output(cmd, runat=runat)['output']
+    return sos_get_command_output(cmd, timeout=timeout, runat=runat)['output']
 
 
 class ImporterHelper(object):