[Cluster-devel] conga/luci cluster/index_html site/luci/Extens ...
rmccabe at sourceware.org
rmccabe at sourceware.org
Wed Jun 27 06:19:23 UTC 2007
CVSROOT: /cvs/cluster
Module name: conga
Branch: RHEL5
Changes by: rmccabe at sourceware.org 2007-06-27 06:19:22
Modified files:
luci/cluster : index_html
luci/site/luci/Extensions: HelperFunctions.py LuciDB.py
RicciQueries.py cluster_adapters.py
ricci_communicator.py
luci/storage : index_html
Log message:
Add support for issuing ricci batch jobs on all cluster nodes simultaneously
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/cluster/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.20.2.10&r2=1.20.2.11
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.4.2.3&r2=1.4.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciDB.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/RicciQueries.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.120.2.31&r2=1.120.2.32
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/ricci_communicator.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.9.2.11&r2=1.9.2.12
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/storage/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.7.2.2&r2=1.7.2.3
--- conga/luci/cluster/index_html 2007/06/18 20:14:55 1.20.2.10
+++ conga/luci/cluster/index_html 2007/06/27 06:19:21 1.20.2.11
@@ -44,7 +44,7 @@
<tal:block tal:condition="firsttime">
<tal:block tal:define="global busywaiting python:True" />
<meta http-equiv="refresh"
- tal:attributes="content isBusy/refreshurl | string:." />
+ tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
</tal:block>
<tal:block tal:define="global busy isBusy/busy | nothing" />
@@ -52,7 +52,7 @@
<tal:block tal:condition="busy">
<tal:block tal:define="global busywaiting python:True" />
<meta http-equiv="refresh"
- tal:attributes="content isBusy/refreshurl | string:." />
+ tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
</tal:block>
</tal:block>
</metal:headslot>
--- conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/18 18:39:32 1.4.2.3
+++ conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/27 06:19:22 1.4.2.4
@@ -18,11 +18,13 @@
return '%s; %s' % (str1, str2)
class Worker(threading.Thread):
- def __init__(self, mutex, hosts, riccis):
+ def __init__(self, mutex, hosts, riccis, func=None, *args):
threading.Thread.__init__(self)
self.mutex = mutex
self.hosts = hosts
self.riccis = riccis
+ self.query_func = func
+ self.query_args = args
def run(self):
while True:
@@ -32,21 +34,45 @@
return
host = self.hosts.pop()
self.mutex.release()
- r = None
+
+ r = { 'ricci': None }
try:
- r = RicciCommunicator(host)
+ rc = RicciCommunicator(host)
+ r['ricci'] = rc
+
+ if self.query_func is not None:
+ if self.query_args:
+ args = list(self.query_args)
+ else:
+ args = list()
+ args.insert(0, rc)
+ r['batch_result'] = self.query_func(*args)
except Exception, e:
- #print host, 'failed', str(e)
- pass
- except:
- #print host, 'failed'
pass
self.mutex.acquire()
self.riccis[host] = r
self.mutex.release()
+def send_batch_to_hosts(system_list, max_threads, func, *args):
+ mutex = threading.RLock()
+ threads = list()
+ hosts = list()
+ num_hosts = 0
+ ret = {}
+
+ for host in system_list:
+ hosts.append(host)
+ num_hosts += 1
+ if num_hosts <= max_threads:
+ threads.append(Worker(mutex, hosts, ret, func, *args))
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ return ret
+
def get_system_info(self, system_list):
mutex = threading.RLock()
hive = [] # workers
@@ -72,7 +98,7 @@
trusted = ''
authed = False
trusted = False
- ricci = ss[hostname]
+ ricci = ss[hostname]['ricci']
if ricci is not None:
OS = ricci.os()
--- conga/luci/site/luci/Extensions/LuciDB.py 2007/06/18 18:39:32 1.1.4.1
+++ conga/luci/site/luci/Extensions/LuciDB.py 2007/06/27 06:19:22 1.1.4.2
@@ -6,7 +6,6 @@
# Free Software Foundation.
from AccessControl import getSecurityManager
-import RicciQueries as rq
from ricci_communicator import RicciCommunicator
from LuciZopePerm import isAdmin
from LuciSyslog import get_logger
@@ -158,7 +157,7 @@
return True
batch_id = item[1].getProperty(BATCH_ID)
- batch_ret = rq.checkBatch(rc, batch_id)
+ batch_ret = rc.batch_status(batch_id)
finished = batch_ret[0]
if finished is True or finished == -1:
if finished == -1:
--- conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/18 18:39:32 1.1.4.1
+++ conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/27 06:19:22 1.1.4.2
@@ -6,52 +6,12 @@
# Free Software Foundation.
from xml.dom import minidom
-from ricci_communicator import RicciCommunicator, extract_module_status
+from ricci_communicator import RicciCommunicator
from LuciSyslog import get_logger
from conga_constants import LUCI_DEBUG_MODE
luci_log = get_logger()
-def checkBatch(rc, batch_id):
- err_msg = 'An unknown Ricci error occurred on %s' % rc.hostname()
-
- try:
- batch = rc.batch_report(batch_id)
- if batch is None:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch0: batch id %s not found' \
- % batch_id)
- return (True, 'batch id %s was not found' % batch_id)
- except Exception, e:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch1: %s: %r %s' \
- % (rc.hostname(), e, str(e)))
- return (-1, err_msg)
-
- try:
- code, new_err_msg = extract_module_status(batch, 1)
- if new_err_msg:
- err_msg = 'A Ricci error occurred on %s: %s' \
- % (rc.hostname(), str(new_err_msg))
- except Exception, e:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch2: %s: %r %s' \
- % (rc.hostname(), e, str(e)))
- return (-1, err_msg)
-
- # In progress.
- if code == -101 or code == -102:
- return (False, 'in progress')
-
- # Done successfully.
- if code == 0:
- return (True, 'completed sucessfully')
-
- # Error
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch3: %d: %s' % (code, rc.hostname()))
- return (-1, err_msg)
-
def addClusterNodeBatch(cluster_name,
install_base,
install_services,
@@ -310,9 +270,9 @@
for i in batch:
try:
- batch_number = i.getAttribute('batch_id')
- result = i.getAttribute('status')
- return (str(batch_number), str(result))
+ batch_number = str(i.getAttribute('batch_id'))
+ result = str(i.getAttribute('status'))
+ return (batch_number, result)
except Exception, e:
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('BAR1: %s' % e)
--- conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/19 15:54:10 1.120.2.31
+++ conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/27 06:19:22 1.120.2.32
@@ -640,7 +640,7 @@
# abort the whole process.
try:
while True:
- batch_ret = rq.checkBatch(cluster_ricci, batch_number)
+ batch_ret = cluster_ricci.batch_status(batch_number)
code = batch_ret[0]
if code is True:
break
@@ -3106,7 +3106,7 @@
if rc is not None:
batch_num = item[1].getProperty(BATCH_ID)
- batch_res = rq.checkBatch(rc, batch_num)
+ batch_res = rc.batch_status(batch_num)
finished = batch_res[0]
err_msg = batch_res[1]
--- conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/18 18:39:33 1.9.2.11
+++ conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/27 06:19:22 1.9.2.12
@@ -53,7 +53,10 @@
% (self.__hostname, hello.toxml()))
except:
if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('RC:init0: error receiving header from %s: %r %s' % (self.__hostname, e, str(e)))
+ luci_log.debug_verbose('RC:init0: error receiving header from %s:%d: %r %s' % (self.__hostname, self.__port, e, str(e)))
+ else:
+ luci_log.info('Error reading from %s:%d: %s' \
+ % (self.__hostname, self.__port, str(e)))
self.__authed = hello.firstChild.getAttribute('authenticated') == 'true'
self.__cluname = hello.firstChild.getAttribute('clustername')
@@ -231,8 +234,8 @@
if doc.firstChild.getAttribute('success') != '0':
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('RC:PB4: batch command failed')
- raise RicciError, 'The last ricci command to host %s failed' \
- % self.__hostname
+ raise RicciError, 'The last ricci command to host %s:%d failed' \
+ % (self.__hostname, self.__port)
batch_node = None
for node in doc.firstChild.childNodes:
@@ -243,8 +246,8 @@
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('RC:PB5: no <batch/> from %s' \
% self.__hostname)
- raise RicciError, 'missing <batch/> in ricci response from "%s"' \
- % self.__hostname
+ raise RicciError, 'missing <batch/> in ricci response from %s:%d' \
+ % (self.__hostname, self.__port)
return batch_node
@@ -264,7 +267,7 @@
ricci_xml = self.process_batch(batch_xml, async)
if LUCI_DEBUG_NET is True:
try:
- luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command.' % (ricci_xml.toxml(), self.__hostname))
+ luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command' % (ricci_xml.toxml(), self.__hostname))
except:
pass
except:
@@ -276,6 +279,48 @@
doc.appendChild(ricci_xml)
return doc
+ def batch_status(self, batch_id):
+ err_msg = 'An unknown ricci error occurred on %s:%d' \
+ % (self.__hostname, self.__port)
+
+ try:
+ batch = self.batch_report(batch_id)
+ if batch is None:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB0: batch id %s not found' \
+ % batch_id)
+ return (True, 'batch id %s was not found' % batch_id)
+ except Exception, e:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB1: %s:%d: %r %s' \
+ % (self.__hostname, self.__port, e, str(e)))
+ return (-1, err_msg)
+
+ try:
+ code, new_err_msg = extract_module_status(batch, 1)
+ if new_err_msg:
+ err_msg = 'A ricci error occurred on %s:%d: %s' \
+ % (self.__hostname, self.__port, str(new_err_msg))
+ except Exception, e:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB2: %s:%d %r %s: %s' \
+ % (self.__hostname, self.__port, e, str(e), err_msg))
+ return (-1, err_msg)
+
+ # In progress.
+ if code == -101 or code == -102:
+ return (False, 'in progress')
+
+ # Done successfully.
+ if code == 0:
+ return (True, 'completed sucessfully')
+
+ # Error
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB3: %s:%d code %d: %s' \
+ % (self.__hostname, self.__port, code, err_msg))
+ return (-1, err_msg)
+
def batch_report(self, batch_id):
if LUCI_DEBUG_NET is True:
luci_log.debug_net('RC:BRep0: [auth=%d] asking for batchid# %s for host %s' % (self.__authed, batch_id, self.__hostname))
@@ -374,6 +419,9 @@
return doc
def get_ricci_communicator(self, hostname, allowed_systems):
+ if not hostname:
+ return None
+
if not self.access_to_host_allowed(hostname, allowed_systems):
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('GRC0: access to host %s is not allowed' % hostname)
--- conga/luci/storage/index_html 2007/06/26 17:00:53 1.7.2.2
+++ conga/luci/storage/index_html 2007/06/27 06:19:22 1.7.2.3
@@ -222,7 +222,7 @@
<h1>Future Site of Forms</h1>
</metal:main-form-content>
<div style="padding-top: 10px;"
- tal:condition="report_cached">
+ tal:condition="python: report_cached and not (request.has_key('pagetype') and request['pagetype']=='commit_changes')">
<form>
<input type="hidden" name="reprobe_url"
tal:attributes="value
More information about the Cluster-devel
mailing list