extras-buildsys/client buildclient.py,1.16,1.17
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Sat Jun 25 19:19:06 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/client
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv11449/client
Modified Files:
buildclient.py
Log Message:
2005-06-25 Dan Williams <dcbw at redhat.com>
* client/buildclient.py
- Break out some functions from process() to make code easier to follow
- Use unique extensions on buildroots so more than one build client
can run at the same time on the same machine
- Execute mock after the build to clean up the buildroot before
deleting the buildroot directory
Requires latest mock from CVS
Index: buildclient.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/client/buildclient.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -r1.16 -r1.17
--- buildclient.py 24 Jun 2005 21:07:30 -0000 1.16
+++ buildclient.py 25 Jun 2005 19:19:04 -0000 1.17
@@ -71,6 +71,7 @@
self._target = target
self._srpm_url = srpm_url
self._log_fd = None
+ self._mock_config = None
self._result_dir = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "result")
if not os.path.exists(self._result_dir):
@@ -150,9 +151,9 @@
os.makedirs(self._result_dir)
if not os.path.exists(self._result_dir):
os.makedirs(self._result_dir)
- cmd = '%s %s -r %s --resultdir=%s --statedir=%s %s' % (self.arch_command,
+ cmd = '%s %s -r %s --resultdir=%s --statedir=%s --uniqueext=%s %s' % (self.arch_command,
CONFIG.get('builder_cmd'), self.buildroot,
- self._result_dir, self._state_dir, self._srpm_path)
+ self._result_dir, self._state_dir, self._uniqid, self._srpm_path)
self.log(" %s\n" % cmd)
self._pobj = popen2.Popen4(cmd=cmd, bufsize=1024)
fcntl.fcntl(self._pobj.fromchild.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
@@ -164,6 +165,14 @@
mockstatusfile = os.path.join(self._state_dir, 'status')
while not os.path.exists(mockstatusfile):
time.sleep(0.5)
+ # if mock exited with an error report that error and not
+ # the missing status file.
+ exit_status = self._pobj.poll()
+ if exit_status > 0:
+ self._status = 'failed'
+ break
+
+ # Kill mock after 5s if it didn't dump the status file
if time.time() - start_time > 5:
self.log("Timed out waiting for the mock status file! %s\n" % mockstatusfile)
try:
@@ -176,9 +185,27 @@
self._status = 'failed'
break
+ def _cleanup(self):
+ self.log("Cleaning up the buildroot...\n")
+ cmd = '%s %s clean --uniqueext=%s -r %s' % (self.arch_command,
+ CONFIG.get('builder_cmd'), self._uniqid,
+ self.buildroot)
+ self.log(" %s\n" % cmd)
+ self._pobj = popen2.Popen4(cmd=cmd)
+ self._status = 'cleanup'
+
def _mock_is_prepping(self):
mock_status = self._get_mock_status()
if mock_status:
+ if mock_status == 'prep':
+ return True
+ elif mock_status == 'setu':
+ return True
+ return False
+
+ def _mock_using_repo(self):
+ mock_status = self._get_mock_status()
+ if mock_status:
if mock_status == 'init':
return True
elif mock_status == 'clea':
@@ -198,7 +225,6 @@
def _get_mock_status(self):
mockstatusfile = os.path.join(self._state_dir, 'status')
if not os.path.exists(mockstatusfile):
- self.log("mock status file doesn't exist! %s" % mockstatusfile)
return None
f = open(mockstatusfile, "r")
@@ -220,64 +246,96 @@
string = string.lower()
return string
- def process(self):
- if not self.is_done_status():
- if self._status == 'downloading':
- pass
- elif self._status == 'downloaded':
- # We can't start doing anything with yum until the build
- # server tells us the repo is unlocked.
- if not self._repo_locked:
- self._build()
- else:
- # Only show this message once
- if not self._repo_locked_msg:
- self.log("Waiting for repository to unlock before starting the build...\n")
- self._repo_locked_msg = True
- elif self._status == 'prepping':
- if not self._mock_is_prepping():
- self._status = 'building'
- elif self._status == 'building':
- exit_status = self._pobj.poll()
- if exit_status == 0:
- # mock completed successfully
- if self._status != 'building':
- self.log("Bad job end status %s encountered!" % self._status)
- self._status = 'done'
- elif exit_status > 0:
- # mock exited with an error
- self._status = 'failed'
-
- if self._pobj:
- # Grab any mock output and write it to a log
- string = ' '
- while len(string) > 0:
- try:
- string = os.read(self._pobj.fromchild.fileno(), 1024)
- except OSError, e:
- if e.errno == errno.EAGAIN: # Resource temporarily unavailable
- break
- else:
- self.log("Error reading mock output: %s\n" % e)
+ def _read_mock_config(self):
+ mockconfigfile = os.path.join(self._result_dir, 'mockconfig.log')
+ if not os.path.exists(mockconfigfile):
+ return None
+
+ f = open(mockconfigfile, "r")
+ contents = {}
+ for line in f:
+ (item, loc) = line.split('=')
+ item = item.strip()
+ loc = loc.strip()
+ contents[item] = loc
+ f.close()
+ return contents
+
+ def _grab_mock_output(self):
+ # We don't care about output from the 'cleanup' stage
+ if self._pobj and self._status != 'cleanup':
+ # Grab any mock output and write it to a log
+ string = ' '
+ while len(string) > 0:
+ try:
+ string = os.read(self._pobj.fromchild.fileno(), 1024)
+ except OSError, e:
+ if e.errno == errno.EAGAIN: # Resource temporarily unavailable
+ break
else:
- self._log_fd.write(string)
- self._log_fd.flush()
- os.fsync(self._log_fd.fileno())
-
- if self.is_done_status():
- self._files = self._find_files()
- self.log("\n\n-----------------------\n\n")
- if self._status == 'done':
- self.log("Job completed successfully.\n")
- elif self._status == 'failed':
- exit_status = self._pobj.poll()
- self.log("Job failed due to mock errors! mock exit status: %d\n" % exit_status)
- elif self._status == 'killed':
- self.log("Job failed because it was killed.\n")
- else:
- if self._log_fd:
- self._log_fd.close()
- self._log_fd = None
+ self.log("Error reading mock output: %s\n" % e)
+ else:
+ self._log_fd.write(string)
+ self._log_fd.flush()
+ os.fsync(self._log_fd.fileno())
+
+ def _mock_done(self):
+ self._files = self._find_files()
+ self.log("\n\n-----------------------\n\n")
+ if self._status == 'done':
+ self.log("Job completed successfully.\n")
+ elif self._status == 'failed':
+ exit_status = self._pobj.poll()
+ self.log("Job failed due to mock errors! mock exit status: %d\n" % exit_status)
+ elif self._status == 'killed':
+ self.log("Job failed because it was killed.\n")
+
+ if self._log_fd:
+ self._log_fd.close()
+ self._log_fd = None
+
+ def process(self):
+ if self.is_done_status():
+ return
+
+ if self._status == 'downloading':
+ pass
+ elif self._status == 'downloaded':
+ # We can't start doing anything with yum until the build
+ # server tells us the repo is unlocked.
+ if not self._repo_locked:
+ self._build()
+ else:
+ # Only show this message once
+ if not self._repo_locked_msg:
+ self.log("Waiting for repository to unlock before starting the build...\n")
+ self._repo_locked_msg = True
+ elif self._status == 'prepping':
+ if not self._mock_config and self._mock_is_prepping():
+ self._mock_config = self._read_mock_config()
+ if not self._mock_using_repo():
+ self._status = 'building'
+ elif self._status == 'building':
+ exit_status = self._pobj.poll()
+ if exit_status == 0:
+ # mock completed successfully
+ if self._status != 'building':
+ self.log("Bad job end status %s encountered!" % self._status)
+ self._cleanup()
+ elif exit_status > 0:
+ # mock exited with an error
+ self._status = 'failed'
+ elif self._status == 'cleanup':
+ exit_status = self._pobj.poll()
+ if exit_status >= 0:
+ # We ignore mock errors when cleaning the buildroot
+ self._status = 'done'
+ if self._mock_config.has_key('rootdir'):
+ shutil.rmtree(self._mock_config['rootdir'], ignore_errors=True)
+
+ self._grab_mock_output()
+ if self.is_done_status():
+ self._mock_done()
def _find_files(self):
# Grab the list of files in our job's result dir and URL encode them
@@ -397,12 +455,12 @@
self.localarches = localarches
self.cur_job = 0
- def _process(self):
+ def process(self):
# Give jobs some time to update their status and do their thing
job = 0
for (uniqid, bcp) in self.ids.iteritems():
- bcp.process()
if not bcp.is_done_status():
+ bcp.process()
job = uniqid
self.cur_job = job # Update current job
@@ -514,7 +572,7 @@
cur_time = time.time()
if cur_time >= last_time + 3:
# do some work every 3s or so
- bcs._process()
+ bcs.process()
last_time = time.time()
os._exit(0)
More information about the fedora-extras-commits
mailing list