extras-buildsys/server ArchJob.py, 1.2, 1.3 Builder.py, 1.2, 1.3 PackageJob.py, 1.1, 1.2 Repo.py, 1.6, 1.7
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Thu Jul 7 16:21:14 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv30317/server
Modified Files:
ArchJob.py Builder.py PackageJob.py Repo.py
Log Message:
2005-07-07 Dan Williams <dcbw at redhat.com>
* server/PackageJob.py
- Use Python exceptions for prep and build errors
- Clean up a bunch of stuff
- Specify dest file when copying files to the repo
* server/Repo.py
- Ensure that the destination file is actually the file rather
than a path, since shutil.copy() just copies the file over top
of the directory in some instances, corrupting the repo
Index: ArchJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/ArchJob.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- ArchJob.py 6 Jul 2005 21:21:00 -0000 1.2
+++ ArchJob.py 7 Jul 2005 16:21:12 -0000 1.3
@@ -182,6 +182,7 @@
return self.status
def get_files(self):
+ """ Return a list of base filenames we got from the builder """
files = []
for url in self.downloads.keys():
fname = FileDownloader.get_base_filename_from_url(url, ['.rpm', '.log'])
Index: Builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Builder.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- Builder.py 6 Jul 2005 21:21:00 -0000 1.2
+++ Builder.py 7 Jul 2005 16:21:12 -0000 1.3
@@ -23,6 +23,7 @@
import threading
from plague import XMLRPCServerProxy
from plague import CommonErrors
+import OpenSSL
import ArchJob
# Load in the config
@@ -104,7 +105,7 @@
self._server_lock.acquire()
try:
(jobid, status) = self._server.get_cur_job()
- except (socket.error, xmlrpclib.ProtocolError):
+ except (socket.error, OpenSSL.SSL.SysCallError, xmlrpclib.ProtocolError):
self._unavail_count = self._unavail_count + 1
else:
self._unavail_count = 0
Index: PackageJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/PackageJob.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- PackageJob.py 5 Jul 2005 21:08:03 -0000 1.1
+++ PackageJob.py 7 Jul 2005 16:21:12 -0000 1.2
@@ -1,4 +1,4 @@
-# This program is free software; you can redistribute it and/or modify
+ # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
@@ -51,18 +51,16 @@
print stuff
class PrepError(exceptions.Exception):
- def __init__(self, errno=0, args=None):
+ def __init__(self, args=None):
exceptions.Exception.__init__(self)
self.args = args
- self.errno = errno
def __str__(self):
return self.args
class BuildError(exceptions.Exception):
- def __init__(self, errno=0, args=None):
+ def __init__(self, args=None):
exceptions.Exception.__init__(self)
self.args = args
- self.errno = errno
def __str__(self):
return self.args
@@ -77,7 +75,7 @@
Validate a job stage.
"""
- stages = ['initialize', 'checkout', 'make_srpm', 'prep', 'building', 'cleanup', 'failed', 'addtorepo', 'repodone', 'needsign', 'finished', 'killed']
+ stages = ['initialize', 'checkout', 'make_srpm', 'prep', 'building', 'failed', 'addtorepo', 'repodone', 'needsign', 'finished', 'killed']
if stage in stages:
return True
return False
@@ -99,7 +97,6 @@
self.name = None
self.target = repo.target()
self.repo = repo
- self.failed = False
self.no_cvs = config_opts['use_srpm_not_cvs']
self.cvs_tag = cvs_tag
self.stage_dir = None
@@ -252,13 +249,8 @@
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not check out %s from %s - output was:\n %s" % (self.cvs_tag, self.target, o)
- self.email_result(resultstring=msg, subject=subj)
- self._set_cur_stage('finished')
- self.failed = True
- shutil.rmtree(self.checkout_tmpdir, True)
- return
+ raise PrepError(msg)
# Just in case the 'common' directory didn't come along for the ride,
# get it from CVS
@@ -270,26 +262,16 @@
s, o = commands.getstatusoutput(cmd)
os.chdir(self.checkout_tmpdir)
if s != 0:
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not check out common directory - output was:\n %s" % (self.cvs_tag, self.target, o)
- self.email_result(resultstring=msg, subject=subj)
- self._set_cur_stage('finished')
- self.failed = True
- shutil.rmtree(self.checkout_tmpdir, True)
- return
+ raise PrepError(msg)
def _make_srpm(self):
self._set_cur_stage('make_srpm')
self.srpm_path = None
srpm_dir = os.path.join(self.checkout_tmpdir, self.package, self.target)
if not os.path.exists(srpm_dir):
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not find path %s for %s." % (srpm_dir, self.cvs_tag)
- self.email_result(resultstring=msg, subject=subj)
- self._set_cur_stage('finished')
- self.failed = True
- shutil.rmtree(self.checkout_tmpdir, True)
- return
+ raise PrepError(msg)
os.chdir(srpm_dir)
@@ -297,13 +279,8 @@
debugprint("%d: Running %s in %s" % (self.uid, cmd, srpm_dir))
s, o = commands.getstatusoutput(cmd)
if s != 0:
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not make srpm for %s - output was:\n %s" % (self.cvs_tag, o)
- self.email_result(resultstring=msg, subject=subj)
- self._set_cur_stage('finished')
- self.failed = True
- shutil.rmtree(self.checkout_tmpdir, True)
- return
+ raise PrepError(msg)
srpmpath = None
for line in o.split("\n"):
@@ -313,13 +290,9 @@
srpmpath = path.strip()
break
if not srpmpath:
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not find srpm for %s - output was:\n %s" % (self.cvs_tag, o)
- self.email_result(resultstring=msg, subject=subj)
- self._set_cur_stage('finished')
- self.failed = True
- shutil.rmtree(self.checkout_tmpdir, True)
- return
+ raise PrepError(msg)
+
self.srpm_path = srpmpath
def _prep(self):
@@ -339,12 +312,8 @@
del ts
if len(self.archjobs) == 0:
- subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
- msg = "could not find any architectures to build for %s - output was:\n" % (self.cvs_tag)
- self.email_result(resultstring=msg, subject=subj)
- self.failed = True
- self._set_cur_stage('finished')
- return
+ msg = "Package %s does not build on any architectures this build system supports.\n" % self.cvs_tag
+ raise PrepError(msg)
self.stage_dir = self._make_stage_dir(config_opts['server_work_dir'])
for arch in self.archjobs.keys():
@@ -433,79 +402,74 @@
while not self.is_done():
# Advance to next stage based on current stage
wait = False
- if self.curstage == 'initialize':
- self._checkout()
- elif self.curstage == 'checkout':
- self._make_srpm()
- elif self.curstage == 'make_srpm':
- self._prep()
- elif self.curstage == 'prep' or self.curstage == 'building':
- wait = self._monitor()
- elif self.curstage == 'finished':
- self._cleanup()
- elif self.curstage == 'cleanup':
- if self.failed:
- self._failed()
- else:
+
+ try:
+ if self.curstage == 'initialize':
+ self._checkout()
+ elif self.curstage == 'checkout':
+ self._make_srpm()
+ elif self.curstage == 'make_srpm':
+ self._prep()
+ elif self.curstage == 'prep' or self.curstage == 'building':
+ wait = self._monitor()
+ elif self.curstage == 'finished':
self._add_to_repo()
- elif self.curstage == 'addtorepo':
- wait = True
- elif self.curstage == 'repodone':
- self._succeeded()
-
- if wait:
- while not self._event.isSet():
- self._event.wait()
- self._event.clear()
+ elif self.curstage == 'addtorepo':
+ wait = True
+ elif self.curstage == 'repodone':
+ self._succeeded()
+ except PrepError, e:
+ if not self.no_cvs:
+ shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)
+ subj = 'Prep Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
+ self.email_result(resultstring=e.args, subject=subj)
+ self._failed()
+ except BuildError, e:
+ subj = 'Build Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
+ self.email_result(resultstring=e.args, subject=subj)
+ # Kill remaining jobs on other arches
+ self._archjobs_lock.acquire()
+ for job in self.archjobs.values():
+ if job:
+ job.die()
+ self._archjobs_lock.release()
+ self._failed()
+ else:
+ # Wait to be woken up when long-running operations complete
+ if wait:
+ while not self._event.isSet():
+ self._event.wait()
+ self._event.clear()
def _monitor(self):
self._set_cur_stage('building')
# Count failed and completed jobs
completed_jobs = 0
+ self._archjobs_lock.acquire()
for job in self.archjobs.values():
if not job:
continue
if job.get_status() is 'done':
if job.builder_failed():
- self.failed = True
+ self._archjobs_lock.release()
+ msg = "Job failed on arch %s\n"
+ raise BuildError(msg)
else:
completed_jobs = completed_jobs + 1
+ self._archjobs_lock.release()
- # If any jobs have failed, or if all jobs have completed successfully, advance
- if self.failed or (completed_jobs == len(self.archjobs)):
+ if completed_jobs == len(self.archjobs):
self._set_cur_stage('finished')
return False # Don't want to wait
return True
- def _cleanup(self):
- self._set_cur_stage('cleanup')
- if self.failed:
- # Kill remaining jobs on other arches
- for job in self.archjobs.values():
- if job:
- job.die()
-
def get_stage_dir(self):
return self.stage_dir
def _failed(self):
self._set_cur_stage('failed')
-
- resultstring = """
- %s (%s): %s on %s failed to complete on one or more archs.
-""" % (self.uid, self.package, self.cvs_tag, self.target)
- resultstring = resultstring + "\n"
-
- # Add each arch job and its result:
- for job in self.archjobs.values():
- if job and job.builder_status != 'killed':
- resultstring = resultstring + " " + "%s - %s: %s" % (job.arch, job.jobid, job.builder_status) + "\n"
-
- resultstring = resultstring + "\n"
- self.email_result(resultstring)
-
self.bm.notify_job_done(self)
def _add_to_repo(self):
@@ -516,8 +480,7 @@
for job in self.archjobs.values():
if not job:
continue
- file_list = job.get_files()
- for f in file_list:
+ for f in job.get_files():
if not f.endswith(".rpm"):
continue
src_file = os.path.join(self.stage_dir, job.arch, f)
@@ -526,7 +489,7 @@
dst_path = os.path.join(config_opts['repo_dir'], self.target, self.name, verrel, "SRPM")
else:
dst_path = os.path.join(config_opts['repo_dir'], self.target, self.name, verrel, job.arch)
- self.repofiles[src_file] = dst_path
+ self.repofiles[src_file] = os.path.join(dst_path, f)
self._event.clear()
Index: Repo.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Repo.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- Repo.py 6 Jul 2005 21:21:00 -0000 1.6
+++ Repo.py 7 Jul 2005 16:21:12 -0000 1.7
@@ -71,13 +71,13 @@
if not os.path.exists(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
- # Only copy SRPMs to the repo dir if there's not already one there
+ file_in_dst = os.path.join(os.path.dirname(dst), os.path.basename(src))
if src.endswith(".src.rpm"):
- file_in_dst = os.path.join(os.path.dirname(dst),os.path.basename(src))
+ # Only copy SRPMs to the repo dir if there's not already one there
if not os.path.exists(file_in_dst):
- shutil.copy(src, dst)
+ shutil.copy(src, file_in_dst)
else:
- shutil.copy(src, dst)
+ shutil.copy(src, file_in_dst)
# Notify the build job that we've copied its files to the repo
buildjob.repo_add_callback()
More information about the fedora-extras-commits
mailing list