extras-buildsys/server BuildMaster.py, 1.23, 1.24 PackageJob.py, 1.14, 1.15 UserInterface.py, 1.29, 1.30
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue Jul 26 17:47:26 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv27510/server
Modified Files:
BuildMaster.py PackageJob.py UserInterface.py
Log Message:
2005-07-26 Dan Williams <dcbw at redhat.com>
- Throttle CVS checkouts to 5 jobs at a time
- Validate package and cvs_tag inputs to enqueue functions since they
get passed directly to commands.getstatusoutput()
Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -r1.23 -r1.24
--- BuildMaster.py 25 Jul 2005 21:44:52 -0000 1.23
+++ BuildMaster.py 26 Jul 2005 17:47:24 -0000 1.24
@@ -97,6 +97,9 @@
class BuildMaster(threading.Thread):
+
+ MAX_CHECKOUT_JOBS = 5
+
def __init__(self, hostname, builder_manager):
self.builder_manager = builder_manager
self.hostname = hostname
@@ -113,6 +116,9 @@
self._new_queue_lock = threading.Lock()
self._restart_queue = []
self._restart_queue_lock = threading.Lock()
+ self._checkout_wait_queue = []
+ self._checkout_num = 0
+ self._checkout_wait_queue_lock = threading.Lock()
self._status_updates = {}
self._status_updates_lock = threading.Lock()
@@ -139,7 +145,7 @@
def _requeue_interrupted_jobs(self):
""" Restart interrupted jobs from our db. """
- self.curs.execute('SELECT uid FROM jobs WHERE (status!="needsign" AND status!="failed" AND status!="killed" AND status!="initialize") ORDER BY uid')
+ self.curs.execute('SELECT uid FROM jobs WHERE (status!="needsign" AND status!="failed" AND status!="killed") ORDER BY uid')
self.dbcx.commit()
uids = self.curs.fetchall()
@@ -239,6 +245,28 @@
self._archjob_status_updates[lcl_uid] = lcl_attrdict
self._archjob_status_updates_lock.release()
+ def queue_checkout_wait(self, job):
+ self._checkout_wait_queue_lock.acquire()
+ self._checkout_wait_queue.append(job)
+ self._checkout_wait_queue_lock.release()
+
+ def notify_checkout_done(self, job):
+ self._checkout_wait_queue_lock.acquire()
+ self._checkout_num = self._checkout_num - 1
+ self._checkout_wait_queue_lock.release()
+
+ def _process_checkout_wait_jobs(self):
+ self._checkout_wait_queue_lock.acquire()
+
+ # We allow only 5 jobs at a time in checkout stage
+ allowed_jobs = min(self.MAX_CHECKOUT_JOBS - self._checkout_num, len(self._checkout_wait_queue))
+ for i in range(allowed_jobs):
+ job = self._checkout_wait_queue[i]
+ self._checkout_num = self._checkout_num + 1
+ job.checkout_wait_done_callback()
+ self._checkout_wait_queue = self._checkout_wait_queue[allowed_jobs:]
+ self._checkout_wait_queue_lock.release()
+
def notify_job_done(self, job):
self._done_queue_lock.acquire()
self._done_queue.append(job)
@@ -374,38 +402,41 @@
def _have_work(self):
have_work = False
- self._new_queue_lock.acquire()
- if len(self._new_queue) > 0:
- have_work = True
- self._new_queue_lock.release()
- if have_work:
- return True
- self._restart_queue_lock.acquire()
- if len(self._restart_queue) > 0:
- have_work = True
- self._restart_queue_lock.release()
- if have_work:
- return True
+ if not have_work:
+ self._new_queue_lock.acquire()
+ if len(self._new_queue) > 0:
+ have_work = True
+ self._new_queue_lock.release()
+
+ if not have_work:
+ self._restart_queue_lock.acquire()
+ if len(self._restart_queue) > 0:
+ have_work = True
+ self._restart_queue_lock.release()
+
+ if not have_work:
+ self._checkout_wait_queue_lock.acquire()
+ if (self._checkout_num < self.MAX_CHECKOUT_JOBS) and len(self._checkout_wait_queue) > 0:
+ have_work = True
+ self._checkout_wait_queue_lock.release()
- self._status_updates_lock.acquire()
- if len(self._status_updates) > 0:
- have_work = True
- self._status_updates_lock.release()
- if have_work:
- return True
+ if not have_work:
+ self._status_updates_lock.acquire()
+ if len(self._status_updates) > 0:
+ have_work = True
+ self._status_updates_lock.release()
- self._archjob_status_updates_lock.acquire()
- if len(self._archjob_status_updates) > 0:
- have_work = True
- self._archjob_status_updates_lock.release()
- if have_work:
- return True
+ if not have_work:
+ self._archjob_status_updates_lock.acquire()
+ if len(self._archjob_status_updates) > 0:
+ have_work = True
+ self._archjob_status_updates_lock.release()
- if self.builder_manager.have_work():
- return True
+ if not have_work and self.builder_manager.have_work():
+ have_work = True
- return False
+ return have_work
def get_job(self, uid):
self._building_jobs_lock.acquire()
@@ -427,6 +458,9 @@
# Clean up jobs that have finished
self._process_finished_jobs()
+ # Let a few jobs through the checkout_wait gate if needed
+ self._process_checkout_wait_jobs()
+
# Start any new jobs
self._start_new_jobs()
self._start_requeued_jobs()
Index: PackageJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/PackageJob.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -r1.14 -r1.15
--- PackageJob.py 25 Jul 2005 19:47:15 -0000 1.14
+++ PackageJob.py 26 Jul 2005 17:47:24 -0000 1.15
@@ -91,7 +91,7 @@
Validate a job stage.
"""
- stages = ['initialize', 'checkout', 'make_srpm', 'prep', 'waiting', 'building', 'finished', 'addtorepo', 'repodone', 'needsign', 'failed', 'killed']
+ stages = ['initialize', 'checkout_wait', 'checkout_wait_done', 'checkout', 'make_srpm', 'prep', 'waiting', 'building', 'finished', 'addtorepo', 'repodone', 'needsign', 'failed', 'killed']
if stage in stages:
return True
return False
@@ -116,7 +116,6 @@
self.epoch = None
self.ver = None
self.release = None
- pjc = PackageJobController(self, 'initialize', 'waiting')
self.hostname = hostname
self.username = username
@@ -129,13 +128,15 @@
self.stage_dir = None
self.srpm_path = None
self.srpm_http_path = None
- # Deal with straight SRPM builds
- if self.no_cvs and self.curstage is 'initialize':
- self._set_cur_stage('make_srpm')
self.repofiles = {}
self.archjobs = {}
self._archjobs_lock = threading.Lock()
self._event = threading.Event()
+
+ first_stage = 'initialize'
+ if self.no_cvs == True:
+ first_stage = 'make_srpm'
+ pjc = PackageJobController(self, first_stage, 'waiting')
pjc.start()
def get_cur_stage(self):
@@ -259,9 +260,13 @@
os.makedirs(stage_dir)
return stage_dir
-
+ def checkout_wait_done_callback(self):
+ self._set_cur_stage('checkout_wait_done')
+ self.wake()
+
def _checkout(self):
self._set_cur_stage('checkout')
+ err_msg = None
# Create the temporary checkout directory
dirname = "%s-%s-%d" % (self.uid, self.cvs_tag, time.time())
@@ -275,19 +280,23 @@
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
- msg = "could not check out %s from %s - output was:\n %s" % (self.cvs_tag, self.target, o)
- raise PrepError(msg)
-
- # Just in case the 'common' directory didn't come along for the ride,
- # get it from CVS
- pkg_path = os.path.join(self.checkout_tmpdir, self.package)
- if not os.path.exists(os.path.join(pkg_path, "common")):
- cmd = 'cd %s; %s co common' % (pkg_path, config_opts['cvs_cmd'])
- debugprint("%d: Running %s" % (self.uid, cmd))
- s, o = commands.getstatusoutput(cmd)
- if s != 0:
- msg = "could not check out common directory - output was:\n %s" % (self.cvs_tag, self.target, o)
- raise PrepError(msg)
+ err_msg = "could not check out %s from %s - output was:\n %s" % (self.cvs_tag, self.target, o)
+ else:
+ # Just in case the 'common' directory didn't come along for the ride,
+ # get it from CVS
+ pkg_path = os.path.join(self.checkout_tmpdir, self.package)
+ if not os.path.exists(os.path.join(pkg_path, "common")):
+ cmd = 'cd %s; %s co common' % (pkg_path, config_opts['cvs_cmd'])
+ debugprint("%d: Running %s" % (self.uid, cmd))
+ s, o = commands.getstatusoutput(cmd)
+ if s != 0:
+ err_msg = "could not check out common directory - output was:\n %s" % (self.cvs_tag, self.target, o)
+
+ self.bm.notify_checkout_done(self)
+
+ if err_msg:
+ raise PrepError(err_msg)
+
def _make_srpm(self):
self._set_cur_stage('make_srpm')
@@ -450,6 +459,11 @@
try:
wait = False
if self.curstage == 'initialize':
+ self.bm.queue_checkout_wait(self)
+ self._set_cur_stage('checkout_wait')
+ elif self.curstage == 'checkout_wait':
+ wait = True
+ elif self.curstage == 'checkout_wait_done':
self._checkout()
elif self.curstage == 'checkout':
self._make_srpm()
Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -r1.29 -r1.30
--- UserInterface.py 25 Jul 2005 21:44:52 -0000 1.29
+++ UserInterface.py 26 Jul 2005 17:47:24 -0000 1.30
@@ -39,12 +39,10 @@
def validate_email(email):
+ safe_list = ['@', '_', '-', '.', '+']
for c in email:
# For now, legal characters are '@_-.+' plus alphanumeric
- if (c == '@') or (c == '_') or (c == '-') or (c == '.') or (c == '+') or c.isalnum():
- pass
- else:
- print "Bad char is '%s'" % c
+ if not (c in safe_list) and not c.isalnum():
return False
return True
@@ -59,6 +57,20 @@
return None
return uid
+def validate_package_name(name):
+ safe_list = ['_', '-', '+']
+ for c in name:
+ if not (c in safe_list) and not c.isalnum():
+ return False
+ return True
+
+def validate_cvs_tag(tag):
+ safe_list = ['-', '_', '.', ':', '~', '[', ']']
+ for c in tag:
+ if not (c in safe_list) and not c.isalnum():
+ return False
+ return True
+
class InvalidTargetError(exceptions.Exception): pass
@@ -111,6 +123,16 @@
"%s: this server builds SRPMs, not CVS checkouts." % (cvs_tag, target))
return (-1, "This build server is set up for building SRPMS only. Use the 'enqueue_srpm' command instead.")
+ if not validate_package_name(package):
+ email_result(email, cvs_tag, "Error setting up build for %s on "\
+ "%s: Package name '%s' contained an illegal character. Submit a bug report?" % (cvs_tag, target, package))
+ return (-1, "The package name contained an illegal character.")
+
+ if not validate_cvs_tag(cvs_tag):
+ email_result(email, cvs_tag, "Error setting up build for %s on "\
+ "%s: The CVS tag '%s' contained an illegal character. Submit a bug report?" % (package, target, cvs_tag))
+ return (-1, "The CVS tag contained an illegal character.")
+
try:
real_target = resolve_target(target)
except InvalidTargetError:
@@ -130,6 +152,11 @@
"%s: this server builds CVS checkouts, not SRPMS." % (srpm_file, target))
return (-1, "This build server is set up for building from CVS. Use the 'enqueue' command instead.")
+ if not validate_package_name(package):
+ email_result(email, srpm_file, "Error setting up build for %s on "\
+ "%s: Package name '%s' contained an illegal character. Submit a bug report?" % (package, target, package))
+ return (-1, "The package name contained an illegal character.")
+
# We limit the database field to 255 chars
if len(srpm_file) > 255:
email_result(email, srpm_file, "Error setting up build for %s on "\
More information about the fedora-extras-commits
mailing list