extras-buildsys/server BuildMaster.py, 1.25, 1.26 PackageJob.py, 1.15, 1.16 User.py, 1.5, 1.6 UserInterface.py, 1.37, 1.38
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue Aug 2 04:10:34 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv27005/server
Modified Files:
BuildMaster.py PackageJob.py User.py UserInterface.py
Log Message:
2005-08-01 Dan Williams <dcbw at redhat.com>
* Implement 'finished' state, allow users to move failed/needsign jobs
to the 'finished' state
* Random cleanups of database access code to make it more readable
* Simplify user permissions
* server/PackageJob.py
- Each state/stage now has a private method called "_stage_<the-stage>"
that gets called when the job is in that stage. This simplifies
the job's process routine quite a bit
- Jobs now record a 'result', either 'success', 'failed', or 'killed'
Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -r1.25 -r1.26
--- BuildMaster.py 29 Jul 2005 06:05:37 -0000 1.25
+++ BuildMaster.py 2 Aug 2005 04:10:32 -0000 1.26
@@ -50,6 +50,7 @@
'starttime BIGINT, ' \
'endtime BIGINT, ' \
'status VARCHAR(15), ' \
+ 'result VARCHAR(15), ' \
'epoch VARCHAR(4), ' \
'version VARCHAR(25), ' \
'release VARCHAR(25), ' \
@@ -146,7 +147,7 @@
def _requeue_interrupted_jobs(self):
""" Restart interrupted jobs from our db. """
- self.curs.execute('SELECT uid FROM jobs WHERE (status!="needsign" AND status!="failed" AND status!="killed") ORDER BY uid')
+ self.curs.execute('SELECT uid FROM jobs WHERE (status!="needsign" AND status!="failed" AND status!="finished") ORDER BY uid')
self.dbcx.commit()
uids = self.curs.fetchall()
@@ -154,7 +155,7 @@
return
for item in uids:
- self.requeue_job(item[0])
+ self.requeue_job(item['uid'])
def requeue_job(self, uid):
self._restart_queue_lock.acquire()
@@ -183,19 +184,37 @@
return
for row in jobs:
- uid = row[0]
+ uid = row['uid']
# Kill any archjobs that are left around
self.curs.execute('DELETE FROM archjobs WHERE parent_uid=%d' % uid)
self.dbcx.commit()
+ curs.execute('CREATE TABLE jobs (' \
+ 'uid INTEGER PRIMARY KEY, ' \
+ 'username VARCHAR(20), ' \
+ 'package VARCHAR(50), ' \
+ 'cvs_tag VARCHAR(255), ' \
+ 'target VARCHAR(20), ' \
+ 'buildreq VARCHAR(75), ' \
+ 'starttime BIGINT, ' \
+ 'endtime BIGINT, ' \
+ 'status VARCHAR(15), ' \
+ 'result VARCHAR(15), ' \
+ 'epoch VARCHAR(4), ' \
+ 'version VARCHAR(25), ' \
+ 'release VARCHAR(25), ' \
+ 'archlist VARCHAR(75), ' \
+ 'result_msg TEXT' \
+ ')')
+
# Now requeue the job
try:
- repo = self.repos[row[4]]
+ repo = self.repos[row['target']]
except KeyError:
- print "%s (%s): Target '%s' not found." % (uid, row[2], row[4])
+ print "%s (%s): Target '%s' not found." % (uid, row['package'], row['target'])
else:
- job = PackageJob.PackageJob(uid, row[1], row[2], row[3], repo, self, self.hostname)
- print "%s (%s): Restarting '%s' on target '%s'" % (uid, row[2], row[3], row[4])
+ job = PackageJob.PackageJob(uid, row['username'], row['package'], row['cvs_tag'], repo, self, self.hostname)
+ print "%s (%s): Restarting '%s' on target '%s'" % (uid, row['package'], row['cvs_tag'], row['target'])
self._building_jobs_lock.acquire()
self._building_jobs[uid] = job
self._building_jobs_lock.release()
@@ -311,6 +330,8 @@
if attrdict.has_key('result_msg'):
import urllib
sql = sql + ', result_msg="%s"' % (urllib.quote(attrdict['result_msg']))
+ if attrdict.has_key('result'):
+ sql = sql + ', result="%s"' % attrdict['result']
sql = 'UPDATE jobs SET ' + sql + ' WHERE uid=%d' % uid
try:
@@ -368,8 +389,8 @@
continue
self.curs.execute('INSERT INTO jobs (uid, username, package,' \
- ' cvs_tag, target, buildreq, starttime, endtime, status)' \
- ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, 0, "%s")' \
+ ' cvs_tag, target, buildreq, starttime, endtime, status, result)' \
+ ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, 0, "%s", "")' \
% (item['email'], item['package'], locator, item['target'], \
item['buildreq'], item['time'], 'initialize'))
self.dbcx.commit()
Index: PackageJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/PackageJob.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -r1.15 -r1.16
--- PackageJob.py 26 Jul 2005 17:47:24 -0000 1.15
+++ PackageJob.py 2 Aug 2005 04:10:32 -0000 1.16
@@ -87,15 +87,19 @@
def is_package_job_stage_valid(stage):
- """
- Validate a job stage.
- """
-
- stages = ['initialize', 'checkout_wait', 'checkout_wait_done', 'checkout', 'make_srpm', 'prep', 'waiting', 'building', 'finished', 'addtorepo', 'repodone', 'needsign', 'failed', 'killed']
+ """ Validate a job stage """
+ stages = ['initialize', 'checkout_wait', 'checkout_wait_done', 'checkout', 'make_srpm', 'prep', 'waiting', 'building', 'build_done', 'add_to_repo', 'repodone', 'needsign', 'failed', 'finished']
if stage in stages:
return True
return False
+def is_package_job_result_valid(result):
+ """ Validate a job result """
+ results = ['failed', 'success', 'killed', 'in-progress']
+ if result in results:
+ return True
+ return False
+
def make_job_log_url(target, uid, name, ver, release):
if target and uid and name and ver and release:
return "%s/%s/%s-%s-%s-%s/" % (config_opts['log_url'], target, uid, name, ver, release)
@@ -109,6 +113,7 @@
def __init__(self, uid, username, package, cvs_tag, repo, buildmaster, hostname):
self.curstage = ''
+ self.result = 'in-progress'
self.bm = buildmaster
self.uid = uid
self.package = package
@@ -135,7 +140,7 @@
first_stage = 'initialize'
if self.no_cvs == True:
- first_stage = 'make_srpm'
+ first_stage = 'prep'
pjc = PackageJobController(self, first_stage, 'waiting')
pjc.start()
@@ -151,6 +156,8 @@
if oldstage != stage:
attrdict = {}
attrdict['status'] = copy.copy(stage)
+ if self.result:
+ attrdict['result'] = copy.copy(self.result)
if self.name and self.epoch and self.ver and self.release:
attrdict['epoch'] = self.epoch
attrdict['version'] = self.ver
@@ -260,12 +267,23 @@
os.makedirs(stage_dir)
return stage_dir
+ def _stage_initialize(self):
+ self._set_cur_stage('checkout_wait')
+ self.bm.queue_checkout_wait(self)
+ return True
+
+ def _stage_checkout_wait(self):
+ return True
+
def checkout_wait_done_callback(self):
self._set_cur_stage('checkout_wait_done')
self.wake()
- def _checkout(self):
+ def _stage_checkout_wait_done(self):
self._set_cur_stage('checkout')
+ return False
+
+ def _stage_checkout(self):
err_msg = None
# Create the temporary checkout directory
@@ -297,10 +315,10 @@
if err_msg:
raise PrepError(err_msg)
-
- def _make_srpm(self):
self._set_cur_stage('make_srpm')
+ return False
+ def _stage_make_srpm(self):
# Map our target (self.target) to the CVS target alias, since CVS may have
# different target names that we expose
cvs_target_map = config_opts['cvs_target_map']
@@ -335,8 +353,10 @@
self.srpm_path = srpmpath
- def _prep(self):
self._set_cur_stage('prep')
+ return False
+
+ def _stage_prep(self):
# In SRPM-only mode, cvs_tag is path to the SRPM to build
if self.no_cvs:
@@ -385,6 +405,8 @@
shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)
self._request_arch_jobs()
+ self._set_cur_stage('waiting')
+ return False
def _request_one_arch_job(self, arch, orphaned):
# Construct SPRM URL
@@ -430,7 +452,7 @@
self._archjobs_lock.release()
def is_done(self):
- if self.curstage == 'needsign' or self.curstage == 'failed' or self.curstage == 'killed':
+ if self.curstage == 'needsign' or self.curstage == 'failed' or self.curstage == 'finished':
return True
return False
@@ -438,7 +460,8 @@
# Kill any building jobs
resultstring = "%s (%s): Build on target %s was killed by %s." % (self.uid, self.name, self.target, username)
- self._set_cur_stage('killed', resultstring)
+ self.result = 'killed'
+ self._set_cur_stage('finished', resultstring)
self.email_result(self.username, resultstring)
self._archjobs_lock.acquire()
@@ -455,36 +478,22 @@
self._event.set()
def process(self):
- # Advance to next stage based on current stage
+ if self.is_done():
+ return
+
try:
- wait = False
- if self.curstage == 'initialize':
- self.bm.queue_checkout_wait(self)
- self._set_cur_stage('checkout_wait')
- elif self.curstage == 'checkout_wait':
- wait = True
- elif self.curstage == 'checkout_wait_done':
- self._checkout()
- elif self.curstage == 'checkout':
- self._make_srpm()
- elif self.curstage == 'make_srpm':
- self._prep()
- self._set_cur_stage('waiting')
- # When prep is done, the first thread ends
- elif self.curstage == 'building':
- wait = self._monitor()
- elif self.curstage == 'finished':
- self._add_to_repo()
- elif self.curstage == 'addtorepo':
- wait = True
- elif self.curstage == 'repodone':
- self._succeeded()
+ func = getattr(self, "_stage_%s" % self.curstage)
+ if func():
+ # Wait to be woken up when long-running operations complete
+ while not self._event.isSet():
+ self._event.wait()
+ self._event.clear()
except PrepError, e:
if not self.no_cvs:
shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)
subj = 'Prep Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
self.email_result(self.username, resultstring=e.args, subject=subj)
- self._failed(e.args)
+ self._stage_failed(e.args)
except BuildError, e:
subj = 'Build Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
log_url = make_job_log_url(self.target, self.uid, self.name, self.ver, self.release)
@@ -498,17 +507,9 @@
if job:
job.die()
self._archjobs_lock.release()
- self._failed(e.msg)
- else:
- # Wait to be woken up when long-running operations complete
- if wait:
- while not self._event.isSet():
- self._event.wait()
- self._event.clear()
-
- def _monitor(self):
- self._set_cur_stage('building')
+ self._stage_failed(e.msg)
+ def _stage_building(self):
# Count failed and completed jobs
completed_jobs = 0
self._archjobs_lock.acquire()
@@ -525,7 +526,7 @@
self._archjobs_lock.release()
if completed_jobs == len(self.archjobs):
- self._set_cur_stage('finished')
+ self._set_cur_stage('add_to_repo')
return False # Don't want to wait
return True
@@ -533,14 +534,13 @@
def get_stage_dir(self):
return self.stage_dir
- def _failed(self, msg=None):
+ def _stage_failed(self, msg=None):
+ self.result = 'failed'
self._set_cur_stage('failed', msg)
self.endtime = time.time()
self.bm.notify_job_done(self)
- def _add_to_repo(self):
- self._set_cur_stage('addtorepo')
-
+ def _stage_add_to_repo(self):
# Create a list of files that the repo should copy to
# the repo dir
for job in self.archjobs.values():
@@ -566,13 +566,15 @@
self.repo.request_copy(self)
self.endtime = time.time()
+ return True
def repo_add_callback(self):
self._set_cur_stage('repodone')
self.wake()
- def _succeeded(self):
+ def _stage_repodone(self):
resultstring = " %s (%s): Build on target %s succeeded." % (self.uid, self.name, self.target)
+ self.result = 'success'
self._set_cur_stage('needsign', resultstring)
log_url = make_job_log_url(self.target, self.uid, self.name, self.ver, self.release)
Index: User.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/User.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- User.py 22 Jul 2005 21:35:27 -0000 1.5
+++ User.py 2 Aug 2005 04:10:32 -0000 1.6
@@ -29,8 +29,8 @@
self.email = email
self.guest = guest
self.own_jobs = False
- self.kill_any_job = False
- self.modify_users = False
+ self.job_admin = False
+ self.user_admin = False
self.server_admin = False
@@ -63,9 +63,12 @@
create = True
if create:
- curs.execute('CREATE TABLE users (email VARCHAR(50), ' \
- 'own_jobs BOOLEAN, kill_any_job BOOLEAN, ' \
- 'modify_users BOOLEAN, server_admin BOOLEAN)')
+ curs.execute('CREATE TABLE users (' \
+ 'email VARCHAR(50), ' \
+ 'own_jobs BOOLEAN, ' \
+ 'job_admin BOOLEAN, ' \
+ 'user_admin BOOLEAN, ' \
+ 'server_admin BOOLEAN)')
dbcx.commit()
@@ -75,15 +78,14 @@
(dbcx, curs) = get_userdb_dbcx()
user = None
- curs.execute('SELECT email, own_jobs, kill_any_job, modify_users, ' \
- 'server_admin FROM users WHERE email="%s"' % email)
+ curs.execute('SELECT * FROM users WHERE email="%s"' % email)
dbcx.commit()
item = curs.fetchone()
if item:
user = User(email, False)
user.own_jobs = item['own_jobs']
- user.kill_any_job = item['kill_any_job']
- user.modify_users = item['modify_users']
+ user.job_admin = item['job_admin']
+ user.user_admin = item['user_admin']
user.server_admin = item['server_admin']
else:
if config_opts['guest_allowed']:
Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.37
retrieving revision 1.38
diff -u -r1.37 -r1.38
--- UserInterface.py 30 Jul 2005 15:05:19 -0000 1.37
+++ UserInterface.py 2 Aug 2005 04:10:32 -0000 1.38
@@ -183,7 +183,7 @@
if not uid:
return (-1, "Error: Invalid job UID.")
- sql = 'SELECT uid, username, status FROM jobs WHERE uid=%d' % uid
+ sql = 'SELECT uid, username, status, result FROM jobs WHERE uid=%d' % uid
# Run the query for the job
try:
@@ -196,7 +196,7 @@
return (-1, "Error: Invalid job UID.")
# Ensure the job failed or was killed
- if job[2] != 'failed' and job[2] != 'killed':
+ if job['result'] != 'failed' and job['result'] != 'killed':
return (-1, "Error: Job %d must be either 'failed' or 'killed' to requeue." % uid)
self._bm.requeue_job(uid)
@@ -214,7 +214,7 @@
def list_jobs(self, args_dict):
""" Query job information and return it to the user """
- sql = 'SELECT uid, username, package, cvs_tag, target, starttime, endtime, status FROM jobs WHERE '
+ sql = 'SELECT uid, username, package, cvs_tag, target, starttime, endtime, status, result FROM jobs WHERE '
sql_args = []
if args_dict.has_key('email') and args_dict['email']:
@@ -230,6 +230,13 @@
else:
return (-1, "Error: Invalid job status.", [])
+ if args_dict.has_key('result') and args_dict['result']:
+ result = args_dict['result']
+ if PackageJob.is_package_job_result_valid(result):
+ sql_args.append('result="%s"' % result)
+ else:
+ return (-1, "Error: Invalid job result.", [])
+
if args_dict.has_key('uid') and args_dict['uid']:
uid = validate_uid(args_dict['uid'])
if uid == None:
@@ -282,14 +289,15 @@
jobs = []
for row in data:
jobrec = {}
- jobrec['uid'] = row[0]
- jobrec['username'] = row[1]
- jobrec['package'] = row[2]
- jobrec['source'] = row[3]
- jobrec['target'] = row[4]
- jobrec['starttime'] = row[5]
- jobrec['endtime'] = row[6]
- jobrec['status'] = row[7]
+ jobrec['uid'] = row['uid']
+ jobrec['username'] = row['username']
+ jobrec['package'] = row['package']
+ jobrec['source'] = row['cvs_tag']
+ jobrec['target'] = row['target']
+ jobrec['starttime'] = row['starttime']
+ jobrec['endtime'] = row['endtime']
+ jobrec['status'] = row['status']
+ jobrec['result'] = row['result']
jobrec['archjobs'] = []
jobs.append(copy.deepcopy(jobrec))
@@ -308,14 +316,14 @@
data = curs.fetchall()
for row in data:
ajrec = {}
- ajrec['jobid'] = row[0]
- ajrec['parent_uid'] = row[1]
- ajrec['starttime'] = row[2]
- ajrec['endtime'] = row[3]
- ajrec['arch'] = row[4]
- ajrec['builder_addr'] = row[5]
- ajrec['status'] = row[6]
- ajrec['builder_status'] = row[7]
+ ajrec['jobid'] = row['jobid']
+ ajrec['parent_uid'] = row['parent_uid']
+ ajrec['starttime'] = row['starttime']
+ ajrec['endtime'] = row['endtime']
+ ajrec['arch'] = row['arch']
+ ajrec['builder_addr'] = row['builder_addr']
+ ajrec['status'] = row['status']
+ ajrec['builder_status'] = row['builder_status']
for job in jobs:
if job['uid'] == ajrec['parent_uid']:
job['archjobs'].append(copy.deepcopy(ajrec))
@@ -346,23 +354,23 @@
if not job:
return (-1, "Error: Invalid job UID.", {})
jobrec = {}
- jobrec['uid'] = job[0]
- jobrec['username'] = job[1]
- jobrec['package'] = job[2]
- jobrec['source'] = job[3]
- jobrec['target'] = job[4]
- jobrec['starttime'] = job[5]
- jobrec['endtime'] = job[6]
- jobrec['status'] = job[7]
- if job[8] and job[9] and job[10]:
- jobrec['epoch'] = job[8]
- jobrec['version'] = job[9]
- jobrec['release'] = job[10]
+ jobrec['uid'] = job['uid']
+ jobrec['username'] = job['username']
+ jobrec['package'] = job['package']
+ jobrec['source'] = job['cvs_tag']
+ jobrec['target'] = job['target']
+ jobrec['starttime'] = job['starttime']
+ jobrec['endtime'] = job['endtime']
+ jobrec['status'] = job['status']
+ if job['epoch'] and job['version'] and job['release']:
+ jobrec['epoch'] = job['epoch']
+ jobrec['version'] = job['version']
+ jobrec['release'] = job['release']
log_url = PackageJob.make_job_log_url(jobrec['target'], str(uid), jobrec['package'], jobrec['version'], jobrec['release'])
if log_url and len(log_url):
jobrec['log_url'] = log_url
- if job[12]:
- jobrec['result_msg'] = job[12]
+ if job['result_msg']:
+ jobrec['result_msg'] = job['result_msg']
jobrec['archjobs'] = []
# Get all archjobs for this job
@@ -372,14 +380,14 @@
data = curs.fetchall()
for row in data:
ajrec = {}
- ajrec['jobid'] = row[0]
- ajrec['parent_uid'] = row[1]
- ajrec['starttime'] = row[2]
- ajrec['endtime'] = row[3]
- ajrec['arch'] = row[4]
- ajrec['builder_addr'] = row[5]
- ajrec['status'] = row[6]
- ajrec['builder_status'] = row[7]
+ ajrec['jobid'] = row['jobid']
+ ajrec['parent_uid'] = row['parent_uid']
+ ajrec['starttime'] = row['starttime']
+ ajrec['endtime'] = row['endtime']
+ ajrec['arch'] = row['arch']
+ ajrec['builder_addr'] = row['builder_addr']
+ ajrec['status'] = row['status']
+ ajrec['builder_status'] = row['builder_status']
jobrec['archjobs'].append(ajrec)
del curs
@@ -417,6 +425,28 @@
def is_paused(self):
return self._bm.is_paused()
+ def finish(self, uid_list):
+ uids = ''
+ for uid in uid_list:
+ uid = validate_uid(uid)
+ if not uid:
+ continue
+ if len(uids) == 0:
+ uids = uids + "uid=%d" % uid
+ else:
+ uids = uids + " OR uid=%d" % uid
+
+ if len(uids):
+ sql = 'UPDATE jobs SET status="finished" WHERE %s' % uids
+
+ try:
+ dbcx, curs = get_dbcx()
+ except sqlite.DatabaseError, e:
+ return (-1, "Unable to access job database.")
+ curs.execute(sql)
+ dbcx.commit()
+ return (0, "Success.")
+
class UserInterfaceSSLAuth(UserInterface):
"""
@@ -458,7 +488,7 @@
return (-1, "Error: Invalid job UID.")
# Ensure matching usernames
- if job[1] != user.email and not user.server_admin:
+ if job['username'] != user.email and not user.job_admin:
return (-1, "Error: You are not the original submitter for Job %d." % uid)
return UserInterface.requeue(self, uid)
@@ -471,7 +501,7 @@
return (-1, "Insufficient privileges.")
jobid = int(jobid)
job = self._bm.get_job(jobid)
- if job and not user.kill_any_job and user.email != job.username:
+ if job and not user.job_admin and user.email != job.username:
return (-1, "Insufficient privileges.")
return self._kill_job(user.email, job, jobid)
@@ -510,6 +540,52 @@
return UserInterface.pause(self, paused)
+ def finish(self, uid_list):
+ user = AuthedXMLRPCServer.get_authinfo()
+ if not user or not user.own_jobs:
+ return (-1, "Insufficient privileges.")
+
+ uids = ''
+ for uid in uid_list:
+ uid = validate_uid(uid)
+ if not uid:
+ return (-1, "Error: Invalid job UID.")
+ if len(uids) == 0:
+ uids = uids + "uid=%d" % uid
+ else:
+ uids = uids + " OR uid=%d" % uid
+
+ sql = 'SELECT uid, username, status FROM jobs WHERE %s' % uids
+
+ try:
+ dbcx, curs = get_dbcx()
+ except sqlite.DatabaseError, e:
+ return (-1, "Unable to access job database.")
+ curs.execute(sql)
+ data = curs.fetchall()
+
+ # Ensure that the user can actually finish the jobs they requested
+ final_uid_list = []
+ for row in data:
+ uid = row['uid']
+ username = row['username']
+ status = row['status']
+
+ if status != 'needsign' and status != 'failed':
+ return (-1, "Error: Job %d must be either 'needsign' or 'failed' to finish it." % uid)
+
+ # Marking 'needsign' jobs as finished requires admin privs
+ if status == 'needsign' and not user.job_admin:
+ return (-1, "Insufficient privileges to finish job %d." % uid)
+ # 'failed' jobs can only be finished by the job owner or a job_admin user
+ if status == 'failed' and user != user.email and not user.job_admin:
+ return (-1, "Insufficient privileges to finish job %d." % uid)
+
+ final_uid_list.append(uid)
+
+ return UserInterface.finish(self, final_uid_list)
+
+
class UserInterfaceNoAuth(UserInterface):
"""
Allow all operations, NULL authentication
More information about the fedora-extras-commits
mailing list