extras-buildsys/server ArchJob.py,1.3,1.4 BuildMaster.py,1.10,1.11
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Fri Jul 8 17:23:47 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv3608/server
Modified Files:
ArchJob.py BuildMaster.py
Log Message:
2005-07-07 Dan Williams <dcbw at redhat.com>
* builder/builder.py
- Clean up the buildroot no matter what
* Make archjobs write their status to the database too
Index: ArchJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/ArchJob.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- ArchJob.py 7 Jul 2005 16:21:12 -0000 1.3
+++ ArchJob.py 8 Jul 2005 17:23:45 -0000 1.4
@@ -53,7 +53,7 @@
self.starttime = time.time()
def _builder_finished(self):
- if self.builder_status == 'done' or self.builder_status == 'killed' or self.builder_status == 'failed':
+ if self.builder_status == 'done' or self.builder_status == 'killed' or self.builder_status == 'failed' or self.builder_status == 'orphaned':
return True
return False
@@ -67,12 +67,44 @@
return True
return False
+ def _to_dict(self):
+ attrdict = {}
+ attrdict['jobid'] = self.jobid
+ attrdict['parent_uid'] = self.par_job.uid
+ attrdict['arch'] = self.arch
+ addr = self.builder.address()
+ # for some reason, splithost doesn't like the protocol
+ # method, you have to give it a string starting with "//"
+ if addr.startswith("http"):
+ idx = addr.find('//')
+ addr = addr[idx:]
+ host_port, path = urllib.splithost(addr)
+ host, port = urllib.splitport(host_port)
+ attrdict['builder_addr'] = host
+ attrdict['status'] = self.status
+ attrdict['builder_status'] = self.builder_status
+ return attrdict
+
def set_builder_status(self, status):
if status != 'idle':
- self.builder_status = status
+ oldstatus = self.builder_status
+ self.builder_status = status
+ if oldstatus != self.builder_status:
+ attrdict = self._to_dict()
+ self.par_job.bm.queue_archjob_status_update(self.jobid, attrdict)
+ del attrdict
+
if status == 'killed' or status == 'failed':
self.par_job.wake()
+ def _set_status(self, status):
+ oldstatus = self.status
+ self.status = status
+ if oldstatus != self.status:
+ attrdict = self._to_dict()
+ self.par_job.bm.queue_archjob_status_update(self.jobid, attrdict)
+ del attrdict
+
def _send_repo_unlocked(self):
try:
self._server.repo_unlocked(self.jobid)
@@ -108,7 +140,7 @@
# if the builder is done, grab list of files to download
if self._builder_finished():
- self.status = 'downloading'
+ self._set_status('downloading')
for f in self._dl_files():
uf = urllib.unquote(f)
self.downloads[uf] = 0
@@ -153,7 +185,7 @@
# All done downloading?
if not undownloaded:
self._print_downloaded_files()
- self.status = 'done'
+ self._set_status('done')
self.par_job.wake()
def _print_downloaded_files(self):
@@ -196,11 +228,13 @@
def builder_gone(self):
if self.status != 'done':
- self._par_job.remove_arch_job(self)
+ self.builder_status = 'orphaned'
+ self._set_status('done')
+ self.par_job.remove_arch_job(self)
def die(self):
if self.status == 'initialize' or self.status == 'running':
self._server.die(self.jobid)
- self.status = 'done'
+ self._set_status('done')
Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -r1.10 -r1.11
--- BuildMaster.py 5 Jul 2005 21:08:03 -0000 1.10
+++ BuildMaster.py 8 Jul 2005 17:23:45 -0000 1.11
@@ -68,6 +68,28 @@
else:
dbcx.commit()
+ # Create the archjobs table
+ try:
+ curs.execute('SELECT * FROM archjobs')
+ dbcx.commit()
+ except Exception, e:
+ # If DB wasn't created, try to create it
+ try:
+ curs.execute('CREATE TABLE archjobs (jobid VARCHAR(40) PRIMARY KEY, ' \
+ 'parent_uid INTEGER, starttime BIGINT, endtime BIGINT, ' \
+ 'arch VARCHAR(15), builder_addr VARCHAR(100), ' \
+ 'status VARCHAR(15), builder_status VARCHAR(15))')
+ except sqlite.OperationalError, e:
+ print "Could not access the job database. Reason: '%s'. Exiting..." % e
+ os._exit(1)
+ except sqlite.DatabaseError, e:
+ s = "%s" % e
+ if s != "table archjobs already exists":
+ print "Could not access the job database. Reason: '%s'. Exiting..." % e
+ os._exit(1)
+ else:
+ dbcx.commit()
+
class BuildMaster(threading.Thread):
def __init__(self, hostname, builder_manager):
@@ -86,6 +108,8 @@
self._new_queue_lock = threading.Lock()
self._status_updates = {}
self._status_updates_lock = threading.Lock()
+ self._archjob_status_updates = {}
+ self._archjob_status_updates_lock = threading.Lock()
self._query_queue = []
self._query_queue_lock = threading.Lock()
@@ -147,6 +171,13 @@
self._status_updates[lcl_uid] = lcl_status
self._status_updates_lock.release()
+ def queue_archjob_status_update(self, jobid, attrdict):
+ self._archjob_status_updates_lock.acquire()
+ lcl_jobid = copy.copy(jobid)
+ lcl_attrdict = copy.deepcopy(attrdict)
+ self._archjob_status_updates[lcl_jobid] = lcl_attrdict
+ self._archjob_status_updates_lock.release()
+
def notify_job_done(self, job):
self._done_queue_lock.acquire()
self._done_queue.append(job)
@@ -157,7 +188,7 @@
for job in self._done_queue:
curstage = job.get_cur_stage()
- self._write_status_to_db(job.get_uid(), curstage)
+ self._write_job_status_to_db(job.get_uid(), curstage)
print "%s (%s): Job finished." % (job.get_uid(), job.package)
self._building_jobs_lock.acquire()
self._building_jobs.remove(job)
@@ -166,7 +197,7 @@
self._done_queue = []
self._done_queue_lock.release()
- def _write_status_to_db(self, uid, status):
+ def _write_job_status_to_db(self, uid, status):
try:
self.curs.execute('UPDATE jobs SET status="%s" WHERE uid=%d' \
% (status, uid))
@@ -175,16 +206,41 @@
self.dbcx.commit()
+ def _write_archjob_status_to_db(self, jobid, attrdict):
+ self.curs.execute('SELECT * FROM archjobs WHERE jobid="%s"' % jobid)
+ self.dbcx.commit()
+ if len(self.curs.fetchall()) == 0:
+ try:
+ self.curs.execute('INSERT INTO archjobs (jobid, parent_uid, starttime, arch, builder_addr, status, builder_status) ' \
+ 'VALUES ("%s", %d, %d, "%s", "%s", "%s", "%s")' % (jobid, attrdict['parent_uid'], time.time(), attrdict['arch'], \
+ attrdict['builder_addr'], attrdict['status'], attrdict['builder_status']))
+ except sqlite.OperationalError, e:
+ print "DB Error: could not access jobs database. Reason: '%s'" % e
+ else:
+ try:
+ self.curs.execute('UPDATE archjobs SET status="%s", builder_status="%s" ' \
+ 'WHERE jobid="%s" AND parent_uid=%d' % (attrdict['status'], attrdict['builder_status'], jobid, attrdict['parent_uid']))
+ except sqlite.OperationalError, e:
+ print "DB Error: could not access jobs database. Reason: '%s'" % e
+
+ self.dbcx.commit()
+
def _save_job_status(self):
# Write new job status to the database
self._status_updates_lock.acquire()
for uid in self._status_updates.keys():
- self._write_status_to_db(uid, self._status_updates[uid])
+ self._write_job_status_to_db(uid, self._status_updates[uid])
self._status_updates = {}
self._status_updates_lock.release()
+ self._archjob_status_updates_lock.acquire()
+ for jobid in self._archjob_status_updates.keys():
+ self._write_archjob_status_to_db(jobid, self._archjob_status_updates[jobid])
+ self._archjob_status_updates = {}
+ self._archjob_status_updates_lock.release()
+
def _start_new_jobs(self):
self._new_queue_lock.acquire()
@@ -277,6 +333,13 @@
if have_work:
return True
+ self._archjob_status_updates_lock.acquire()
+ if len(self._archjob_status_updates) > 0:
+ have_work = True
+ self._archjob_status_updates_lock.release()
+ if have_work:
+ return True
+
if self.builder_manager.have_work():
return True
More information about the fedora-extras-commits
mailing list